From 47664a7d51f83df0a9e5aa61308d81abcfd993c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Jim=C3=A9nez=20S=C3=A1nchez?= Date: Thu, 10 Apr 2025 14:55:49 +0200 Subject: [PATCH] Provisioning: unit test sync job (#103636) * Split in multiple files * Refactor sync even further * Move more things between RepositoryResources * Add status patcher * Interface for sync functions * Interface for compare function * Add syncer back * Move interfaces * Move execute complete * Return currentRef in syncer * Add repository status test * Add initial sync tests * Fix a couple of spots * Make initial sync tests work * Fix register.go * Add initial tests for sync worker * Finish tests for sync * Add incremental tests * Add TODO * Finish incremental tests * Move folder creation to full sync * Move interfaces * Add initial full sync tests * Update tests * Reshape things * Add changes test * Fix register * Add some tests * Add more tests * Add test * WIP * WIP: delete test * Add more full test * More tests * Add tests for folder creation * Add folder tests full sync * Full coverage full sync * Clean up tests * Add more tests for changes function * Enhance tests for Changes function to cover error scenarios and empty paths - Added test cases for handling empty file paths and folder resources. - Updated error message formatting in the Compare function for clarity. * Add unit tests * Failed initial patch * Add tests failed repository resources * Add test failed getting client * Test for successful and unsuccessful syncs * Add final tests for worker * Fix existing tests * Add missing test * Fix spelling mistake * Fix flake in changes test --- .../apis/provisioning/controller/status.go | 37 + .../provisioning/controller/status_test.go | 130 ++++ .../jobs/job_progress_recorder_mock.go | 78 -- .../apis/provisioning/jobs/progress.go | 9 - pkg/registry/apis/provisioning/jobs/queue.go | 2 - .../apis/provisioning/jobs/sync/changes.go | 41 +- .../provisioning/jobs/sync/changes_test.go | 227 +++++- .../provisioning/jobs/sync/compare_fn_mock.go | 100 +++ .../apis/provisioning/jobs/sync/full.go | 133 ++++ .../jobs/sync/full_sync_fn_mock.go | 93 +++ .../apis/provisioning/jobs/sync/full_test.go | 705 ++++++++++++++++++ .../provisioning/jobs/sync/incremental.go | 111 +++ .../jobs/sync/incremental_sync_fn_mock.go | 92 +++ .../jobs/sync/incremental_test.go | 396 ++++++++++ .../jobs/sync/repository_patch_fn_mock.go | 85 +++ .../apis/provisioning/jobs/sync/sync.go | 62 ++ .../apis/provisioning/jobs/sync/sync_test.go | 180 +++++ .../provisioning/jobs/sync/syncer_mock.go | 104 +++ .../apis/provisioning/jobs/sync/worker.go | 370 ++------- .../provisioning/jobs/sync/worker_test.go | 569 ++++++++++++++ pkg/registry/apis/provisioning/register.go | 14 +- .../provisioning/repository/repository.go | 2 + .../provisioning/repository/versioned_mock.go | 213 ++++++ .../apis/provisioning/resources/repository.go | 38 +- .../resources/repository_resources_mock.go | 455 +++++++++++ pkg/storage/legacysql/dualwrite/utils_test.go | 76 ++ 26 files changed, 3894 insertions(+), 428 deletions(-) create mode 100644 pkg/registry/apis/provisioning/controller/status.go create mode 100644 pkg/registry/apis/provisioning/controller/status_test.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/compare_fn_mock.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/full.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/full_test.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/incremental.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/incremental_test.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/repository_patch_fn_mock.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/sync.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/sync_test.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/syncer_mock.go create mode 100644 pkg/registry/apis/provisioning/jobs/sync/worker_test.go create mode 100644 pkg/registry/apis/provisioning/repository/versioned_mock.go create mode 100644 pkg/storage/legacysql/dualwrite/utils_test.go diff --git a/pkg/registry/apis/provisioning/controller/status.go b/pkg/registry/apis/provisioning/controller/status.go new file mode 100644 index 00000000000..ba014426c17 --- /dev/null +++ b/pkg/registry/apis/provisioning/controller/status.go @@ -0,0 +1,37 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + client "github.com/grafana/grafana/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +type RepositoryStatusPatcher struct { + client client.ProvisioningV0alpha1Interface +} + +func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *RepositoryStatusPatcher { + return &RepositoryStatusPatcher{ + client: client, + } +} + +func (r *RepositoryStatusPatcher) Patch(ctx context.Context, repo *provisioning.Repository, patchOperations []map[string]interface{}) error { + patch, err := json.Marshal(patchOperations) + if err != nil { + return fmt.Errorf("unable to marshal patch data: %w", err) + } + + _, err = r.client.Repositories(repo.Namespace). + Patch(ctx, repo.Name, types.JSONPatchType, patch, metav1.PatchOptions{}, "status") + if err != nil { + return fmt.Errorf("unable to update repo with job status: %w", err) + } + + return nil +} diff --git a/pkg/registry/apis/provisioning/controller/status_test.go b/pkg/registry/apis/provisioning/controller/status_test.go new file mode 100644 index 00000000000..6c9ef409056 --- /dev/null +++ b/pkg/registry/apis/provisioning/controller/status_test.go @@ -0,0 +1,130 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1/fake" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8testing "k8s.io/client-go/testing" +) + +func TestNewRepositoryStatusPatcher(t *testing.T) { + client := &fake.FakeProvisioningV0alpha1{} + patcher := NewRepositoryStatusPatcher(client) + require.NotNil(t, patcher) + require.Equal(t, client, patcher.client) +} + +func TestRepositoryStatusPatcher_Patch(t *testing.T) { + tests := []struct { + name string + repo *provisioning.Repository + patchOperations []map[string]interface{} + reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) + expectedError string + }{ + { + name: "successful patch", + repo: &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + }, + patchOperations: []map[string]interface{}{ + { + "op": "replace", + "path": "/status/health", + "value": map[string]interface{}{ + "healthy": true, + "message": []string{}, + }, + }, + }, + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, &provisioning.Repository{}, nil + }, + }, + { + name: "patch marshal error", + repo: &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + }, + patchOperations: []map[string]interface{}{ + { + "op": "replace", + "path": "/status/health", + "value": make(chan int), // This will cause json.Marshal to fail + }, + }, + expectedError: "unable to marshal patch data: json: unsupported type: chan int", + }, + { + name: "patch request error", + repo: &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + }, + patchOperations: []map[string]interface{}{ + { + "op": "replace", + "path": "/status/health", + "value": map[string]interface{}{ + "healthy": true, + "message": []string{}, + }, + }, + }, + reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("patch request failed") + }, + expectedError: "unable to update repo with job status: patch request failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.FakeProvisioningV0alpha1{ + Fake: &k8testing.Fake{}, + } + + if tt.reactorFunc != nil { + client.AddReactor("patch", "repositories", tt.reactorFunc) + } + + patcher := NewRepositoryStatusPatcher(&client) + err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations) + + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError) + } else { + require.NoError(t, err) + } + + if tt.reactorFunc != nil { + actions := client.Actions() + require.Len(t, actions, 1) + + patchAction := actions[0].(k8testing.PatchAction) + require.Equal(t, "status", patchAction.GetSubresource()) + require.Equal(t, tt.repo.Namespace, patchAction.GetNamespace()) + require.Equal(t, tt.repo.Name, patchAction.GetName()) + + // Verify patch data + expectedPatch, _ := json.Marshal(tt.patchOperations) + require.Equal(t, expectedPatch, patchAction.GetPatch()) + } + }) + } +} diff --git a/pkg/registry/apis/provisioning/jobs/job_progress_recorder_mock.go b/pkg/registry/apis/provisioning/jobs/job_progress_recorder_mock.go index 9f69727655e..48c03742d28 100644 --- a/pkg/registry/apis/provisioning/jobs/job_progress_recorder_mock.go +++ b/pkg/registry/apis/provisioning/jobs/job_progress_recorder_mock.go @@ -69,51 +69,6 @@ func (_c *MockJobProgressRecorder_Complete_Call) RunAndReturn(run func(context.C return _c } -// GetRef provides a mock function with no fields -func (_m *MockJobProgressRecorder) GetRef() string { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for GetRef") - } - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// MockJobProgressRecorder_GetRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRef' -type MockJobProgressRecorder_GetRef_Call struct { - *mock.Call -} - -// GetRef is a helper method to define mock.On call -func (_e *MockJobProgressRecorder_Expecter) GetRef() *MockJobProgressRecorder_GetRef_Call { - return &MockJobProgressRecorder_GetRef_Call{Call: _e.mock.On("GetRef")} -} - -func (_c *MockJobProgressRecorder_GetRef_Call) Run(run func()) *MockJobProgressRecorder_GetRef_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *MockJobProgressRecorder_GetRef_Call) Return(_a0 string) *MockJobProgressRecorder_GetRef_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *MockJobProgressRecorder_GetRef_Call) RunAndReturn(run func() string) *MockJobProgressRecorder_GetRef_Call { - _c.Call.Return(run) - return _c -} - // Record provides a mock function with given fields: ctx, result func (_m *MockJobProgressRecorder) Record(ctx context.Context, result JobResourceResult) { _m.Called(ctx, result) @@ -248,39 +203,6 @@ func (_c *MockJobProgressRecorder_SetMessage_Call) RunAndReturn(run func(context return _c } -// SetRef provides a mock function with given fields: ref -func (_m *MockJobProgressRecorder) SetRef(ref string) { - _m.Called(ref) -} - -// MockJobProgressRecorder_SetRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetRef' -type MockJobProgressRecorder_SetRef_Call struct { - *mock.Call -} - -// SetRef is a helper method to define mock.On call -// - ref string -func (_e *MockJobProgressRecorder_Expecter) SetRef(ref interface{}) *MockJobProgressRecorder_SetRef_Call { - return &MockJobProgressRecorder_SetRef_Call{Call: _e.mock.On("SetRef", ref)} -} - -func (_c *MockJobProgressRecorder_SetRef_Call) Run(run func(ref string)) *MockJobProgressRecorder_SetRef_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) - }) - return _c -} - -func (_c *MockJobProgressRecorder_SetRef_Call) Return() *MockJobProgressRecorder_SetRef_Call { - _c.Call.Return() - return _c -} - -func (_c *MockJobProgressRecorder_SetRef_Call) RunAndReturn(run func(string)) *MockJobProgressRecorder_SetRef_Call { - _c.Run(run) - return _c -} - // SetTotal provides a mock function with given fields: ctx, total func (_m *MockJobProgressRecorder) SetTotal(ctx context.Context, total int) { _m.Called(ctx, total) diff --git a/pkg/registry/apis/provisioning/jobs/progress.go b/pkg/registry/apis/provisioning/jobs/progress.go index 949512ddb0a..7d1e1e50c54 100644 --- a/pkg/registry/apis/provisioning/jobs/progress.go +++ b/pkg/registry/apis/provisioning/jobs/progress.go @@ -38,7 +38,6 @@ type JobResourceResult struct { type jobProgressRecorder struct { started time.Time total int - ref string message string finalMessage string resultCount int @@ -96,14 +95,6 @@ func (r *jobProgressRecorder) SetFinalMessage(ctx context.Context, msg string) { logging.FromContext(ctx).Info("job final message", "message", msg) } -func (r *jobProgressRecorder) SetRef(ref string) { - r.ref = ref -} - -func (r *jobProgressRecorder) GetRef() string { - return r.ref -} - func (r *jobProgressRecorder) SetTotal(ctx context.Context, total int) { r.total = total diff --git a/pkg/registry/apis/provisioning/jobs/queue.go b/pkg/registry/apis/provisioning/jobs/queue.go index 6782a003343..16c5173f3f4 100644 --- a/pkg/registry/apis/provisioning/jobs/queue.go +++ b/pkg/registry/apis/provisioning/jobs/queue.go @@ -22,8 +22,6 @@ type JobProgressRecorder interface { ResetResults() SetFinalMessage(ctx context.Context, msg string) SetMessage(ctx context.Context, msg string) - SetRef(ref string) - GetRef() string SetTotal(ctx context.Context, total int) TooManyErrors() error Complete(ctx context.Context, err error) provisioning.JobStatus diff --git a/pkg/registry/apis/provisioning/jobs/sync/changes.go b/pkg/registry/apis/provisioning/jobs/sync/changes.go index bbc46a93e3d..2e9b789c992 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/changes.go +++ b/pkg/registry/apis/provisioning/jobs/sync/changes.go @@ -1,6 +1,7 @@ package sync import ( + "context" "fmt" "sort" "strings" @@ -20,6 +21,31 @@ type ResourceFileChange struct { Existing *provisioning.ResourceListItem } +func Compare(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error) { + target, err := repositoryResources.List(ctx) + if err != nil { + return nil, fmt.Errorf("error listing current: %w", err) + } + + source, err := repo.ReadTree(ctx, ref) + if err != nil { + return nil, fmt.Errorf("error reading tree: %w", err) + } + + changes, err := Changes(source, target) + if err != nil { + return nil, fmt.Errorf("calculate changes: %w", err) + } + + if len(changes) > 0 { + // FIXME: this is a way to load in different ways the resources + // maybe we can structure the code in better way to avoid this + repositoryResources.SetTree(resources.NewFolderTreeFromResourceList(target)) + } + + return changes, nil +} + func Changes(source []repository.FileTreeEntry, target *provisioning.ResourceList) ([]ResourceFileChange, error) { lookup := make(map[string]*provisioning.ResourceListItem, len(target.Items)) for _, item := range target.Items { @@ -41,6 +67,11 @@ func Changes(source []repository.FileTreeEntry, target *provisioning.ResourceLis keep := safepath.NewTrie() changes := make([]ResourceFileChange, 0, len(source)) for _, file := range source { + // TODO: why do we have to do this here? + if !file.Blob && !strings.HasSuffix(file.Path, "/") { + file.Path = file.Path + "/" + } + check, ok := lookup[file.Path] if ok { if check.Hash != file.Hash && check.Resource != resources.FolderResource.Resource { @@ -113,7 +144,15 @@ func Changes(source []repository.FileTreeEntry, target *provisioning.ResourceLis // Deepest first (stable sort order) sort.Slice(changes, func(i, j int) bool { - return safepath.Depth(changes[i].Path) > safepath.Depth(changes[j].Path) + if safepath.Depth(changes[i].Path) > safepath.Depth(changes[j].Path) { + return true + } + + if safepath.Depth(changes[i].Path) < safepath.Depth(changes[j].Path) { + return false + } + + return changes[i].Path < changes[j].Path }) return changes, nil diff --git a/pkg/registry/apis/provisioning/jobs/sync/changes_test.go b/pkg/registry/apis/provisioning/jobs/sync/changes_test.go index 43b7ffb2e95..2618e8757e2 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/changes_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/changes_test.go @@ -1,12 +1,16 @@ package sync import ( + "context" + "fmt" "testing" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) func TestChanges(t *testing.T) { @@ -39,6 +43,29 @@ func TestChanges(t *testing.T) { Path: "muta.json", }, changes[0]) }) + t.Run("empty file path", func(t *testing.T) { + source := []repository.FileTreeEntry{} + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "", Resource: "dashboard", Group: "dashboard.grafana.app"}, + }, + } + _, err := Changes(source, target) + require.EqualError(t, err, "empty path on a non folder") + }) + + t.Run("empty path with folder resource", func(t *testing.T) { + source := []repository.FileTreeEntry{} + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "", Resource: resources.FolderResource.Resource, Group: resources.FolderResource.Group}, + }, + } + + changes, err := Changes(source, target) + require.NoError(t, err) + require.Empty(t, changes) + }) t.Run("create empty folder structure for folders with unsupported file types", func(t *testing.T) { source := []repository.FileTreeEntry{ @@ -131,7 +158,7 @@ func TestChanges(t *testing.T) { t.Run("folder deletion order", func(t *testing.T) { source := []repository.FileTreeEntry{ - {Path: "x/y/z/ignored.md"}, // ignored + {Path: "x/y/z/ignored.md", Blob: true}, // ignored {Path: "aaa/bbb.yaml", Hash: "xyz", Blob: true}, } target := &provisioning.ResourceList{ @@ -153,8 +180,8 @@ func TestChanges(t *testing.T) { } require.Equal(t, []string{ "zzz/longest/path/here.json", // not sorted yet - "x/y/z/", "x/y/file.json", + "x/y/z/", "short/file.yml", "a.json", }, order) @@ -287,4 +314,200 @@ func TestChanges(t *testing.T) { require.NoError(t, err) require.Empty(t, changes) }) + + t.Run("error on empty path for non-folder resource", func(t *testing.T) { + source := []repository.FileTreeEntry{ + {Path: "", Hash: "xyz", Blob: true}, + } + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "", Resource: "dashboard", Group: "dashboard.grafana.app"}, + }, + } + + changes, err := Changes(source, target) + require.Error(t, err) + require.Contains(t, err.Error(), "empty path on a non folder") + require.Nil(t, changes) + }) + + t.Run("complex nested folder hierarchy with mixed file types", func(t *testing.T) { + source := []repository.FileTreeEntry{ + {Path: "root/folder1/dashboard.json", Hash: "abc", Blob: true}, + {Path: "root/folder1/subfolder/.gitkeep", Hash: "def", Blob: true}, + {Path: "root/folder2/alert.json", Hash: "ghi", Blob: true}, + {Path: "root/folder2/README.md", Hash: "jkl", Blob: true}, + } + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "root/", Resource: "folders"}, + {Path: "root/folder1/", Resource: "folders"}, + {Path: "root/folder2/", Resource: "folders"}, + {Path: "root/folder1/dashboard.json", Hash: "abc", Resource: "dashboard"}, + {Path: "root/folder2/alert.json", Hash: "old", Resource: "alert"}, + }, + } + + changes, err := Changes(source, target) + require.NoError(t, err) + require.Len(t, changes, 2) + require.Equal(t, "root/folder1/subfolder/", changes[0].Path) + require.Equal(t, "root/folder2/alert.json", changes[1].Path) + }) + + t.Run("folder path suffix handling", func(t *testing.T) { + source := []repository.FileTreeEntry{ + {Path: "folder1/", Hash: "abc", Blob: false}, + {Path: "folder2/", Hash: "def", Blob: false}, + {Path: "folder3", Hash: "ghi", Blob: false}, + {Path: "folder4/", Hash: "jkl", Blob: false}, + } + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "folder1", Resource: resources.FolderResource.Resource, Group: resources.FolderResource.Group}, + {Path: "folder2/", Resource: "folders", Group: resources.FolderResource.Group}, + {Path: "folder3", Resource: "folders", Group: resources.FolderResource.Group}, + {Path: "folder4/", Resource: "folders", Group: resources.FolderResource.Group}, + }, + } + + changes, err := Changes(source, target) + require.NoError(t, err) + require.Empty(t, changes, "Should handle folder paths with and without trailing slash") + }) + + t.Run("empty source with populated target", func(t *testing.T) { + source := []repository.FileTreeEntry{} + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "folder1/", Resource: "folders"}, + {Path: "folder1/dashboard.json", Resource: "dashboard"}, + {Path: "folder2/", Resource: "folders"}, + }, + } + + changes, err := Changes(source, target) + require.NoError(t, err) + require.Len(t, changes, 3) + + // Verify deletion order (deepest first) + require.Equal(t, "folder1/dashboard.json", changes[0].Path) + require.Equal(t, "folder1/", changes[1].Path) + require.Equal(t, "folder2/", changes[2].Path) + + for _, change := range changes { + require.Equal(t, repository.FileActionDeleted, change.Action) + } + }) + + t.Run("empty target with populated source", func(t *testing.T) { + source := []repository.FileTreeEntry{ + {Path: "folder1/", Hash: "abc", Blob: false}, + {Path: "folder1/dashboard.json", Hash: "def", Blob: true}, + {Path: "folder2/", Hash: "ghi", Blob: false}, + } + target := &provisioning.ResourceList{} + + changes, err := Changes(source, target) + require.NoError(t, err) + require.Len(t, changes, 3) // Only non-blob entries should create changes + + require.Equal(t, ResourceFileChange{ + Action: repository.FileActionCreated, + Path: "folder1/dashboard.json", + }, changes[0]) + + require.Equal(t, ResourceFileChange{ + Action: repository.FileActionCreated, + Path: "folder1/", + }, changes[1]) + + require.Equal(t, ResourceFileChange{ + Action: repository.FileActionCreated, + Path: "folder2/", + }, changes[2]) + }) +} + +func TestCompare(t *testing.T) { + tests := []struct { + name string + setupMocks func(*repository.MockRepository, *resources.MockRepositoryResources) + expectedError string + expectedChanges []ResourceFileChange + description string + }{ + { + name: "error listing current resources", + description: "Should return error when listing current resources fails", + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) { + repoResources.On("List", mock.Anything).Return(nil, fmt.Errorf("listing failed")) + }, + expectedError: "error listing current: listing failed", + }, + { + name: "error reading tree", + description: "Should return error when reading tree fails", + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) { + repoResources.On("List", mock.Anything).Return(&provisioning.ResourceList{}, nil) + repo.On("ReadTree", mock.Anything, "current-ref").Return(nil, fmt.Errorf("read tree failed")) + }, + expectedError: "error reading tree: read tree failed", + }, + { + name: "no changes between source and target", + description: "Should return empty changes when source and target are identical", + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) { + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + {Path: "dashboard.json", Hash: "xyz"}, + }, + } + source := []repository.FileTreeEntry{ + {Path: "dashboard.json", Hash: "xyz", Blob: true}, + } + + repoResources.On("List", mock.Anything).Return(target, nil) + repo.On("ReadTree", mock.Anything, "current-ref").Return(source, nil) + }, + expectedChanges: []ResourceFileChange{}, + }, + { + name: "compare function error", + description: "Should return error when comparing fails", + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) { + target := &provisioning.ResourceList{ + Items: []provisioning.ResourceListItem{ + // Empty path to trigger error + {Path: "", Hash: "xyz", Resource: "dashboard", Group: "dashboard.grafana.app"}, + }, + } + source := []repository.FileTreeEntry{ + {Path: "dashboard.json", Hash: "xyz", Blob: true}, + } + repoResources.On("List", mock.Anything).Return(target, nil) + repo.On("ReadTree", mock.Anything, "current-ref").Return(source, nil) + }, + expectedError: "calculate changes: empty path on a non folder", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + + tt.setupMocks(repo, repoResources) + + changes, err := Compare(context.Background(), repo, repoResources, "current-ref") + + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError, tt.description) + require.Nil(t, changes) + } else { + require.NoError(t, err, tt.description) + require.Equal(t, tt.expectedChanges, changes, tt.description) + } + }) + } } diff --git a/pkg/registry/apis/provisioning/jobs/sync/compare_fn_mock.go b/pkg/registry/apis/provisioning/jobs/sync/compare_fn_mock.go new file mode 100644 index 00000000000..7cb74e9497a --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/compare_fn_mock.go @@ -0,0 +1,100 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package sync + +import ( + context "context" + + repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + mock "github.com/stretchr/testify/mock" + + resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" +) + +// MockCompareFn is an autogenerated mock type for the CompareFn type +type MockCompareFn struct { + mock.Mock +} + +type MockCompareFn_Expecter struct { + mock *mock.Mock +} + +func (_m *MockCompareFn) EXPECT() *MockCompareFn_Expecter { + return &MockCompareFn_Expecter{mock: &_m.Mock} +} + +// Execute provides a mock function with given fields: ctx, repo, repositoryResources, ref +func (_m *MockCompareFn) Execute(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error) { + ret := _m.Called(ctx, repo, repositoryResources, ref) + + if len(ret) == 0 { + panic("no return value specified for Execute") + } + + var r0 []ResourceFileChange + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, resources.RepositoryResources, string) ([]ResourceFileChange, error)); ok { + return rf(ctx, repo, repositoryResources, ref) + } + if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, resources.RepositoryResources, string) []ResourceFileChange); ok { + r0 = rf(ctx, repo, repositoryResources, ref) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ResourceFileChange) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, repository.Reader, resources.RepositoryResources, string) error); ok { + r1 = rf(ctx, repo, repositoryResources, ref) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCompareFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute' +type MockCompareFn_Execute_Call struct { + *mock.Call +} + +// Execute is a helper method to define mock.On call +// - ctx context.Context +// - repo repository.Reader +// - repositoryResources resources.RepositoryResources +// - ref string +func (_e *MockCompareFn_Expecter) Execute(ctx interface{}, repo interface{}, repositoryResources interface{}, ref interface{}) *MockCompareFn_Execute_Call { + return &MockCompareFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, repositoryResources, ref)} +} + +func (_c *MockCompareFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string)) *MockCompareFn_Execute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(repository.Reader), args[2].(resources.RepositoryResources), args[3].(string)) + }) + return _c +} + +func (_c *MockCompareFn_Execute_Call) Return(_a0 []ResourceFileChange, _a1 error) *MockCompareFn_Execute_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockCompareFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Reader, resources.RepositoryResources, string) ([]ResourceFileChange, error)) *MockCompareFn_Execute_Call { + _c.Call.Return(run) + return _c +} + +// NewMockCompareFn creates a new instance of MockCompareFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockCompareFn(t interface { + mock.TestingT + Cleanup(func()) +}) *MockCompareFn { + mock := &MockCompareFn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/full.go b/pkg/registry/apis/provisioning/jobs/sync/full.go new file mode 100644 index 00000000000..cfd8ac3cee8 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/full.go @@ -0,0 +1,133 @@ +package sync + +import ( + "context" + "errors" + "fmt" + + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func FullSync( + ctx context.Context, + repo repository.Reader, + compare CompareFn, + clients resources.ResourceClients, + currentRef string, + repositoryResources resources.RepositoryResources, + progress jobs.JobProgressRecorder, +) error { + cfg := repo.Config() + + // Ensure the configured folder exists and is managed by the repository + rootFolder := resources.RootFolder(cfg) + if rootFolder != "" { + if err := repositoryResources.EnsureFolderExists(ctx, resources.Folder{ + ID: rootFolder, // will not change if exists + Title: cfg.Spec.Title, + Path: "", // at the root of the repository + }, ""); err != nil { + return fmt.Errorf("create root folder: %w", err) + } + } + + changes, err := compare(ctx, repo, repositoryResources, currentRef) + if err != nil { + return fmt.Errorf("compare changes: %w", err) + } + + if len(changes) == 0 { + progress.SetFinalMessage(ctx, "no changes to sync") + return nil + } + + return applyChanges(ctx, changes, clients, repositoryResources, progress) +} + +func applyChanges(ctx context.Context, changes []ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + progress.SetTotal(ctx, len(changes)) + + for _, change := range changes { + if ctx.Err() != nil { + return ctx.Err() + } + + if err := progress.TooManyErrors(); err != nil { + return err + } + + if change.Action == repository.FileActionDeleted { + result := jobs.JobResourceResult{ + Path: change.Path, + Action: change.Action, + } + + if change.Existing == nil || change.Existing.Name == "" { + result.Error = errors.New("missing existing reference") + progress.Record(ctx, result) + continue + } + + result.Name = change.Existing.Name + result.Resource = change.Existing.Resource + result.Group = change.Existing.Group + + versionlessGVR := schema.GroupVersionResource{ + Group: change.Existing.Group, + Resource: change.Existing.Resource, + } + + // TODO: should we use the clients or the resource manager instead? + client, _, err := clients.ForResource(versionlessGVR) + if err != nil { + result.Error = fmt.Errorf("get client for deleted object: %w", err) + progress.Record(ctx, result) + continue + } + + result.Error = client.Delete(ctx, change.Existing.Name, metav1.DeleteOptions{}) + progress.Record(ctx, result) + continue + } + + // If folder ensure it exists + if safepath.IsDir(change.Path) { + result := jobs.JobResourceResult{ + Path: change.Path, + Action: change.Action, + Resource: resources.FolderResource.Resource, + Group: resources.FolderResource.Group, + } + + folder, err := repositoryResources.EnsureFolderPathExist(ctx, change.Path) + if err != nil { + result.Error = err + progress.Record(ctx, result) + continue + } + + result.Name = folder + progress.Record(ctx, result) + + continue + } + + name, gvk, err := repositoryResources.WriteResourceFromFile(ctx, change.Path, "") + result := jobs.JobResourceResult{ + Path: change.Path, + Action: change.Action, + Name: name, + Error: err, + Resource: gvk.Kind, + Group: gvk.Group, + } + progress.Record(ctx, result) + } + + return nil +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go b/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go new file mode 100644 index 00000000000..29e44c5c02c --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package sync + +import ( + context "context" + + jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + mock "github.com/stretchr/testify/mock" + + repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + + resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" +) + +// MockFullSyncFn is an autogenerated mock type for the FullSyncFn type +type MockFullSyncFn struct { + mock.Mock +} + +type MockFullSyncFn_Expecter struct { + mock *mock.Mock +} + +func (_m *MockFullSyncFn) EXPECT() *MockFullSyncFn_Expecter { + return &MockFullSyncFn_Expecter{mock: &_m.Mock} +} + +// Execute provides a mock function with given fields: ctx, repo, compare, clients, currentRef, repositoryResources, progress +func (_m *MockFullSyncFn) Execute(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + ret := _m.Called(ctx, repo, compare, clients, currentRef, repositoryResources, progress) + + if len(ret) == 0 { + panic("no return value specified for Execute") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder) error); ok { + r0 = rf(ctx, repo, compare, clients, currentRef, repositoryResources, progress) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockFullSyncFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute' +type MockFullSyncFn_Execute_Call struct { + *mock.Call +} + +// Execute is a helper method to define mock.On call +// - ctx context.Context +// - repo repository.Reader +// - compare CompareFn +// - clients resources.ResourceClients +// - currentRef string +// - repositoryResources resources.RepositoryResources +// - progress jobs.JobProgressRecorder +func (_e *MockFullSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, compare interface{}, clients interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}) *MockFullSyncFn_Execute_Call { + return &MockFullSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, compare, clients, currentRef, repositoryResources, progress)} +} + +func (_c *MockFullSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder)) *MockFullSyncFn_Execute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(repository.Reader), args[2].(CompareFn), args[3].(resources.ResourceClients), args[4].(string), args[5].(resources.RepositoryResources), args[6].(jobs.JobProgressRecorder)) + }) + return _c +} + +func (_c *MockFullSyncFn_Execute_Call) Return(_a0 error) *MockFullSyncFn_Execute_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockFullSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder) error) *MockFullSyncFn_Execute_Call { + _c.Call.Return(run) + return _c +} + +// NewMockFullSyncFn creates a new instance of MockFullSyncFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockFullSyncFn(t interface { + mock.TestingT + Cleanup(func()) +}) *MockFullSyncFn { + mock := &MockFullSyncFn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/full_test.go b/pkg/registry/apis/provisioning/jobs/sync/full_test.go new file mode 100644 index 00000000000..7524fe27f2a --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/full_test.go @@ -0,0 +1,705 @@ +package sync + +import ( + "context" + "errors" + "fmt" + "testing" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8testing "k8s.io/client-go/testing" +) + +func TestFullSync_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "Test Repo", + }, + }) + + compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{{}}, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + + err := FullSync(ctx, repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + require.EqualError(t, err, "context canceled") +} + +func TestFullSync_Error(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + }) + + compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("some error")) + + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + require.EqualError(t, err, "compare changes: some error") +} + +func TestFullSync_NoChanges(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + }) + + compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{}, nil) + progress.On("SetFinalMessage", mock.Anything, "no changes to sync").Return() + + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + require.NoError(t, err) +} + +func TestFullSync_SuccessfulFolderCreation(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "Test Repo", + Sync: provisioning.SyncOptions{ + Target: provisioning.SyncTargetTypeFolder, + }, + }, + }) + + compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{}, nil) + progress.On("SetFinalMessage", mock.Anything, "no changes to sync").Return() + repoResources.On("EnsureFolderExists", mock.Anything, resources.Folder{ + ID: "test-repo", + Title: "Test Repo", + Path: "", + }, "").Return(nil) + + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + require.NoError(t, err) +} + +func TestFullSync_FolderCreationFailed(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "Test Repo", + Sync: provisioning.SyncOptions{ + Target: provisioning.SyncTargetTypeFolder, + }, + }, + }) + + repoResources.On("EnsureFolderExists", mock.Anything, resources.Folder{ + ID: "test-repo", + Title: "Test Repo", + Path: "", + }, "").Return(fmt.Errorf("folder creation failed")) + + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + require.Error(t, err) + require.Contains(t, err.Error(), "create root folder: folder creation failed") +} + +func TestFullSync_FolderCreationFailedWithInstanceTarget(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "Test Repo", + Sync: provisioning.SyncOptions{ + Target: provisioning.SyncTargetTypeInstance, + }, + }, + }) + + // No folder creation should be attempted with instance target + // But we should still test the error path for completeness + compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, fmt.Errorf("compare error")) + + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + require.Error(t, err) + require.Contains(t, err.Error(), "compare changes: compare error") +} + +func TestFullSync_ApplyChanges(t *testing.T) { + tests := []struct { + name string + setupMocks func(*repository.MockRepository, *resources.MockRepositoryResources, *resources.MockResourceClients, *jobs.MockJobProgressRecorder, *MockCompareFn) + changes []ResourceFileChange + expectedError string + description string + }{ + { + name: "too many errors", + description: "Should return an error when too many errors occur", + changes: []ResourceFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/one.json", + }, + { + Action: repository.FileActionCreated, + Path: "dashboards/two.json", + }, + { + Action: repository.FileActionCreated, + Path: "dashboards/three.json", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + // First call returns nil, second call returns error + progress.On("TooManyErrors").Return(nil).Once() + progress.On("TooManyErrors").Return(fmt.Errorf("too many errors")).Once() + + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/one.json", ""). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "dashboards/one.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }).Return() + }, + expectedError: "too many errors", + }, + { + name: "successful apply with file creation", + description: "Should successfully apply changes when creating a new file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", ""). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }).Return() + }, + }, + { + name: "failed apply with file creation", + description: "Should record an error when creating a new file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", ""). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write error")) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + Error: fmt.Errorf("write error"), + }).Return() + }, + }, + { + name: "successful apply with file update", + description: "Should successfully apply changes when updating an existing file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionUpdated, + Path: "dashboards/test.json", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", ""). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionUpdated, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }).Return() + }, + }, + { + name: "failed apply with file update", + description: "Should record an error when updating a file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionUpdated, + Path: "dashboards/test.json", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", ""). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write error")) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionUpdated, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + Error: fmt.Errorf("write error"), + }).Return() + }, + }, + { + name: "successful apply with folder creation", + description: "Should successfully apply changes when creating a new folder", + changes: []ResourceFileChange{ + { + Action: repository.FileActionCreated, + Path: "one/two/three/", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + repoResources.On("EnsureFolderPathExist", mock.Anything, "one/two/three/").Return("some-folder", nil) + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "one/two/three/", + Name: "some-folder", + // FIXME: this is probably inconsistent across the codebase + Resource: "folders", + Group: "folder.grafana.app", + }).Return() + }, + }, + { + name: "failed apply folder creation", + description: "Should record an error when creating a new folder", + changes: []ResourceFileChange{ + { + Action: repository.FileActionCreated, + Path: "one/two/three/", + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + repoResources.On( + "EnsureFolderPathExist", + mock.Anything, + "one/two/three/", + ).Return("some-folder", errors.New("folder creation error")) + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "one/two/three/", + Name: "", + // FIXME: this is probably inconsistent across the codebase + Resource: "folders", + Group: "folder.grafana.app", + Error: errors.New("folder creation error"), + }).Return() + }, + }, + { + name: "successful apply with file deletion", + description: "Should successfully apply changes when deleting an existing file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Existing: &provisioning.ResourceListItem{ + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + scheme := runtime.NewScheme() + require.NoError(t, metav1.AddMetaToScheme(scheme)) + listGVK := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: "DashboardList", + } + + scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: resources.DashboardResource.Resource, + }, &metav1.PartialObjectMetadata{}) + + fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + resources.DashboardResource: listGVK.Kind, + }) + + fakeDynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + clients.On("ForResource", schema.GroupVersionResource{ + Group: "dashboards", + Resource: "Dashboard", + }).Return(fakeDynamicClient.Resource(resources.DashboardResource), schema.GroupVersionKind{ + Kind: "Dashboard", + Group: "dashboards", + Version: "v1", + }, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + Error: nil, + }).Return() + }, + }, + { + name: "file delete error", + description: "Should return an error when deleting a file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Existing: &provisioning.ResourceListItem{ + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + scheme := runtime.NewScheme() + require.NoError(t, metav1.AddMetaToScheme(scheme)) + listGVK := schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: "DashboardList", + } + + scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: resources.DashboardResource.Resource, + }, &metav1.PartialObjectMetadata{}) + + fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + resources.DashboardResource: listGVK.Kind, + }) + + fakeDynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("delete failed") + }) + + clients.On("ForResource", schema.GroupVersionResource{ + Group: "dashboards", + Resource: "Dashboard", + }).Return(fakeDynamicClient.Resource(resources.DashboardResource), schema.GroupVersionKind{ + Kind: "Dashboard", + Group: "dashboards", + Version: "v1", + }, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + Error: fmt.Errorf("delete failed"), + }).Return() + }, + }, + { + name: "without existing for delete", + description: "Should record an error when deleting a file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Existing: nil, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Error: fmt.Errorf("missing existing reference"), + }).Return() + }, + }, + { + name: "without existing name for delete", + description: "Should record an error when deleting a file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Existing: &provisioning.ResourceListItem{}, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Error: fmt.Errorf("missing existing reference"), + }).Return() + }, + }, + { + name: "error finding client for delete", + description: "Should record an error when deleting a file", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Existing: &provisioning.ResourceListItem{ + Name: "test-dashboard", + Group: "dashboards", + Resource: "Dashboard", + }, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + clients.On("ForResource", schema.GroupVersionResource{ + Group: "dashboards", + Resource: "Dashboard", + }).Return(nil, schema.GroupVersionKind{}, errors.New("didn't work")) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Name: "test-dashboard", + Group: "dashboards", + Resource: "Dashboard", + Action: repository.FileActionDeleted, + Path: "dashboards/test.json", + Error: fmt.Errorf("get client for deleted object: %w", errors.New("didn't work")), + }).Return() + }, + }, + { + name: "successful apply with folder deletion", + description: "Should successfully apply changes when deleting an existing folder", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "to-be-deleted/", + Existing: &provisioning.ResourceListItem{ + Name: "test-folder", + Resource: "Folder", + Group: "folders", + }, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + scheme := runtime.NewScheme() + require.NoError(t, metav1.AddMetaToScheme(scheme)) + listGVK := schema.GroupVersionKind{ + Group: resources.FolderResource.Group, + Version: resources.FolderResource.Version, + Kind: "FolderList", + } + + scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: resources.FolderResource.Group, + Version: resources.FolderResource.Version, + Kind: resources.FolderResource.Resource, + }, &metav1.PartialObjectMetadata{}) + + fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + resources.FolderResource: listGVK.Kind, + }) + + fakeDynamicClient.PrependReactor("delete", "folders", func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + clients.On("ForResource", schema.GroupVersionResource{ + Group: "folders", + Resource: "Folder", + }).Return(fakeDynamicClient.Resource(resources.FolderResource), schema.GroupVersionKind{ + Kind: "Folder", + Group: "folders", + Version: "v1", + }, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "to-be-deleted/", + Name: "test-folder", + Resource: "Folder", + Group: "folders", + Error: nil, + }).Return() + }, + }, + { + name: "failed to apply with folder deletion", + description: "Should record an error when deleting a folder", + changes: []ResourceFileChange{ + { + Action: repository.FileActionDeleted, + Path: "to-be-deleted/", + Existing: &provisioning.ResourceListItem{ + Name: "test-folder", + Resource: "Folder", + Group: "folders", + }, + }, + }, + setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) { + progress.On("TooManyErrors").Return(nil) + + scheme := runtime.NewScheme() + require.NoError(t, metav1.AddMetaToScheme(scheme)) + listGVK := schema.GroupVersionKind{ + Group: resources.FolderResource.Group, + Version: resources.FolderResource.Version, + Kind: "FolderList", + } + + scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: resources.FolderResource.Group, + Version: resources.FolderResource.Version, + Kind: resources.FolderResource.Resource, + }, &metav1.PartialObjectMetadata{}) + + fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ + resources.FolderResource: listGVK.Kind, + }) + + fakeDynamicClient.PrependReactor("delete", "folders", func(action k8testing.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("delete failed") + }) + + clients.On("ForResource", schema.GroupVersionResource{ + Group: "folders", + Resource: "Folder", + }).Return(fakeDynamicClient.Resource(resources.FolderResource), schema.GroupVersionKind{ + Kind: "Folder", + Group: "folders", + Version: "v1", + }, nil) + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "to-be-deleted/", + Name: "test-folder", + Resource: "Folder", + Group: "folders", + Error: fmt.Errorf("delete failed"), + }).Return() + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := repository.NewMockRepository(t) + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + + tt.setupMocks(repo, repoResources, clients, progress, compareFn) + compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.changes, nil) + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "Test Repo", + }, + }) + + progress.On("SetTotal", mock.Anything, len(tt.changes)).Return() + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError, tt.description) + } else { + require.NoError(t, err, tt.description) + } + }) + } +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/incremental.go b/pkg/registry/apis/provisioning/jobs/sync/incremental.go new file mode 100644 index 00000000000..43aafada339 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/incremental.go @@ -0,0 +1,111 @@ +package sync + +import ( + "context" + "fmt" + + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath" +) + +// Convert git changes into resource file changes +func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + if previousRef == currentRef { + progress.SetFinalMessage(ctx, "same commit as last time") + return nil + } + + diff, err := repo.CompareFiles(ctx, previousRef, currentRef) + if err != nil { + return fmt.Errorf("compare files error: %w", err) + } + + if len(diff) < 1 { + progress.SetFinalMessage(ctx, "no changes detected between commits") + return nil + } + + progress.SetTotal(ctx, len(diff)) + progress.SetMessage(ctx, "replicating versioned changes") + + for _, change := range diff { + if ctx.Err() != nil { + return ctx.Err() + } + if err := progress.TooManyErrors(); err != nil { + return err + } + + if err := resources.IsPathSupported(change.Path); err != nil { + // Maintain the safe segment for empty folders + safeSegment := safepath.SafeSegment(change.Path) + if !safepath.IsDir(safeSegment) { + safeSegment = safepath.Dir(safeSegment) + } + + if safeSegment != "" && resources.IsPathSupported(safeSegment) == nil { + folder, err := repositoryResources.EnsureFolderPathExist(ctx, safeSegment) + if err != nil { + return fmt.Errorf("unable to create empty file folder: %w", err) + } + + progress.Record(ctx, jobs.JobResourceResult{ + Path: safeSegment, + Action: repository.FileActionCreated, + Resource: resources.FolderResource.Resource, + Group: resources.FolderResource.Group, + Name: folder, + }) + + continue + } + + progress.Record(ctx, jobs.JobResourceResult{ + Path: change.Path, + Action: repository.FileActionIgnored, + }) + continue + } + + result := jobs.JobResourceResult{ + Path: change.Path, + Action: change.Action, + } + + switch change.Action { + case repository.FileActionCreated, repository.FileActionUpdated: + name, gvk, err := repositoryResources.WriteResourceFromFile(ctx, change.Path, change.Ref) + if err != nil { + result.Error = err + } + result.Name = name + result.Resource = gvk.Kind + result.Group = gvk.Group + case repository.FileActionDeleted: + name, gvk, err := repositoryResources.RemoveResourceFromFile(ctx, change.Path, change.PreviousRef) + if err != nil { + result.Error = err + } + result.Name = name + result.Resource = gvk.Kind + result.Group = gvk.Group + case repository.FileActionRenamed: + name, gvk, err := repositoryResources.RenameResourceFile(ctx, change.Path, change.PreviousRef, change.Path, change.Ref) + if err != nil { + result.Error = err + } + result.Name = name + result.Resource = gvk.Kind + result.Group = gvk.Group + case repository.FileActionIgnored: + // do nothing + } + progress.Record(ctx, result) + } + + progress.SetMessage(ctx, "versioned changes replicated") + + return nil +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go b/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go new file mode 100644 index 00000000000..cef2b0deb07 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go @@ -0,0 +1,92 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package sync + +import ( + context "context" + + jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + mock "github.com/stretchr/testify/mock" + + repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + + resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" +) + +// MockIncrementalSyncFn is an autogenerated mock type for the IncrementalSyncFn type +type MockIncrementalSyncFn struct { + mock.Mock +} + +type MockIncrementalSyncFn_Expecter struct { + mock *mock.Mock +} + +func (_m *MockIncrementalSyncFn) EXPECT() *MockIncrementalSyncFn_Expecter { + return &MockIncrementalSyncFn_Expecter{mock: &_m.Mock} +} + +// Execute provides a mock function with given fields: ctx, repo, previousRef, currentRef, repositoryResources, progress +func (_m *MockIncrementalSyncFn) Execute(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { + ret := _m.Called(ctx, repo, previousRef, currentRef, repositoryResources, progress) + + if len(ret) == 0 { + panic("no return value specified for Execute") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder) error); ok { + r0 = rf(ctx, repo, previousRef, currentRef, repositoryResources, progress) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockIncrementalSyncFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute' +type MockIncrementalSyncFn_Execute_Call struct { + *mock.Call +} + +// Execute is a helper method to define mock.On call +// - ctx context.Context +// - repo repository.Versioned +// - previousRef string +// - currentRef string +// - repositoryResources resources.RepositoryResources +// - progress jobs.JobProgressRecorder +func (_e *MockIncrementalSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, previousRef interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}) *MockIncrementalSyncFn_Execute_Call { + return &MockIncrementalSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, previousRef, currentRef, repositoryResources, progress)} +} + +func (_c *MockIncrementalSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder)) *MockIncrementalSyncFn_Execute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(repository.Versioned), args[2].(string), args[3].(string), args[4].(resources.RepositoryResources), args[5].(jobs.JobProgressRecorder)) + }) + return _c +} + +func (_c *MockIncrementalSyncFn_Execute_Call) Return(_a0 error) *MockIncrementalSyncFn_Execute_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockIncrementalSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder) error) *MockIncrementalSyncFn_Execute_Call { + _c.Call.Return(run) + return _c +} + +// NewMockIncrementalSyncFn creates a new instance of MockIncrementalSyncFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockIncrementalSyncFn(t interface { + mock.TestingT + Cleanup(func()) +}) *MockIncrementalSyncFn { + mock := &MockIncrementalSyncFn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go b/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go new file mode 100644 index 00000000000..03c75d898e5 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go @@ -0,0 +1,396 @@ +package sync + +import ( + "context" + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestIncrementalSync_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + repo := repository.NewMockVersioned(t) + repoResources := resources.NewMockRepositoryResources(t) + progress := jobs.NewMockJobProgressRecorder(t) + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return([]repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Ref: "new-ref", + }, + }, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + + err := IncrementalSync(ctx, repo, "old-ref", "new-ref", repoResources, progress) + require.EqualError(t, err, "context canceled") +} + +func TestIncrementalSync(t *testing.T) { + tests := []struct { + name string + setupMocks func(*repository.MockVersioned, *resources.MockRepositoryResources, *jobs.MockJobProgressRecorder) + previousRef string + currentRef string + expectedError string + expectedCalls int + expectedFiles []repository.VersionedFileChange + }{ + { + name: "same commit as last time", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + progress.On("SetFinalMessage", mock.Anything, "same commit as last time").Return() + }, + previousRef: "old-ref", + currentRef: "old-ref", + }, + { + name: "no changes between commits", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return([]repository.VersionedFileChange{}, nil) + progress.On("SetFinalMessage", mock.Anything, "no changes detected between commits").Return() + }, + previousRef: "old-ref", + currentRef: "new-ref", + }, + { + name: "error comparing files", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(nil, fmt.Errorf("compare error")) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedError: "compare files error: compare error", + }, + { + name: "successful sync with file changes", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Ref: "new-ref", + }, + { + Action: repository.FileActionUpdated, + Path: "alerts/alert.yaml", + Ref: "new-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 2).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + // Mock successful resource writes + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "new-ref"). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil) + repoResources.On("WriteResourceFromFile", mock.Anything, "alerts/alert.yaml", "new-ref"). + Return("test-alert", schema.GroupVersionKind{Kind: "Alert", Group: "alerts"}, nil) + + // Mock progress recording + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Action == repository.FileActionCreated && result.Path == "dashboards/test.json" + })).Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Action == repository.FileActionUpdated && result.Path == "alerts/alert.yaml" + })).Return() + + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 2, + }, + { + name: "unsupported file path with valid folder", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: "unsupported/path/file.txt", + Ref: "new-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + // Mock folder creation + repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/path/"). + Return("test-folder", nil) + + // Mock progress recording + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "unsupported/path/", + Resource: resources.FolderResource.Resource, + Group: resources.FolderResource.Group, + Name: "test-folder", + }).Return() + + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "unsupported file path with invalid folder", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: ".unsupported/path/file.txt", + Ref: "new-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionIgnored, + Path: ".unsupported/path/file.txt", + }).Return() + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "file deletion", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/old.json", + PreviousRef: "old-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + // Mock resource deletion + repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/old.json", "old-ref"). + Return("old-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil) + + // Mock progress recording + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "dashboards/old.json", + Name: "old-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }).Return() + + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "file rename", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionRenamed, + Path: "dashboards/new.json", + PreviousPath: "dashboards/old.json", + Ref: "new-ref", + PreviousRef: "old-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + // Mock resource rename + repoResources.On("RenameResourceFile", mock.Anything, "dashboards/new.json", "old-ref", "dashboards/new.json", "new-ref"). + Return("renamed-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil) + + // Mock progress recording + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionRenamed, + Path: "dashboards/new.json", + Name: "renamed-dashboard", + Resource: "Dashboard", + Group: "dashboards", + }).Return() + + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "file ignored", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionIgnored, + Path: "dashboards/ignored.json", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionIgnored, + Path: "dashboards/ignored.json", + }).Return() + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "error creating folder", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: "unsupported/path/file.txt", + Ref: "new-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + + // Mock folder creation error + repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/path/"). + Return("", fmt.Errorf("failed to create folder")) + + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedError: "unable to create empty file folder: failed to create folder", + }, + { + name: "error writing resource", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Ref: "new-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + // Mock resource write error + repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "new-ref"). + Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write failed")) + + // Mock progress recording with error + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Name: "test-dashboard", + Resource: "Dashboard", + Group: "dashboards", + Error: fmt.Errorf("write failed"), + }).Return() + + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "error deleting resource", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionDeleted, + Path: "dashboards/old.json", + PreviousRef: "old-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return() + + // Mock resource deletion error + repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/old.json", "old-ref"). + Return("old-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("delete failed")) + + // Mock progress recording with error + progress.On("Record", mock.Anything, jobs.JobResourceResult{ + Action: repository.FileActionDeleted, + Path: "dashboards/old.json", + Name: "old-dashboard", + Resource: "Dashboard", + Group: "dashboards", + Error: fmt.Errorf("delete failed"), + }).Return() + progress.On("TooManyErrors").Return(nil) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedCalls: 1, + }, + { + name: "too many errors", + setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) { + changes := []repository.VersionedFileChange{ + { + Action: repository.FileActionCreated, + Path: "dashboards/test.json", + Ref: "new-ref", + }, + } + repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil) + progress.On("SetTotal", mock.Anything, 1).Return() + progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() + // Mock too many errors + progress.On("TooManyErrors").Return(fmt.Errorf("too many errors occurred")) + }, + previousRef: "old-ref", + currentRef: "new-ref", + expectedError: "too many errors occurred", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := repository.NewMockVersioned(t) + repoResources := resources.NewMockRepositoryResources(t) + progress := jobs.NewMockJobProgressRecorder(t) + + tt.setupMocks(repo, repoResources, progress) + + err := IncrementalSync(context.Background(), repo, tt.previousRef, tt.currentRef, repoResources, progress) + + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/repository_patch_fn_mock.go b/pkg/registry/apis/provisioning/jobs/sync/repository_patch_fn_mock.go new file mode 100644 index 00000000000..f17e4cadf06 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/repository_patch_fn_mock.go @@ -0,0 +1,85 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package sync + +import ( + context "context" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + mock "github.com/stretchr/testify/mock" +) + +// MockRepositoryPatchFn is an autogenerated mock type for the RepositoryPatchFn type +type MockRepositoryPatchFn struct { + mock.Mock +} + +type MockRepositoryPatchFn_Expecter struct { + mock *mock.Mock +} + +func (_m *MockRepositoryPatchFn) EXPECT() *MockRepositoryPatchFn_Expecter { + return &MockRepositoryPatchFn_Expecter{mock: &_m.Mock} +} + +// Execute provides a mock function with given fields: ctx, repo, ops +func (_m *MockRepositoryPatchFn) Execute(ctx context.Context, repo *v0alpha1.Repository, ops []map[string]interface{}) error { + ret := _m.Called(ctx, repo, ops) + + if len(ret) == 0 { + panic("no return value specified for Execute") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Repository, []map[string]interface{}) error); ok { + r0 = rf(ctx, repo, ops) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockRepositoryPatchFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute' +type MockRepositoryPatchFn_Execute_Call struct { + *mock.Call +} + +// Execute is a helper method to define mock.On call +// - ctx context.Context +// - repo *v0alpha1.Repository +// - ops []map[string]interface{} +func (_e *MockRepositoryPatchFn_Expecter) Execute(ctx interface{}, repo interface{}, ops interface{}) *MockRepositoryPatchFn_Execute_Call { + return &MockRepositoryPatchFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, ops)} +} + +func (_c *MockRepositoryPatchFn_Execute_Call) Run(run func(ctx context.Context, repo *v0alpha1.Repository, ops []map[string]interface{})) *MockRepositoryPatchFn_Execute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*v0alpha1.Repository), args[2].([]map[string]interface{})) + }) + return _c +} + +func (_c *MockRepositoryPatchFn_Execute_Call) Return(_a0 error) *MockRepositoryPatchFn_Execute_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockRepositoryPatchFn_Execute_Call) RunAndReturn(run func(context.Context, *v0alpha1.Repository, []map[string]interface{}) error) *MockRepositoryPatchFn_Execute_Call { + _c.Call.Return(run) + return _c +} + +// NewMockRepositoryPatchFn creates a new instance of MockRepositoryPatchFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRepositoryPatchFn(t interface { + mock.TestingT + Cleanup(func()) +}) *MockRepositoryPatchFn { + mock := &MockRepositoryPatchFn{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/sync.go b/pkg/registry/apis/provisioning/jobs/sync/sync.go new file mode 100644 index 00000000000..9aca33ebcd3 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/sync.go @@ -0,0 +1,62 @@ +package sync + +import ( + "context" + "fmt" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" +) + +//go:generate mockery --name FullSyncFn --structname MockFullSyncFn --inpackage --filename full_sync_fn_mock.go --with-expecter +type FullSyncFn func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error + +//go:generate mockery --name CompareFn --structname MockCompareFn --inpackage --filename compare_fn_mock.go --with-expecter +type CompareFn func(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error) + +//go:generate mockery --name IncrementalSyncFn --structname MockIncrementalSyncFn --inpackage --filename incremental_sync_fn_mock.go --with-expecter +type IncrementalSyncFn func(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error + +//go:generate mockery --name Syncer --structname MockSyncer --inpackage --filename syncer_mock.go --with-expecter +type Syncer interface { + Sync(ctx context.Context, repo repository.ReaderWriter, options provisioning.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder) (string, error) +} + +type syncer struct { + compare CompareFn + fullSync FullSyncFn + incrementalSync IncrementalSyncFn +} + +func NewSyncer(compare CompareFn, fullSync FullSyncFn, incrementalSync IncrementalSyncFn) Syncer { + return &syncer{ + compare: compare, + fullSync: fullSync, + incrementalSync: incrementalSync, + } +} + +func (r *syncer) Sync(ctx context.Context, repo repository.ReaderWriter, options provisioning.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder) (string, error) { + cfg := repo.Config() + + var currentRef string + versionedRepo, ok := repo.(repository.Versioned) + if ok && versionedRepo != nil { + var err error + currentRef, err = versionedRepo.LatestRef(ctx) + if err != nil { + return "", fmt.Errorf("get latest ref: %w", err) + } + + if cfg.Status.Sync.LastRef != "" && options.Incremental { + progress.SetMessage(ctx, "incremental sync") + return currentRef, r.incrementalSync(ctx, versionedRepo, cfg.Status.Sync.LastRef, currentRef, repositoryResources, progress) + } + } + + progress.SetMessage(ctx, "full sync") + + return currentRef, r.fullSync(ctx, repo, r.compare, clients, currentRef, repositoryResources, progress) +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/sync_test.go b/pkg/registry/apis/provisioning/jobs/sync/sync_test.go new file mode 100644 index 00000000000..943e32da35c --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/sync_test.go @@ -0,0 +1,180 @@ +package sync + +import ( + "context" + "fmt" + "testing" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + mock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type mockReaderWriter struct { + *repository.MockRepository + *repository.MockVersioned +} + +// FIXME: understand how the MockRepository was generated as it seems +// stale and it's causing collisions for the embedded +func (m *mockReaderWriter) History(ctx context.Context, path, ref string) ([]provisioning.HistoryItem, error) { + return m.MockVersioned.History(ctx, path, ref) +} + +func (m *mockReaderWriter) LatestRef(ctx context.Context) (string, error) { + return m.MockVersioned.LatestRef(ctx) +} + +func (m *mockReaderWriter) CompareFiles(ctx context.Context, base, ref string) ([]repository.VersionedFileChange, error) { + return m.MockVersioned.CompareFiles(ctx, base, ref) +} + +func TestSyncer_Sync(t *testing.T) { + tests := []struct { + name string + options provisioning.SyncJobOptions + setupMocks func(*mockReaderWriter, *resources.MockRepositoryResources, *resources.MockResourceClients, *jobs.MockJobProgressRecorder, *MockCompareFn, *MockFullSyncFn, *MockIncrementalSyncFn) + expectedRef string + expectedError string + expectedMessages []string + expectedFinalMsg string + }{ + { + name: "successful full sync", + options: provisioning.SyncJobOptions{ + Incremental: false, + }, + setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) { + repo.MockRepository.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "Test Repo", + }, + }) + repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil) + + progress.On("SetMessage", mock.Anything, "full sync").Return() + fullSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, mock.Anything, mock.Anything, "new-ref", mock.Anything, mock.Anything).Return(nil) + }, + expectedMessages: []string{"full sync"}, + }, + { + name: "successful incremental sync", + options: provisioning.SyncJobOptions{ + Incremental: true, + }, + setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) { + repo.MockRepository.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "old-ref", + }, + }, + }) + repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil) + progress.On("SetMessage", mock.Anything, "incremental sync").Return() + incrementalSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything).Return(nil) + }, + expectedRef: "new-ref", + expectedMessages: []string{"incremental sync"}, + }, + { + name: "latest ref error", + options: provisioning.SyncJobOptions{ + Incremental: true, + }, + setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) { + repo.MockRepository.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "old-ref", + }, + }, + }) + repo.MockVersioned.On("LatestRef", mock.Anything).Return("", fmt.Errorf("failed to get latest ref")) + }, + expectedError: "get latest ref: failed to get latest ref", + }, + { + name: "incremental sync error", + options: provisioning.SyncJobOptions{ + Incremental: true, + }, + setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) { + repo.MockRepository.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "old-ref", + }, + }, + }) + repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil) + progress.On("SetMessage", mock.Anything, "incremental sync").Return() + incrementalSyncFn.On("Execute", mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything).Return(fmt.Errorf("incremental sync failed")) + }, + expectedRef: "new-ref", + expectedMessages: []string{"incremental sync"}, + expectedError: "incremental sync failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repoResources := resources.NewMockRepositoryResources(t) + clients := resources.NewMockResourceClients(t) + progress := jobs.NewMockJobProgressRecorder(t) + compareFn := NewMockCompareFn(t) + fullSyncFn := NewMockFullSyncFn(t) + incrementalSyncFn := NewMockIncrementalSyncFn(t) + + repo := &mockReaderWriter{ + MockRepository: repository.NewMockRepository(t), + MockVersioned: repository.NewMockVersioned(t), + } + + tt.setupMocks(repo, repoResources, clients, progress, compareFn, fullSyncFn, incrementalSyncFn) + + syncer := NewSyncer( + compareFn.Execute, + fullSyncFn.Execute, + incrementalSyncFn.Execute, + ) + + ref, err := syncer.Sync(context.Background(), repo, tt.options, repoResources, clients, progress) + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError) + } else { + require.NoError(t, err) + if tt.expectedRef != "" { + require.Equal(t, tt.expectedRef, ref) + } + } + + // Verify expected messages were set + if len(tt.expectedMessages) > 0 { + for _, msg := range tt.expectedMessages { + progress.AssertCalled(t, "SetMessage", mock.Anything, msg) + } + } + + if tt.expectedFinalMsg != "" { + progress.AssertCalled(t, "SetFinalMessage", mock.Anything, tt.expectedFinalMsg) + } + }) + } +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/syncer_mock.go b/pkg/registry/apis/provisioning/jobs/sync/syncer_mock.go new file mode 100644 index 00000000000..79ac697436d --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/syncer_mock.go @@ -0,0 +1,104 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package sync + +import ( + context "context" + + jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + mock "github.com/stretchr/testify/mock" + + repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + + resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" +) + +// MockSyncer is an autogenerated mock type for the Syncer type +type MockSyncer struct { + mock.Mock +} + +type MockSyncer_Expecter struct { + mock *mock.Mock +} + +func (_m *MockSyncer) EXPECT() *MockSyncer_Expecter { + return &MockSyncer_Expecter{mock: &_m.Mock} +} + +// Sync provides a mock function with given fields: ctx, repo, options, repositoryResources, clients, progress +func (_m *MockSyncer) Sync(ctx context.Context, repo repository.ReaderWriter, options v0alpha1.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder) (string, error) { + ret := _m.Called(ctx, repo, options, repositoryResources, clients, progress) + + if len(ret) == 0 { + panic("no return value specified for Sync") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) (string, error)); ok { + return rf(ctx, repo, options, repositoryResources, clients, progress) + } + if rf, ok := ret.Get(0).(func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) string); ok { + r0 = rf(ctx, repo, options, repositoryResources, clients, progress) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) error); ok { + r1 = rf(ctx, repo, options, repositoryResources, clients, progress) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockSyncer_Sync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Sync' +type MockSyncer_Sync_Call struct { + *mock.Call +} + +// Sync is a helper method to define mock.On call +// - ctx context.Context +// - repo repository.ReaderWriter +// - options v0alpha1.SyncJobOptions +// - repositoryResources resources.RepositoryResources +// - clients resources.ResourceClients +// - progress jobs.JobProgressRecorder +func (_e *MockSyncer_Expecter) Sync(ctx interface{}, repo interface{}, options interface{}, repositoryResources interface{}, clients interface{}, progress interface{}) *MockSyncer_Sync_Call { + return &MockSyncer_Sync_Call{Call: _e.mock.On("Sync", ctx, repo, options, repositoryResources, clients, progress)} +} + +func (_c *MockSyncer_Sync_Call) Run(run func(ctx context.Context, repo repository.ReaderWriter, options v0alpha1.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder)) *MockSyncer_Sync_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(repository.ReaderWriter), args[2].(v0alpha1.SyncJobOptions), args[3].(resources.RepositoryResources), args[4].(resources.ResourceClients), args[5].(jobs.JobProgressRecorder)) + }) + return _c +} + +func (_c *MockSyncer_Sync_Call) Return(_a0 string, _a1 error) *MockSyncer_Sync_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockSyncer_Sync_Call) RunAndReturn(run func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) (string, error)) *MockSyncer_Sync_Call { + _c.Call.Return(run) + return _c +} + +// NewMockSyncer creates a new instance of MockSyncer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockSyncer(t interface { + mock.TestingT + Cleanup(func()) +}) *MockSyncer { + mock := &MockSyncer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker.go b/pkg/registry/apis/provisioning/jobs/sync/worker.go index 07915493859..d68b17c60ee 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker.go @@ -2,56 +2,51 @@ package sync import ( "context" - "encoding/json" - "errors" "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" - client "github.com/grafana/grafana/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" - "github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" ) +//go:generate mockery --name RepositoryPatchFn --structname MockRepositoryPatchFn --inpackage --filename repository_patch_fn_mock.go --with-expecter +type RepositoryPatchFn func(ctx context.Context, repo *provisioning.Repository, ops []map[string]interface{}) error + // SyncWorker synchronizes the external repo with grafana database // this function updates the status for both the job and the referenced repository type SyncWorker struct { - // Used to update the repository status with sync info - client client.ProvisioningV0alpha1Interface - - // Lists the values saved in grafana database - lister resources.ResourceLister - - // Parses fields saved in remore repository - parsers resources.ParserFactory - // Clients for the repository clients resources.ClientFactory + // ResourceClients for the repository + repositoryResources resources.RepositoryResourcesFactory + // Check if the system is using unified storage storageStatus dualwrite.Service + + // Patch status for the repository + patchStatus RepositoryPatchFn + + // Sync functions + syncer Syncer } func NewSyncWorker( - client client.ProvisioningV0alpha1Interface, - parsers resources.ParserFactory, clients resources.ClientFactory, - lister resources.ResourceLister, + repositoryResources resources.RepositoryResourcesFactory, storageStatus dualwrite.Service, + patchStatus RepositoryPatchFn, + syncer Syncer, ) *SyncWorker { return &SyncWorker{ - client: client, - parsers: parsers, - clients: clients, - lister: lister, - storageStatus: storageStatus, + clients: clients, + repositoryResources: repositoryResources, + patchStatus: patchStatus, + storageStatus: storageStatus, + syncer: syncer, } } @@ -74,7 +69,8 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo syncStatus := job.Status.ToSyncStatus(job.Name) // Preserve last ref as we use replace operation - syncStatus.LastRef = repo.Config().Status.Sync.LastRef + lastRef := repo.Config().Status.Sync.LastRef + syncStatus.LastRef = lastRef // Update sync status at start using JSON patch patchOperations := []map[string]interface{}{ @@ -90,19 +86,26 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo return fmt.Errorf("update repo with job status at start: %w", err) } - progress.SetMessage(ctx, "execute sync job") - syncJob, err := r.createJob(ctx, rw, progress) + repositoryResources, err := r.repositoryResources.Client(ctx, rw) if err != nil { - return fmt.Errorf("failed to create sync job: %w", err) + return fmt.Errorf("create repository resources client: %w", err) } - syncError := syncJob.run(ctx, *job.Spec.Pull) + clients, err := r.clients.Clients(ctx, cfg.Namespace) + if err != nil { + return fmt.Errorf("get clients for %s: %w", cfg.Name, err) + } + + progress.SetMessage(ctx, "execute sync job") + currentRef, syncError := r.syncer.Sync(ctx, rw, *job.Spec.Pull, repositoryResources, clients, progress) jobStatus := progress.Complete(ctx, syncError) syncStatus = jobStatus.ToSyncStatus(job.Name) // Create sync status and set hash if successful if syncStatus.State == provisioning.JobStateSuccess { - syncStatus.LastRef = progress.GetRef() + syncStatus.LastRef = currentRef + } else { + syncStatus.LastRef = lastRef } // Update final status using JSON patch @@ -116,14 +119,20 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo } // Only add stats patch if stats are not nil - if stats, err := r.lister.Stats(ctx, cfg.Namespace, cfg.Name); err != nil { + stats, err := repositoryResources.Stats(ctx) + switch { + case err != nil: logger.Error("unable to read stats", "error", err) - } else if stats != nil && len(stats.Managed) == 1 { + case stats == nil: + logger.Error("stats are nil") + case len(stats.Managed) == 1: patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/stats", "value": stats.Managed[0].Stats, }) + default: + logger.Warn("unexpected number of managed stats", "count", len(stats.Managed)) } // Only patch the specific fields we want to update, not the entire status @@ -133,298 +142,3 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo return syncError } - -// start a job and run it -func (r *SyncWorker) createJob(ctx context.Context, repo repository.ReaderWriter, progress jobs.JobProgressRecorder) (*syncJob, error) { - cfg := repo.Config() - parser, err := r.parsers.GetParser(ctx, repo) - if err != nil { - return nil, fmt.Errorf("failed to get parser for %s: %w", cfg.Name, err) - } - - clients, err := r.clients.Clients(ctx, cfg.Namespace) - if err != nil { - return nil, fmt.Errorf("failed to get clients for %s: %w", cfg.Name, err) - } - - folderClient, err := clients.Folder() - if err != nil { - return nil, fmt.Errorf("unable to get folder client: %w", err) - } - - folders := resources.NewFolderManager(repo, folderClient, resources.NewEmptyFolderTree()) - job := &syncJob{ - repository: repo, - progress: progress, - lister: r.lister, - folders: folders, - clients: clients, - resourceManager: resources.NewResourcesManager(repo, folders, parser, clients, nil), - } - - return job, nil -} - -func (r *SyncWorker) patchStatus(ctx context.Context, repo *provisioning.Repository, patchOperations []map[string]interface{}) error { - patch, err := json.Marshal(patchOperations) - if err != nil { - return fmt.Errorf("unable to marshal patch data: %w", err) - } - - _, err = r.client.Repositories(repo.Namespace). - Patch(ctx, repo.Name, types.JSONPatchType, patch, metav1.PatchOptions{}, "status") - if err != nil { - return fmt.Errorf("unable to update repo with job status: %w", err) - } - - return nil -} - -// created once for each sync execution -type syncJob struct { - repository repository.Reader - progress jobs.JobProgressRecorder - lister resources.ResourceLister - clients resources.ResourceClients - folders *resources.FolderManager - resourceManager *resources.ResourcesManager -} - -func (r *syncJob) run(ctx context.Context, options provisioning.SyncJobOptions) error { - // Ensure the configured folder exists and is managed by the repository - cfg := r.repository.Config() - rootFolder := resources.RootFolder(cfg) - if rootFolder != "" { - if err := r.folders.EnsureFolderExists(ctx, resources.Folder{ - ID: rootFolder, // will not change if exists - Title: cfg.Spec.Title, - Path: "", // at the root of the repository - }, ""); err != nil { - return fmt.Errorf("unable to create root folder: %w", err) - } - } - - var err error - var currentRef string - - versionedRepo, _ := r.repository.(repository.Versioned) - if versionedRepo != nil { - currentRef, err = versionedRepo.LatestRef(ctx) - if err != nil { - return fmt.Errorf("getting latest ref: %w", err) - } - r.progress.SetRef(currentRef) - - if cfg.Status.Sync.LastRef != "" && options.Incremental { - if currentRef == cfg.Status.Sync.LastRef { - r.progress.SetFinalMessage(ctx, "same commit as last sync") - return nil - } - - return r.applyVersionedChanges(ctx, versionedRepo, cfg.Status.Sync.LastRef, currentRef) - } - } - - // Read the complete change set - target, err := r.lister.List(ctx, cfg.Namespace, cfg.Name) - if err != nil { - return fmt.Errorf("error listing current: %w", err) - } - source, err := r.repository.ReadTree(ctx, currentRef) - if err != nil { - return fmt.Errorf("error reading tree: %w", err) - } - changes, err := Changes(source, target) - if err != nil { - return fmt.Errorf("error calculating changes: %w", err) - } - - if len(changes) == 0 { - r.progress.SetFinalMessage(ctx, "no changes to sync") - return nil - } - - r.folders.SetTree(resources.NewFolderTreeFromResourceList(target)) - - // Now apply the changes - return r.applyChanges(ctx, changes) -} - -func (r *syncJob) applyChanges(ctx context.Context, changes []ResourceFileChange) error { - r.progress.SetTotal(ctx, len(changes)) - r.progress.SetMessage(ctx, "replicating changes") - - for _, change := range changes { - if ctx.Err() != nil { - return ctx.Err() - } - if err := r.progress.TooManyErrors(); err != nil { - return err - } - - if change.Action == repository.FileActionDeleted { - result := jobs.JobResourceResult{ - Name: change.Existing.Name, - Resource: change.Existing.Resource, - Group: change.Existing.Group, - Path: change.Path, - Action: change.Action, - } - - if change.Existing == nil || change.Existing.Name == "" { - result.Error = errors.New("missing existing reference") - r.progress.Record(ctx, result) - continue - } - - versionlessGVR := schema.GroupVersionResource{ - Group: change.Existing.Group, - Resource: change.Existing.Resource, - } - - // TODO: should we use the clients or the resource manager instead? - client, _, err := r.clients.ForResource(versionlessGVR) - if err != nil { - result.Error = fmt.Errorf("unable to get client for deleted object: %w", err) - r.progress.Record(ctx, result) - continue - } - - result.Error = client.Delete(ctx, change.Existing.Name, metav1.DeleteOptions{}) - r.progress.Record(ctx, result) - continue - } - - // If folder ensure it exists - if safepath.IsDir(change.Path) { - result := jobs.JobResourceResult{ - Path: change.Path, - Action: change.Action, - } - - folder, err := r.folders.EnsureFolderPathExist(ctx, change.Path) - if err != nil { - result.Error = fmt.Errorf("create folder: %w", err) - r.progress.Record(ctx, result) - continue - } - - result.Name = folder - result.Resource = resources.FolderResource.Resource - result.Group = resources.FolderResource.Group - r.progress.Record(ctx, result) - - continue - } - - name, gvk, err := r.resourceManager.WriteResourceFromFile(ctx, change.Path, "") - result := jobs.JobResourceResult{ - Path: change.Path, - Action: change.Action, - Name: name, - Error: err, - Resource: gvk.Kind, - Group: gvk.Group, - } - r.progress.Record(ctx, result) - } - - r.progress.SetMessage(ctx, "changes replicated") - - return nil -} - -// Convert git changes into resource file changes -func (r *syncJob) applyVersionedChanges(ctx context.Context, repo repository.Versioned, previousRef, currentRef string) error { - diff, err := repo.CompareFiles(ctx, previousRef, currentRef) - if err != nil { - return fmt.Errorf("compare files error: %w", err) - } - - if len(diff) < 1 { - r.progress.SetFinalMessage(ctx, "no changes detected between commits") - return nil - } - - r.progress.SetTotal(ctx, len(diff)) - r.progress.SetMessage(ctx, "replicating versioned changes") - - for _, change := range diff { - if ctx.Err() != nil { - return ctx.Err() - } - if err := r.progress.TooManyErrors(); err != nil { - return err - } - - if err := resources.IsPathSupported(change.Path); err != nil { - // Maintain the safe segment for empty folders - safeSegment := safepath.SafeSegment(change.Path) - if !safepath.IsDir(safeSegment) { - safeSegment = safepath.Dir(safeSegment) - } - - if safeSegment != "" && resources.IsPathSupported(safeSegment) == nil { - folder, err := r.folders.EnsureFolderPathExist(ctx, safeSegment) - if err != nil { - return fmt.Errorf("unable to create empty file folder: %w", err) - } - - r.progress.Record(ctx, jobs.JobResourceResult{ - Path: safeSegment, - Action: repository.FileActionCreated, - Resource: resources.FolderResource.Resource, - Group: resources.FolderResource.Group, - Name: folder, - }) - - continue - } - - r.progress.Record(ctx, jobs.JobResourceResult{ - Path: change.Path, - Action: repository.FileActionIgnored, - }) - continue - } - - result := jobs.JobResourceResult{ - Path: change.Path, - Action: change.Action, - } - - switch change.Action { - case repository.FileActionCreated, repository.FileActionUpdated: - name, gvk, err := r.resourceManager.WriteResourceFromFile(ctx, change.Path, change.Ref) - if err != nil { - result.Error = fmt.Errorf("write resource: %w", err) - } - result.Name = name - result.Resource = gvk.Kind - result.Group = gvk.Group - case repository.FileActionDeleted: - name, gvk, err := r.resourceManager.RemoveResourceFromFile(ctx, change.Path, change.PreviousRef) - if err != nil { - result.Error = fmt.Errorf("delete resource: %w", err) - } - result.Name = name - result.Resource = gvk.Kind - result.Group = gvk.Group - case repository.FileActionRenamed: - name, gvk, err := r.resourceManager.RenameResourceFile(ctx, change.Path, change.PreviousRef, change.Path, change.Ref) - if err != nil { - result.Error = fmt.Errorf("rename resource: %w", err) - } - result.Name = name - result.Resource = gvk.Kind - result.Group = gvk.Group - case repository.FileActionIgnored: - // do nothing - } - r.progress.Record(ctx, result) - } - - r.progress.SetMessage(ctx, "versioned changes replicated") - - return nil -} diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go new file mode 100644 index 00000000000..f351173467e --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go @@ -0,0 +1,569 @@ +package sync + +import ( + "context" + "errors" + "testing" + + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSyncWorker_IsSupported(t *testing.T) { + tests := []struct { + name string + job provisioning.Job + expected bool + }{ + { + name: "pull action is supported", + job: provisioning.Job{ + Spec: provisioning.JobSpec{ + Action: provisioning.JobActionPull, + }, + }, + expected: true, + }, + { + name: "non-pull action is not supported", + job: provisioning.Job{ + Spec: provisioning.JobSpec{ + Action: provisioning.JobActionPush, + }, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + worker := NewSyncWorker(nil, nil, nil, nil, nil) + result := worker.IsSupported(context.Background(), tt.job) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestSyncWorker_ProcessNotReaderWriter(t *testing.T) { + repo := repository.NewMockReader(t) + repo.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "test-repo", + }, + }) + fakeDualwrite := dualwrite.NewMockService(t) + fakeDualwrite.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil) + err := worker.Process(context.Background(), repo, provisioning.Job{}, jobs.NewMockJobProgressRecorder(t)) + require.EqualError(t, err, "sync job submitted for repository that does not support read-write -- this is a bug") +} + +func TestSyncWorker_Process(t *testing.T) { + tests := []struct { + name string + setupMocks func(*resources.MockClientFactory, *resources.MockRepositoryResourcesFactory, *dualwrite.MockService, *MockRepositoryPatchFn, *MockSyncer, *mockReaderWriter, *jobs.MockJobProgressRecorder) + expectedError string + expectedStatus *provisioning.SyncStatus + }{ + { + name: "legacy storage not migrated", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + rw.MockRepository.On("Config").Return(&provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "test-repo", + }, + }) + + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(false, nil).Twice() + }, + expectedError: "sync not supported until storage has migrated", + }, + { + name: "failed initial status patching", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + // Setup repository config with existing LastRef + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "test-repo", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "existing-ref", + }, + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + pr.On("SetMessage", mock.Anything, "update sync status at start").Return() + + rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool { + if len(patch) != 1 { + return false + } + + if patch[0]["op"] != "replace" || patch[0]["path"] != "/status/sync" { + return false + } + + if patch[0]["value"].(provisioning.SyncStatus).LastRef != "existing-ref" || patch[0]["value"].(provisioning.SyncStatus).JobID != "test-job" { + return false + } + + return true + })).Return(errors.New("failed to patch status")) + }, + expectedError: "update repo with job status at start: failed to patch status", + }, + { + name: "failed getting repository resources", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + // Setup repository config + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + }, + Spec: provisioning.RepositorySpec{ + Title: "test-repo", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "existing-ref", + }, + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + + // Storage is migrated + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + // Initial status update succeeds + pr.On("SetMessage", mock.Anything, "update sync status at start").Return() + rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil) + + // Repository resources creation fails + rrf.On("Client", mock.Anything, mock.Anything).Return(nil, errors.New("failed to create repository resources client")) + }, + expectedError: "create repository resources client: failed to create repository resources client", + }, + { + name: "failed getting clients for namespace", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + // Setup repository config + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + Spec: provisioning.RepositorySpec{ + Title: "test-repo", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "existing-ref", + }, + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + + // Storage is migrated + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + // Initial status update succeeds + pr.On("SetMessage", mock.Anything, "update sync status at start").Return() + rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil) + + // Repository resources creation succeeds + rrf.On("Client", mock.Anything, mock.Anything).Return(&resources.MockRepositoryResources{}, nil) + + // Getting clients for namespace fails + cf.On("Clients", mock.Anything, "test-namespace").Return(nil, errors.New("failed to get clients")) + }, + expectedError: "get clients for test-repo: failed to get clients", + }, + { + name: "successful sync", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "existing-ref", + }, + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + + // Storage is migrated + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + // Initial status update + pr.On("SetMessage", mock.Anything, "update sync status at start").Return() + rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil) + + // Setup resources and clients + mockRepoResources := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Stats", mock.Anything).Return(nil, nil) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, "test-namespace").Return(mockClients, nil) + + // Sync execution succeeds + pr.On("SetMessage", mock.Anything, "execute sync job").Return() + s.On("Sync", mock.Anything, rw, mock.MatchedBy(func(opts provisioning.SyncJobOptions) bool { + return true // Add specific sync options validation if needed + }), mockRepoResources, mock.Anything, pr).Return("new-ref", nil) + + // Final status updates + pr.On("Complete", mock.Anything, nil).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess}) + pr.On("SetMessage", mock.Anything, "update status and stats").Return() + + // Final patch should include new ref + rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool { + if len(patch) != 1 { + return false + } + syncStatus := patch[0]["value"].(provisioning.SyncStatus) + return patch[0]["op"] == "replace" && + patch[0]["path"] == "/status/sync" && + syncStatus.LastRef == "new-ref" && + syncStatus.State == provisioning.JobStateSuccess + })).Return(nil) + }, + expectedError: "", + }, + { + name: "failed sync", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + Status: provisioning.RepositoryStatus{ + Sync: provisioning.SyncStatus{ + LastRef: "existing-ref", + }, + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + + // Storage is migrated + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + // Initial status update + pr.On("SetMessage", mock.Anything, "update sync status at start").Return() + rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil) + + // Setup resources and clients + mockRepoResources := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Stats", mock.Anything).Return(nil, nil) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, "test-namespace").Return(mockClients, nil) + + // Sync execution fails + pr.On("SetMessage", mock.Anything, "execute sync job").Return() + syncError := errors.New("sync operation failed") + s.On("Sync", mock.Anything, rw, mock.MatchedBy(func(opts provisioning.SyncJobOptions) bool { + return true // Add specific sync options validation if needed + }), mockRepoResources, mock.Anything, pr).Return("", syncError) + + // Final status updates + pr.On("Complete", mock.Anything, syncError).Return(provisioning.JobStatus{State: provisioning.JobStateError}) + pr.On("SetMessage", mock.Anything, "update status and stats").Return() + + // Final patch should preserve existing ref on failure + rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool { + if len(patch) != 1 { + return false + } + syncStatus := patch[0]["value"].(provisioning.SyncStatus) + return patch[0]["op"] == "replace" && + patch[0]["path"] == "/status/sync" && + syncStatus.LastRef == "existing-ref" && // LastRef should not change on failure + syncStatus.State == provisioning.JobStateError + })).Return(nil) + }, + expectedError: "sync operation failed", + }, + { + name: "stats call fails", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + mockRepoResources := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Stats", mock.Anything).Return(nil, errors.New("stats error")) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + // Simple mocks for other calls + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil) + pr.On("SetMessage", mock.Anything, mock.Anything).Return() + pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess}) + rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil) + s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil) + }, + expectedError: "", + }, + { + name: "stats returns nil stats and nil error", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + mockRepoResources := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Stats", mock.Anything).Return(nil, nil) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + // Verify only sync status is patched + rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool { + return len(patch) == 1 && patch[0]["path"] == "/status/sync" + })).Return(nil) + + // Simple mocks for other calls + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil) + pr.On("SetMessage", mock.Anything, mock.Anything).Return() + pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess}) + s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil) + }, + expectedError: "", + }, + { + name: "stats returns one managed stats", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + + mockRepoResources := resources.NewMockRepositoryResources(t) + stats := &provisioning.ResourceStats{ + Managed: []provisioning.ManagerStats{ + { + Stats: []provisioning.ResourceCount{ + { + Group: "test", + Resource: "test", + Count: 42, + }, + }, + }, + }, + } + mockRepoResources.On("Stats", mock.Anything).Return(stats, nil) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + // Verify both sync status and stats are patched + rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool { + if len(patch) != 2 { + return false + } + if patch[0]["path"] != "/status/sync" { + return false + } + + if patch[1]["path"] != "/status/stats" { + return false + } + + value := patch[1]["value"].([]provisioning.ResourceCount) + if len(value) != 1 { + return false + } + + if value[0].Group != "test" || value[0].Resource != "test" || value[0].Count != 42 { + return false + } + + return true + })).Return(nil).Once() + + // Simple mocks for other calls + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil) + pr.On("SetMessage", mock.Anything, mock.Anything).Return() + pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess}) + s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil) + }, + expectedError: "", + }, + { + name: "stats returns multiple managed stats", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + mockRepoResources := resources.NewMockRepositoryResources(t) + stats := &provisioning.ResourceStats{ + Managed: []provisioning.ManagerStats{ + { + Stats: []provisioning.ResourceCount{ + { + Group: "test1", + Resource: "test1", + Count: 42, + }, + }, + }, + { + Stats: []provisioning.ResourceCount{ + { + Group: "test2", + Resource: "test2", + Count: 24, + }, + }, + }, + }, + } + mockRepoResources.On("Stats", mock.Anything).Return(stats, nil) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + // Verify only sync status is patched (multiple stats should be ignored) + rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool { + return len(patch) == 1 && patch[0]["path"] == "/status/sync" + })).Return(nil) + + // Simple mocks for other calls + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil) + pr.On("SetMessage", mock.Anything, mock.Anything).Return() + pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess}) + s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil) + }, + expectedError: "", + }, + { + name: "failed final status patch", + setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) { + repoConfig := &provisioning.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo", + Namespace: "test-namespace", + }, + } + rw.MockRepository.On("Config").Return(repoConfig) + ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() + + // Initial status patch succeeds + rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + + // Setup resources and clients + mockRepoResources := resources.NewMockRepositoryResources(t) + mockRepoResources.On("Stats", mock.Anything).Return(nil, nil) + rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil) + + mockClients := resources.NewMockResourceClients(t) + cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil) + + // Sync succeeds + pr.On("SetMessage", mock.Anything, mock.Anything).Return() + s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil) + pr.On("Complete", mock.Anything, nil).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess}) + + // Final status patch fails + rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("failed to patch final status")).Once() + }, + expectedError: "update repo with job final status: failed to patch final status", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create mocks + clientFactory := resources.NewMockClientFactory(t) + repoResourcesFactory := resources.NewMockRepositoryResourcesFactory(t) + dualwriteService := dualwrite.NewMockService(t) + repositoryPatchFn := NewMockRepositoryPatchFn(t) + syncer := NewMockSyncer(t) + readerWriter := &mockReaderWriter{ + MockRepository: repository.NewMockRepository(t), + MockVersioned: repository.NewMockVersioned(t), + } + progressRecorder := jobs.NewMockJobProgressRecorder(t) + + // Setup mocks + tt.setupMocks(clientFactory, repoResourcesFactory, dualwriteService, repositoryPatchFn, syncer, readerWriter, progressRecorder) + + // Create worker + worker := NewSyncWorker( + clientFactory, + repoResourcesFactory, + dualwriteService, + repositoryPatchFn.Execute, + syncer, + ) + + // Create test job + job := provisioning.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-job", + }, + Spec: provisioning.JobSpec{ + Action: provisioning.JobActionPull, + Pull: &provisioning.SyncJobOptions{}, + }, + } + + // Execute test + err := worker.Process(context.Background(), readerWriter, job, progressRecorder) + + // Verify results + if tt.expectedError != "" { + require.EqualError(t, err, tt.expectedError) + } else { + require.NoError(t, err) + } + + // Verify mock expectations + repositoryPatchFn.AssertExpectations(t) + progressRecorder.AssertExpectations(t) + }) + } +} diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index e4a52f044d1..26e6e617a67 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -125,6 +125,7 @@ func NewAPIBuilder( clients := resources.NewClientFactory(configProvider) parsers := resources.NewParserFactory(clients) + resourceLister := resources.NewResourceLister(unified, unified, legacyMigrator, storageStatus) return &APIBuilder{ urlProvider: urlProvider, @@ -135,10 +136,10 @@ func NewAPIBuilder( ghFactory: ghFactory, clients: clients, parsers: parsers, - repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients), + repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister), render: render, clonedir: clonedir, - resourceLister: resources.NewResourceLister(unified, unified, legacyMigrator, storageStatus), + resourceLister: resourceLister, legacyMigrator: legacyMigrator, storageStatus: storageStatus, unified: unified, @@ -536,13 +537,16 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH repository.WrapWithCloneAndPushIfPossible, ) + statusPatcher := controller.NewRepositoryStatusPatcher(b.GetClient()) + syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync) syncWorker := sync.NewSyncWorker( - b.GetClient(), - b.parsers, b.clients, - b.resourceLister, + b.repositoryResources, b.storageStatus, + statusPatcher.Patch, + syncer, ) + migrationWorker := migrate.NewMigrationWorker( b.legacyMigrator, b.parsers, diff --git a/pkg/registry/apis/provisioning/repository/repository.go b/pkg/registry/apis/provisioning/repository/repository.go index 2bebe036d2d..db5f6a45b2a 100644 --- a/pkg/registry/apis/provisioning/repository/repository.go +++ b/pkg/registry/apis/provisioning/repository/repository.go @@ -184,6 +184,8 @@ type VersionedFileChange struct { // Versioned is a repository that supports versioning. // This interface may be extended to make the the original Repository interface more agnostic to the underlying storage system. +// +//go:generate mockery --name Versioned --structname MockVersioned --inpackage --filename versioned_mock.go --with-expecter type Versioned interface { // History of changes for a path History(ctx context.Context, path, ref string) ([]provisioning.HistoryItem, error) diff --git a/pkg/registry/apis/provisioning/repository/versioned_mock.go b/pkg/registry/apis/provisioning/repository/versioned_mock.go new file mode 100644 index 00000000000..cdd59b288e1 --- /dev/null +++ b/pkg/registry/apis/provisioning/repository/versioned_mock.go @@ -0,0 +1,213 @@ +// Code generated by mockery v2.52.4. DO NOT EDIT. + +package repository + +import ( + context "context" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + mock "github.com/stretchr/testify/mock" +) + +// MockVersioned is an autogenerated mock type for the Versioned type +type MockVersioned struct { + mock.Mock +} + +type MockVersioned_Expecter struct { + mock *mock.Mock +} + +func (_m *MockVersioned) EXPECT() *MockVersioned_Expecter { + return &MockVersioned_Expecter{mock: &_m.Mock} +} + +// CompareFiles provides a mock function with given fields: ctx, base, ref +func (_m *MockVersioned) CompareFiles(ctx context.Context, base string, ref string) ([]VersionedFileChange, error) { + ret := _m.Called(ctx, base, ref) + + if len(ret) == 0 { + panic("no return value specified for CompareFiles") + } + + var r0 []VersionedFileChange + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]VersionedFileChange, error)); ok { + return rf(ctx, base, ref) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []VersionedFileChange); ok { + r0 = rf(ctx, base, ref) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]VersionedFileChange) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, base, ref) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockVersioned_CompareFiles_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CompareFiles' +type MockVersioned_CompareFiles_Call struct { + *mock.Call +} + +// CompareFiles is a helper method to define mock.On call +// - ctx context.Context +// - base string +// - ref string +func (_e *MockVersioned_Expecter) CompareFiles(ctx interface{}, base interface{}, ref interface{}) *MockVersioned_CompareFiles_Call { + return &MockVersioned_CompareFiles_Call{Call: _e.mock.On("CompareFiles", ctx, base, ref)} +} + +func (_c *MockVersioned_CompareFiles_Call) Run(run func(ctx context.Context, base string, ref string)) *MockVersioned_CompareFiles_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockVersioned_CompareFiles_Call) Return(_a0 []VersionedFileChange, _a1 error) *MockVersioned_CompareFiles_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockVersioned_CompareFiles_Call) RunAndReturn(run func(context.Context, string, string) ([]VersionedFileChange, error)) *MockVersioned_CompareFiles_Call { + _c.Call.Return(run) + return _c +} + +// History provides a mock function with given fields: ctx, path, ref +func (_m *MockVersioned) History(ctx context.Context, path string, ref string) ([]v0alpha1.HistoryItem, error) { + ret := _m.Called(ctx, path, ref) + + if len(ret) == 0 { + panic("no return value specified for History") + } + + var r0 []v0alpha1.HistoryItem + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]v0alpha1.HistoryItem, error)); ok { + return rf(ctx, path, ref) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []v0alpha1.HistoryItem); ok { + r0 = rf(ctx, path, ref) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]v0alpha1.HistoryItem) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, path, ref) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockVersioned_History_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'History' +type MockVersioned_History_Call struct { + *mock.Call +} + +// History is a helper method to define mock.On call +// - ctx context.Context +// - path string +// - ref string +func (_e *MockVersioned_Expecter) History(ctx interface{}, path interface{}, ref interface{}) *MockVersioned_History_Call { + return &MockVersioned_History_Call{Call: _e.mock.On("History", ctx, path, ref)} +} + +func (_c *MockVersioned_History_Call) Run(run func(ctx context.Context, path string, ref string)) *MockVersioned_History_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockVersioned_History_Call) Return(_a0 []v0alpha1.HistoryItem, _a1 error) *MockVersioned_History_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockVersioned_History_Call) RunAndReturn(run func(context.Context, string, string) ([]v0alpha1.HistoryItem, error)) *MockVersioned_History_Call { + _c.Call.Return(run) + return _c +} + +// LatestRef provides a mock function with given fields: ctx +func (_m *MockVersioned) LatestRef(ctx context.Context) (string, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for LatestRef") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) string); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockVersioned_LatestRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LatestRef' +type MockVersioned_LatestRef_Call struct { + *mock.Call +} + +// LatestRef is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockVersioned_Expecter) LatestRef(ctx interface{}) *MockVersioned_LatestRef_Call { + return &MockVersioned_LatestRef_Call{Call: _e.mock.On("LatestRef", ctx)} +} + +func (_c *MockVersioned_LatestRef_Call) Run(run func(ctx context.Context)) *MockVersioned_LatestRef_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockVersioned_LatestRef_Call) Return(_a0 string, _a1 error) *MockVersioned_LatestRef_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockVersioned_LatestRef_Call) RunAndReturn(run func(context.Context) (string, error)) *MockVersioned_LatestRef_Call { + _c.Call.Return(run) + return _c +} + +// NewMockVersioned creates a new instance of MockVersioned. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockVersioned(t interface { + mock.TestingT + Cleanup(func()) +}) *MockVersioned { + mock := &MockVersioned{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/registry/apis/provisioning/resources/repository.go b/pkg/registry/apis/provisioning/resources/repository.go index 9d4019d8907..85a5eb2c9de 100644 --- a/pkg/registry/apis/provisioning/resources/repository.go +++ b/pkg/registry/apis/provisioning/resources/repository.go @@ -4,8 +4,10 @@ import ( "context" "fmt" + provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" ) //go:generate mockery --name RepositoryResourcesFactory --structname MockRepositoryResourcesFactory --inpackage --filename repository_resources_factory_mock.go --with-expecter @@ -15,21 +17,45 @@ type RepositoryResourcesFactory interface { //go:generate mockery --name RepositoryResources --structname MockRepositoryResources --inpackage --filename repository_resources_mock.go --with-expecter type RepositoryResources interface { + // Folders + SetTree(tree FolderTree) + EnsureFolderPathExist(ctx context.Context, filePath string) (parent string, err error) + EnsureFolderExists(ctx context.Context, folder Folder, parentID string) error EnsureFolderTreeExists(ctx context.Context, ref, path string, tree FolderTree, fn func(folder Folder, created bool, err error) error) error + // File from Resource CreateResourceFileFromObject(ctx context.Context, obj *unstructured.Unstructured, options WriteOptions) (string, error) + // Resource from file + WriteResourceFromFile(ctx context.Context, path, ref string) (string, schema.GroupVersionKind, error) + RemoveResourceFromFile(ctx context.Context, path, ref string) (string, schema.GroupVersionKind, error) + RenameResourceFile(ctx context.Context, path, previousRef, newPath, newRef string) (string, schema.GroupVersionKind, error) + // Stats + Stats(ctx context.Context) (*provisioning.ResourceStats, error) + List(ctx context.Context) (*provisioning.ResourceList, error) } type repositoryResourcesFactor struct { parsers ParserFactory clients ClientFactory + lister ResourceLister } type repositoryResources struct { *FolderManager *ResourcesManager + lister ResourceLister + namespace string + repoName string } -func NewRepositoryResourcesFactory(parsers ParserFactory, clients ClientFactory) RepositoryResourcesFactory { - return &repositoryResourcesFactor{parsers, clients} +func (r *repositoryResources) Stats(ctx context.Context) (*provisioning.ResourceStats, error) { + return r.lister.Stats(ctx, r.namespace, r.repoName) +} + +func (r *repositoryResources) List(ctx context.Context) (*provisioning.ResourceList, error) { + return r.lister.List(ctx, r.namespace, r.repoName) +} + +func NewRepositoryResourcesFactory(parsers ParserFactory, clients ClientFactory, lister ResourceLister) RepositoryResourcesFactory { + return &repositoryResourcesFactor{parsers, clients, lister} } func (r *repositoryResourcesFactor) Client(ctx context.Context, repo repository.ReaderWriter) (RepositoryResources, error) { @@ -50,5 +76,11 @@ func (r *repositoryResourcesFactor) Client(ctx context.Context, repo repository. folders := NewFolderManager(repo, folderClient, NewEmptyFolderTree()) resources := NewResourcesManager(repo, folders, parser, clients, map[string]repository.CommitSignature{}) - return &repositoryResources{folders, resources}, nil + return &repositoryResources{ + FolderManager: folders, + ResourcesManager: resources, + lister: r.lister, + namespace: repo.Config().Namespace, + repoName: repo.Config().Name, + }, nil } diff --git a/pkg/registry/apis/provisioning/resources/repository_resources_mock.go b/pkg/registry/apis/provisioning/resources/repository_resources_mock.go index e50929935e0..cadf9476da8 100644 --- a/pkg/registry/apis/provisioning/resources/repository_resources_mock.go +++ b/pkg/registry/apis/provisioning/resources/repository_resources_mock.go @@ -6,7 +6,11 @@ import ( context "context" mock "github.com/stretchr/testify/mock" + schema "k8s.io/apimachinery/pkg/runtime/schema" + unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" ) // MockRepositoryResources is an autogenerated mock type for the RepositoryResources type @@ -80,6 +84,111 @@ func (_c *MockRepositoryResources_CreateResourceFileFromObject_Call) RunAndRetur return _c } +// EnsureFolderExists provides a mock function with given fields: ctx, folder, parentID +func (_m *MockRepositoryResources) EnsureFolderExists(ctx context.Context, folder Folder, parentID string) error { + ret := _m.Called(ctx, folder, parentID) + + if len(ret) == 0 { + panic("no return value specified for EnsureFolderExists") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, Folder, string) error); ok { + r0 = rf(ctx, folder, parentID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockRepositoryResources_EnsureFolderExists_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnsureFolderExists' +type MockRepositoryResources_EnsureFolderExists_Call struct { + *mock.Call +} + +// EnsureFolderExists is a helper method to define mock.On call +// - ctx context.Context +// - folder Folder +// - parentID string +func (_e *MockRepositoryResources_Expecter) EnsureFolderExists(ctx interface{}, folder interface{}, parentID interface{}) *MockRepositoryResources_EnsureFolderExists_Call { + return &MockRepositoryResources_EnsureFolderExists_Call{Call: _e.mock.On("EnsureFolderExists", ctx, folder, parentID)} +} + +func (_c *MockRepositoryResources_EnsureFolderExists_Call) Run(run func(ctx context.Context, folder Folder, parentID string)) *MockRepositoryResources_EnsureFolderExists_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(Folder), args[2].(string)) + }) + return _c +} + +func (_c *MockRepositoryResources_EnsureFolderExists_Call) Return(_a0 error) *MockRepositoryResources_EnsureFolderExists_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockRepositoryResources_EnsureFolderExists_Call) RunAndReturn(run func(context.Context, Folder, string) error) *MockRepositoryResources_EnsureFolderExists_Call { + _c.Call.Return(run) + return _c +} + +// EnsureFolderPathExist provides a mock function with given fields: ctx, filePath +func (_m *MockRepositoryResources) EnsureFolderPathExist(ctx context.Context, filePath string) (string, error) { + ret := _m.Called(ctx, filePath) + + if len(ret) == 0 { + panic("no return value specified for EnsureFolderPathExist") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok { + return rf(ctx, filePath) + } + if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { + r0 = rf(ctx, filePath) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, filePath) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockRepositoryResources_EnsureFolderPathExist_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnsureFolderPathExist' +type MockRepositoryResources_EnsureFolderPathExist_Call struct { + *mock.Call +} + +// EnsureFolderPathExist is a helper method to define mock.On call +// - ctx context.Context +// - filePath string +func (_e *MockRepositoryResources_Expecter) EnsureFolderPathExist(ctx interface{}, filePath interface{}) *MockRepositoryResources_EnsureFolderPathExist_Call { + return &MockRepositoryResources_EnsureFolderPathExist_Call{Call: _e.mock.On("EnsureFolderPathExist", ctx, filePath)} +} + +func (_c *MockRepositoryResources_EnsureFolderPathExist_Call) Run(run func(ctx context.Context, filePath string)) *MockRepositoryResources_EnsureFolderPathExist_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MockRepositoryResources_EnsureFolderPathExist_Call) Return(parent string, err error) *MockRepositoryResources_EnsureFolderPathExist_Call { + _c.Call.Return(parent, err) + return _c +} + +func (_c *MockRepositoryResources_EnsureFolderPathExist_Call) RunAndReturn(run func(context.Context, string) (string, error)) *MockRepositoryResources_EnsureFolderPathExist_Call { + _c.Call.Return(run) + return _c +} + // EnsureFolderTreeExists provides a mock function with given fields: ctx, ref, path, tree, fn func (_m *MockRepositoryResources) EnsureFolderTreeExists(ctx context.Context, ref string, path string, tree FolderTree, fn func(Folder, bool, error) error) error { ret := _m.Called(ctx, ref, path, tree, fn) @@ -130,6 +239,352 @@ func (_c *MockRepositoryResources_EnsureFolderTreeExists_Call) RunAndReturn(run return _c } +// List provides a mock function with given fields: ctx +func (_m *MockRepositoryResources) List(ctx context.Context) (*v0alpha1.ResourceList, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for List") + } + + var r0 *v0alpha1.ResourceList + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*v0alpha1.ResourceList, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *v0alpha1.ResourceList); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v0alpha1.ResourceList) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockRepositoryResources_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type MockRepositoryResources_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockRepositoryResources_Expecter) List(ctx interface{}) *MockRepositoryResources_List_Call { + return &MockRepositoryResources_List_Call{Call: _e.mock.On("List", ctx)} +} + +func (_c *MockRepositoryResources_List_Call) Run(run func(ctx context.Context)) *MockRepositoryResources_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockRepositoryResources_List_Call) Return(_a0 *v0alpha1.ResourceList, _a1 error) *MockRepositoryResources_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockRepositoryResources_List_Call) RunAndReturn(run func(context.Context) (*v0alpha1.ResourceList, error)) *MockRepositoryResources_List_Call { + _c.Call.Return(run) + return _c +} + +// RemoveResourceFromFile provides a mock function with given fields: ctx, path, ref +func (_m *MockRepositoryResources) RemoveResourceFromFile(ctx context.Context, path string, ref string) (string, schema.GroupVersionKind, error) { + ret := _m.Called(ctx, path, ref) + + if len(ret) == 0 { + panic("no return value specified for RemoveResourceFromFile") + } + + var r0 string + var r1 schema.GroupVersionKind + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (string, schema.GroupVersionKind, error)); ok { + return rf(ctx, path, ref) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) string); ok { + r0 = rf(ctx, path, ref) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) schema.GroupVersionKind); ok { + r1 = rf(ctx, path, ref) + } else { + r1 = ret.Get(1).(schema.GroupVersionKind) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok { + r2 = rf(ctx, path, ref) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockRepositoryResources_RemoveResourceFromFile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveResourceFromFile' +type MockRepositoryResources_RemoveResourceFromFile_Call struct { + *mock.Call +} + +// RemoveResourceFromFile is a helper method to define mock.On call +// - ctx context.Context +// - path string +// - ref string +func (_e *MockRepositoryResources_Expecter) RemoveResourceFromFile(ctx interface{}, path interface{}, ref interface{}) *MockRepositoryResources_RemoveResourceFromFile_Call { + return &MockRepositoryResources_RemoveResourceFromFile_Call{Call: _e.mock.On("RemoveResourceFromFile", ctx, path, ref)} +} + +func (_c *MockRepositoryResources_RemoveResourceFromFile_Call) Run(run func(ctx context.Context, path string, ref string)) *MockRepositoryResources_RemoveResourceFromFile_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockRepositoryResources_RemoveResourceFromFile_Call) Return(_a0 string, _a1 schema.GroupVersionKind, _a2 error) *MockRepositoryResources_RemoveResourceFromFile_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *MockRepositoryResources_RemoveResourceFromFile_Call) RunAndReturn(run func(context.Context, string, string) (string, schema.GroupVersionKind, error)) *MockRepositoryResources_RemoveResourceFromFile_Call { + _c.Call.Return(run) + return _c +} + +// RenameResourceFile provides a mock function with given fields: ctx, path, previousRef, newPath, newRef +func (_m *MockRepositoryResources) RenameResourceFile(ctx context.Context, path string, previousRef string, newPath string, newRef string) (string, schema.GroupVersionKind, error) { + ret := _m.Called(ctx, path, previousRef, newPath, newRef) + + if len(ret) == 0 { + panic("no return value specified for RenameResourceFile") + } + + var r0 string + var r1 schema.GroupVersionKind + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) (string, schema.GroupVersionKind, error)); ok { + return rf(ctx, path, previousRef, newPath, newRef) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) string); ok { + r0 = rf(ctx, path, previousRef, newPath, newRef) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string) schema.GroupVersionKind); ok { + r1 = rf(ctx, path, previousRef, newPath, newRef) + } else { + r1 = ret.Get(1).(schema.GroupVersionKind) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string, string, string) error); ok { + r2 = rf(ctx, path, previousRef, newPath, newRef) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockRepositoryResources_RenameResourceFile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RenameResourceFile' +type MockRepositoryResources_RenameResourceFile_Call struct { + *mock.Call +} + +// RenameResourceFile is a helper method to define mock.On call +// - ctx context.Context +// - path string +// - previousRef string +// - newPath string +// - newRef string +func (_e *MockRepositoryResources_Expecter) RenameResourceFile(ctx interface{}, path interface{}, previousRef interface{}, newPath interface{}, newRef interface{}) *MockRepositoryResources_RenameResourceFile_Call { + return &MockRepositoryResources_RenameResourceFile_Call{Call: _e.mock.On("RenameResourceFile", ctx, path, previousRef, newPath, newRef)} +} + +func (_c *MockRepositoryResources_RenameResourceFile_Call) Run(run func(ctx context.Context, path string, previousRef string, newPath string, newRef string)) *MockRepositoryResources_RenameResourceFile_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(string)) + }) + return _c +} + +func (_c *MockRepositoryResources_RenameResourceFile_Call) Return(_a0 string, _a1 schema.GroupVersionKind, _a2 error) *MockRepositoryResources_RenameResourceFile_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *MockRepositoryResources_RenameResourceFile_Call) RunAndReturn(run func(context.Context, string, string, string, string) (string, schema.GroupVersionKind, error)) *MockRepositoryResources_RenameResourceFile_Call { + _c.Call.Return(run) + return _c +} + +// SetTree provides a mock function with given fields: tree +func (_m *MockRepositoryResources) SetTree(tree FolderTree) { + _m.Called(tree) +} + +// MockRepositoryResources_SetTree_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetTree' +type MockRepositoryResources_SetTree_Call struct { + *mock.Call +} + +// SetTree is a helper method to define mock.On call +// - tree FolderTree +func (_e *MockRepositoryResources_Expecter) SetTree(tree interface{}) *MockRepositoryResources_SetTree_Call { + return &MockRepositoryResources_SetTree_Call{Call: _e.mock.On("SetTree", tree)} +} + +func (_c *MockRepositoryResources_SetTree_Call) Run(run func(tree FolderTree)) *MockRepositoryResources_SetTree_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(FolderTree)) + }) + return _c +} + +func (_c *MockRepositoryResources_SetTree_Call) Return() *MockRepositoryResources_SetTree_Call { + _c.Call.Return() + return _c +} + +func (_c *MockRepositoryResources_SetTree_Call) RunAndReturn(run func(FolderTree)) *MockRepositoryResources_SetTree_Call { + _c.Run(run) + return _c +} + +// Stats provides a mock function with given fields: ctx +func (_m *MockRepositoryResources) Stats(ctx context.Context) (*v0alpha1.ResourceStats, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for Stats") + } + + var r0 *v0alpha1.ResourceStats + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*v0alpha1.ResourceStats, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *v0alpha1.ResourceStats); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v0alpha1.ResourceStats) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockRepositoryResources_Stats_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Stats' +type MockRepositoryResources_Stats_Call struct { + *mock.Call +} + +// Stats is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockRepositoryResources_Expecter) Stats(ctx interface{}) *MockRepositoryResources_Stats_Call { + return &MockRepositoryResources_Stats_Call{Call: _e.mock.On("Stats", ctx)} +} + +func (_c *MockRepositoryResources_Stats_Call) Run(run func(ctx context.Context)) *MockRepositoryResources_Stats_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockRepositoryResources_Stats_Call) Return(_a0 *v0alpha1.ResourceStats, _a1 error) *MockRepositoryResources_Stats_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockRepositoryResources_Stats_Call) RunAndReturn(run func(context.Context) (*v0alpha1.ResourceStats, error)) *MockRepositoryResources_Stats_Call { + _c.Call.Return(run) + return _c +} + +// WriteResourceFromFile provides a mock function with given fields: ctx, path, ref +func (_m *MockRepositoryResources) WriteResourceFromFile(ctx context.Context, path string, ref string) (string, schema.GroupVersionKind, error) { + ret := _m.Called(ctx, path, ref) + + if len(ret) == 0 { + panic("no return value specified for WriteResourceFromFile") + } + + var r0 string + var r1 schema.GroupVersionKind + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (string, schema.GroupVersionKind, error)); ok { + return rf(ctx, path, ref) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) string); ok { + r0 = rf(ctx, path, ref) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) schema.GroupVersionKind); ok { + r1 = rf(ctx, path, ref) + } else { + r1 = ret.Get(1).(schema.GroupVersionKind) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok { + r2 = rf(ctx, path, ref) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockRepositoryResources_WriteResourceFromFile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WriteResourceFromFile' +type MockRepositoryResources_WriteResourceFromFile_Call struct { + *mock.Call +} + +// WriteResourceFromFile is a helper method to define mock.On call +// - ctx context.Context +// - path string +// - ref string +func (_e *MockRepositoryResources_Expecter) WriteResourceFromFile(ctx interface{}, path interface{}, ref interface{}) *MockRepositoryResources_WriteResourceFromFile_Call { + return &MockRepositoryResources_WriteResourceFromFile_Call{Call: _e.mock.On("WriteResourceFromFile", ctx, path, ref)} +} + +func (_c *MockRepositoryResources_WriteResourceFromFile_Call) Run(run func(ctx context.Context, path string, ref string)) *MockRepositoryResources_WriteResourceFromFile_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockRepositoryResources_WriteResourceFromFile_Call) Return(_a0 string, _a1 schema.GroupVersionKind, _a2 error) *MockRepositoryResources_WriteResourceFromFile_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *MockRepositoryResources_WriteResourceFromFile_Call) RunAndReturn(run func(context.Context, string, string) (string, schema.GroupVersionKind, error)) *MockRepositoryResources_WriteResourceFromFile_Call { + _c.Call.Return(run) + return _c +} + // NewMockRepositoryResources creates a new instance of MockRepositoryResources. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockRepositoryResources(t interface { diff --git a/pkg/storage/legacysql/dualwrite/utils_test.go b/pkg/storage/legacysql/dualwrite/utils_test.go new file mode 100644 index 00000000000..f13851da484 --- /dev/null +++ b/pkg/storage/legacysql/dualwrite/utils_test.go @@ -0,0 +1,76 @@ +package dualwrite + +import ( + "context" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" + + dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1alpha1" + folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" +) + +func TestIsReadingLegacyDashboardsAndFolders(t *testing.T) { + tests := []struct { + name string + setupMockSvc func(*MockService) + expectedResult bool + }{ + { + name: "both folders and dashboards are read from unified storage", + setupMockSvc: func(svc *MockService) { + svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(true, nil) + svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{ + Group: dashboard.GROUP, + Resource: dashboard.DASHBOARD_RESOURCE, + }).Return(true, nil) + }, + expectedResult: false, + }, + { + name: "only folders are read from unified storage", + setupMockSvc: func(svc *MockService) { + svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(true, nil) + svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{ + Group: dashboard.GROUP, + Resource: dashboard.DASHBOARD_RESOURCE, + }).Return(false, nil) + }, + expectedResult: true, + }, + { + name: "only dashboards are read from unified storage", + setupMockSvc: func(svc *MockService) { + svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(false, nil) + svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{ + Group: dashboard.GROUP, + Resource: dashboard.DASHBOARD_RESOURCE, + }).Return(true, nil) + }, + expectedResult: true, + }, + { + name: "neither folders nor dashboards are read from unified storage", + setupMockSvc: func(svc *MockService) { + svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(false, nil) + svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{ + Group: dashboard.GROUP, + Resource: dashboard.DASHBOARD_RESOURCE, + }).Return(false, nil) + }, + expectedResult: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockSvc := NewMockService(t) + tt.setupMockSvc(mockSvc) + + result := IsReadingLegacyDashboardsAndFolders(context.Background(), mockSvc) + require.Equal(t, tt.expectedResult, result) + }) + } +}