Provisioning: unit test sync job (#103636)

* Split in multiple files

* Refactor sync even further

* Move more things between RepositoryResources

* Add status patcher

* Interface for sync functions

* Interface for compare function

* Add syncer back

* Move interfaces

* Move execute complete

* Return currentRef in syncer

* Add repository status test

* Add initial sync tests

* Fix a couple of spots

* Make initial sync tests work

* Fix register.go

* Add initial tests for sync worker

* Finish tests for sync

* Add incremental tests

* Add TODO

* Finish incremental tests

* Move folder creation to full sync

* Move interfaces

* Add initial full sync tests

* Update tests

* Reshape things

* Add changes test

* Fix register

* Add some tests

* Add more tests

* Add test

* WIP

* WIP: delete test

* Add more full test

* More tests

* Add tests for folder creation

* Add folder tests full sync

* Full coverage full sync

* Clean up tests

* Add more tests for changes function

* Enhance tests for Changes function to cover error scenarios and empty paths

- Added test cases for handling empty file paths and folder resources.
- Updated error message formatting in the Compare function for clarity.

* Add unit tests

* Failed initial patch

* Add tests failed repository resources

* Add test failed getting client

* Test for successful and unsuccessful syncs

* Add final tests for worker

* Fix existing tests

* Add missing test

* Fix spelling mistake

* Fix flake in changes test
This commit is contained in:
Roberto Jiménez Sánchez
2025-04-10 13:55:49 +01:00
committed by GitHub
parent 8df9c57045
commit 47664a7d51
26 changed files with 3894 additions and 428 deletions
@@ -0,0 +1,37 @@
package controller
import (
"context"
"encoding/json"
"fmt"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
client "github.com/grafana/grafana/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
type RepositoryStatusPatcher struct {
client client.ProvisioningV0alpha1Interface
}
func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *RepositoryStatusPatcher {
return &RepositoryStatusPatcher{
client: client,
}
}
func (r *RepositoryStatusPatcher) Patch(ctx context.Context, repo *provisioning.Repository, patchOperations []map[string]interface{}) error {
patch, err := json.Marshal(patchOperations)
if err != nil {
return fmt.Errorf("unable to marshal patch data: %w", err)
}
_, err = r.client.Repositories(repo.Namespace).
Patch(ctx, repo.Name, types.JSONPatchType, patch, metav1.PatchOptions{}, "status")
if err != nil {
return fmt.Errorf("unable to update repo with job status: %w", err)
}
return nil
}
@@ -0,0 +1,130 @@
package controller
import (
"context"
"encoding/json"
"fmt"
"testing"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1/fake"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
k8testing "k8s.io/client-go/testing"
)
func TestNewRepositoryStatusPatcher(t *testing.T) {
client := &fake.FakeProvisioningV0alpha1{}
patcher := NewRepositoryStatusPatcher(client)
require.NotNil(t, patcher)
require.Equal(t, client, patcher.client)
}
func TestRepositoryStatusPatcher_Patch(t *testing.T) {
tests := []struct {
name string
repo *provisioning.Repository
patchOperations []map[string]interface{}
reactorFunc func(action k8testing.Action) (bool, runtime.Object, error)
expectedError string
}{
{
name: "successful patch",
repo: &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
},
patchOperations: []map[string]interface{}{
{
"op": "replace",
"path": "/status/health",
"value": map[string]interface{}{
"healthy": true,
"message": []string{},
},
},
},
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, &provisioning.Repository{}, nil
},
},
{
name: "patch marshal error",
repo: &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
},
patchOperations: []map[string]interface{}{
{
"op": "replace",
"path": "/status/health",
"value": make(chan int), // This will cause json.Marshal to fail
},
},
expectedError: "unable to marshal patch data: json: unsupported type: chan int",
},
{
name: "patch request error",
repo: &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
},
patchOperations: []map[string]interface{}{
{
"op": "replace",
"path": "/status/health",
"value": map[string]interface{}{
"healthy": true,
"message": []string{},
},
},
},
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("patch request failed")
},
expectedError: "unable to update repo with job status: patch request failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := fake.FakeProvisioningV0alpha1{
Fake: &k8testing.Fake{},
}
if tt.reactorFunc != nil {
client.AddReactor("patch", "repositories", tt.reactorFunc)
}
patcher := NewRepositoryStatusPatcher(&client)
err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations)
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError)
} else {
require.NoError(t, err)
}
if tt.reactorFunc != nil {
actions := client.Actions()
require.Len(t, actions, 1)
patchAction := actions[0].(k8testing.PatchAction)
require.Equal(t, "status", patchAction.GetSubresource())
require.Equal(t, tt.repo.Namespace, patchAction.GetNamespace())
require.Equal(t, tt.repo.Name, patchAction.GetName())
// Verify patch data
expectedPatch, _ := json.Marshal(tt.patchOperations)
require.Equal(t, expectedPatch, patchAction.GetPatch())
}
})
}
}
@@ -69,51 +69,6 @@ func (_c *MockJobProgressRecorder_Complete_Call) RunAndReturn(run func(context.C
return _c
}
// GetRef provides a mock function with no fields
func (_m *MockJobProgressRecorder) GetRef() string {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for GetRef")
}
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// MockJobProgressRecorder_GetRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRef'
type MockJobProgressRecorder_GetRef_Call struct {
*mock.Call
}
// GetRef is a helper method to define mock.On call
func (_e *MockJobProgressRecorder_Expecter) GetRef() *MockJobProgressRecorder_GetRef_Call {
return &MockJobProgressRecorder_GetRef_Call{Call: _e.mock.On("GetRef")}
}
func (_c *MockJobProgressRecorder_GetRef_Call) Run(run func()) *MockJobProgressRecorder_GetRef_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockJobProgressRecorder_GetRef_Call) Return(_a0 string) *MockJobProgressRecorder_GetRef_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockJobProgressRecorder_GetRef_Call) RunAndReturn(run func() string) *MockJobProgressRecorder_GetRef_Call {
_c.Call.Return(run)
return _c
}
// Record provides a mock function with given fields: ctx, result
func (_m *MockJobProgressRecorder) Record(ctx context.Context, result JobResourceResult) {
_m.Called(ctx, result)
@@ -248,39 +203,6 @@ func (_c *MockJobProgressRecorder_SetMessage_Call) RunAndReturn(run func(context
return _c
}
// SetRef provides a mock function with given fields: ref
func (_m *MockJobProgressRecorder) SetRef(ref string) {
_m.Called(ref)
}
// MockJobProgressRecorder_SetRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetRef'
type MockJobProgressRecorder_SetRef_Call struct {
*mock.Call
}
// SetRef is a helper method to define mock.On call
// - ref string
func (_e *MockJobProgressRecorder_Expecter) SetRef(ref interface{}) *MockJobProgressRecorder_SetRef_Call {
return &MockJobProgressRecorder_SetRef_Call{Call: _e.mock.On("SetRef", ref)}
}
func (_c *MockJobProgressRecorder_SetRef_Call) Run(run func(ref string)) *MockJobProgressRecorder_SetRef_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *MockJobProgressRecorder_SetRef_Call) Return() *MockJobProgressRecorder_SetRef_Call {
_c.Call.Return()
return _c
}
func (_c *MockJobProgressRecorder_SetRef_Call) RunAndReturn(run func(string)) *MockJobProgressRecorder_SetRef_Call {
_c.Run(run)
return _c
}
// SetTotal provides a mock function with given fields: ctx, total
func (_m *MockJobProgressRecorder) SetTotal(ctx context.Context, total int) {
_m.Called(ctx, total)
@@ -38,7 +38,6 @@ type JobResourceResult struct {
type jobProgressRecorder struct {
started time.Time
total int
ref string
message string
finalMessage string
resultCount int
@@ -96,14 +95,6 @@ func (r *jobProgressRecorder) SetFinalMessage(ctx context.Context, msg string) {
logging.FromContext(ctx).Info("job final message", "message", msg)
}
func (r *jobProgressRecorder) SetRef(ref string) {
r.ref = ref
}
func (r *jobProgressRecorder) GetRef() string {
return r.ref
}
func (r *jobProgressRecorder) SetTotal(ctx context.Context, total int) {
r.total = total
@@ -22,8 +22,6 @@ type JobProgressRecorder interface {
ResetResults()
SetFinalMessage(ctx context.Context, msg string)
SetMessage(ctx context.Context, msg string)
SetRef(ref string)
GetRef() string
SetTotal(ctx context.Context, total int)
TooManyErrors() error
Complete(ctx context.Context, err error) provisioning.JobStatus
@@ -1,6 +1,7 @@
package sync
import (
"context"
"fmt"
"sort"
"strings"
@@ -20,6 +21,31 @@ type ResourceFileChange struct {
Existing *provisioning.ResourceListItem
}
func Compare(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error) {
target, err := repositoryResources.List(ctx)
if err != nil {
return nil, fmt.Errorf("error listing current: %w", err)
}
source, err := repo.ReadTree(ctx, ref)
if err != nil {
return nil, fmt.Errorf("error reading tree: %w", err)
}
changes, err := Changes(source, target)
if err != nil {
return nil, fmt.Errorf("calculate changes: %w", err)
}
if len(changes) > 0 {
// FIXME: this is a way to load in different ways the resources
// maybe we can structure the code in better way to avoid this
repositoryResources.SetTree(resources.NewFolderTreeFromResourceList(target))
}
return changes, nil
}
func Changes(source []repository.FileTreeEntry, target *provisioning.ResourceList) ([]ResourceFileChange, error) {
lookup := make(map[string]*provisioning.ResourceListItem, len(target.Items))
for _, item := range target.Items {
@@ -41,6 +67,11 @@ func Changes(source []repository.FileTreeEntry, target *provisioning.ResourceLis
keep := safepath.NewTrie()
changes := make([]ResourceFileChange, 0, len(source))
for _, file := range source {
// TODO: why do we have to do this here?
if !file.Blob && !strings.HasSuffix(file.Path, "/") {
file.Path = file.Path + "/"
}
check, ok := lookup[file.Path]
if ok {
if check.Hash != file.Hash && check.Resource != resources.FolderResource.Resource {
@@ -113,7 +144,15 @@ func Changes(source []repository.FileTreeEntry, target *provisioning.ResourceLis
// Deepest first (stable sort order)
sort.Slice(changes, func(i, j int) bool {
return safepath.Depth(changes[i].Path) > safepath.Depth(changes[j].Path)
if safepath.Depth(changes[i].Path) > safepath.Depth(changes[j].Path) {
return true
}
if safepath.Depth(changes[i].Path) < safepath.Depth(changes[j].Path) {
return false
}
return changes[i].Path < changes[j].Path
})
return changes, nil
@@ -1,12 +1,16 @@
package sync
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
func TestChanges(t *testing.T) {
@@ -39,6 +43,29 @@ func TestChanges(t *testing.T) {
Path: "muta.json",
}, changes[0])
})
t.Run("empty file path", func(t *testing.T) {
source := []repository.FileTreeEntry{}
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "", Resource: "dashboard", Group: "dashboard.grafana.app"},
},
}
_, err := Changes(source, target)
require.EqualError(t, err, "empty path on a non folder")
})
t.Run("empty path with folder resource", func(t *testing.T) {
source := []repository.FileTreeEntry{}
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "", Resource: resources.FolderResource.Resource, Group: resources.FolderResource.Group},
},
}
changes, err := Changes(source, target)
require.NoError(t, err)
require.Empty(t, changes)
})
t.Run("create empty folder structure for folders with unsupported file types", func(t *testing.T) {
source := []repository.FileTreeEntry{
@@ -131,7 +158,7 @@ func TestChanges(t *testing.T) {
t.Run("folder deletion order", func(t *testing.T) {
source := []repository.FileTreeEntry{
{Path: "x/y/z/ignored.md"}, // ignored
{Path: "x/y/z/ignored.md", Blob: true}, // ignored
{Path: "aaa/bbb.yaml", Hash: "xyz", Blob: true},
}
target := &provisioning.ResourceList{
@@ -153,8 +180,8 @@ func TestChanges(t *testing.T) {
}
require.Equal(t, []string{
"zzz/longest/path/here.json", // not sorted yet
"x/y/z/",
"x/y/file.json",
"x/y/z/",
"short/file.yml",
"a.json",
}, order)
@@ -287,4 +314,200 @@ func TestChanges(t *testing.T) {
require.NoError(t, err)
require.Empty(t, changes)
})
t.Run("error on empty path for non-folder resource", func(t *testing.T) {
source := []repository.FileTreeEntry{
{Path: "", Hash: "xyz", Blob: true},
}
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "", Resource: "dashboard", Group: "dashboard.grafana.app"},
},
}
changes, err := Changes(source, target)
require.Error(t, err)
require.Contains(t, err.Error(), "empty path on a non folder")
require.Nil(t, changes)
})
t.Run("complex nested folder hierarchy with mixed file types", func(t *testing.T) {
source := []repository.FileTreeEntry{
{Path: "root/folder1/dashboard.json", Hash: "abc", Blob: true},
{Path: "root/folder1/subfolder/.gitkeep", Hash: "def", Blob: true},
{Path: "root/folder2/alert.json", Hash: "ghi", Blob: true},
{Path: "root/folder2/README.md", Hash: "jkl", Blob: true},
}
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "root/", Resource: "folders"},
{Path: "root/folder1/", Resource: "folders"},
{Path: "root/folder2/", Resource: "folders"},
{Path: "root/folder1/dashboard.json", Hash: "abc", Resource: "dashboard"},
{Path: "root/folder2/alert.json", Hash: "old", Resource: "alert"},
},
}
changes, err := Changes(source, target)
require.NoError(t, err)
require.Len(t, changes, 2)
require.Equal(t, "root/folder1/subfolder/", changes[0].Path)
require.Equal(t, "root/folder2/alert.json", changes[1].Path)
})
t.Run("folder path suffix handling", func(t *testing.T) {
source := []repository.FileTreeEntry{
{Path: "folder1/", Hash: "abc", Blob: false},
{Path: "folder2/", Hash: "def", Blob: false},
{Path: "folder3", Hash: "ghi", Blob: false},
{Path: "folder4/", Hash: "jkl", Blob: false},
}
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "folder1", Resource: resources.FolderResource.Resource, Group: resources.FolderResource.Group},
{Path: "folder2/", Resource: "folders", Group: resources.FolderResource.Group},
{Path: "folder3", Resource: "folders", Group: resources.FolderResource.Group},
{Path: "folder4/", Resource: "folders", Group: resources.FolderResource.Group},
},
}
changes, err := Changes(source, target)
require.NoError(t, err)
require.Empty(t, changes, "Should handle folder paths with and without trailing slash")
})
t.Run("empty source with populated target", func(t *testing.T) {
source := []repository.FileTreeEntry{}
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "folder1/", Resource: "folders"},
{Path: "folder1/dashboard.json", Resource: "dashboard"},
{Path: "folder2/", Resource: "folders"},
},
}
changes, err := Changes(source, target)
require.NoError(t, err)
require.Len(t, changes, 3)
// Verify deletion order (deepest first)
require.Equal(t, "folder1/dashboard.json", changes[0].Path)
require.Equal(t, "folder1/", changes[1].Path)
require.Equal(t, "folder2/", changes[2].Path)
for _, change := range changes {
require.Equal(t, repository.FileActionDeleted, change.Action)
}
})
t.Run("empty target with populated source", func(t *testing.T) {
source := []repository.FileTreeEntry{
{Path: "folder1/", Hash: "abc", Blob: false},
{Path: "folder1/dashboard.json", Hash: "def", Blob: true},
{Path: "folder2/", Hash: "ghi", Blob: false},
}
target := &provisioning.ResourceList{}
changes, err := Changes(source, target)
require.NoError(t, err)
require.Len(t, changes, 3) // Only non-blob entries should create changes
require.Equal(t, ResourceFileChange{
Action: repository.FileActionCreated,
Path: "folder1/dashboard.json",
}, changes[0])
require.Equal(t, ResourceFileChange{
Action: repository.FileActionCreated,
Path: "folder1/",
}, changes[1])
require.Equal(t, ResourceFileChange{
Action: repository.FileActionCreated,
Path: "folder2/",
}, changes[2])
})
}
func TestCompare(t *testing.T) {
tests := []struct {
name string
setupMocks func(*repository.MockRepository, *resources.MockRepositoryResources)
expectedError string
expectedChanges []ResourceFileChange
description string
}{
{
name: "error listing current resources",
description: "Should return error when listing current resources fails",
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) {
repoResources.On("List", mock.Anything).Return(nil, fmt.Errorf("listing failed"))
},
expectedError: "error listing current: listing failed",
},
{
name: "error reading tree",
description: "Should return error when reading tree fails",
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) {
repoResources.On("List", mock.Anything).Return(&provisioning.ResourceList{}, nil)
repo.On("ReadTree", mock.Anything, "current-ref").Return(nil, fmt.Errorf("read tree failed"))
},
expectedError: "error reading tree: read tree failed",
},
{
name: "no changes between source and target",
description: "Should return empty changes when source and target are identical",
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) {
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
{Path: "dashboard.json", Hash: "xyz"},
},
}
source := []repository.FileTreeEntry{
{Path: "dashboard.json", Hash: "xyz", Blob: true},
}
repoResources.On("List", mock.Anything).Return(target, nil)
repo.On("ReadTree", mock.Anything, "current-ref").Return(source, nil)
},
expectedChanges: []ResourceFileChange{},
},
{
name: "compare function error",
description: "Should return error when comparing fails",
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources) {
target := &provisioning.ResourceList{
Items: []provisioning.ResourceListItem{
// Empty path to trigger error
{Path: "", Hash: "xyz", Resource: "dashboard", Group: "dashboard.grafana.app"},
},
}
source := []repository.FileTreeEntry{
{Path: "dashboard.json", Hash: "xyz", Blob: true},
}
repoResources.On("List", mock.Anything).Return(target, nil)
repo.On("ReadTree", mock.Anything, "current-ref").Return(source, nil)
},
expectedError: "calculate changes: empty path on a non folder",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
tt.setupMocks(repo, repoResources)
changes, err := Compare(context.Background(), repo, repoResources, "current-ref")
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError, tt.description)
require.Nil(t, changes)
} else {
require.NoError(t, err, tt.description)
require.Equal(t, tt.expectedChanges, changes, tt.description)
}
})
}
}
@@ -0,0 +1,100 @@
// Code generated by mockery v2.52.4. DO NOT EDIT.
package sync
import (
context "context"
repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
mock "github.com/stretchr/testify/mock"
resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
// MockCompareFn is an autogenerated mock type for the CompareFn type
type MockCompareFn struct {
mock.Mock
}
type MockCompareFn_Expecter struct {
mock *mock.Mock
}
func (_m *MockCompareFn) EXPECT() *MockCompareFn_Expecter {
return &MockCompareFn_Expecter{mock: &_m.Mock}
}
// Execute provides a mock function with given fields: ctx, repo, repositoryResources, ref
func (_m *MockCompareFn) Execute(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error) {
ret := _m.Called(ctx, repo, repositoryResources, ref)
if len(ret) == 0 {
panic("no return value specified for Execute")
}
var r0 []ResourceFileChange
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, resources.RepositoryResources, string) ([]ResourceFileChange, error)); ok {
return rf(ctx, repo, repositoryResources, ref)
}
if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, resources.RepositoryResources, string) []ResourceFileChange); ok {
r0 = rf(ctx, repo, repositoryResources, ref)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]ResourceFileChange)
}
}
if rf, ok := ret.Get(1).(func(context.Context, repository.Reader, resources.RepositoryResources, string) error); ok {
r1 = rf(ctx, repo, repositoryResources, ref)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockCompareFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute'
type MockCompareFn_Execute_Call struct {
*mock.Call
}
// Execute is a helper method to define mock.On call
// - ctx context.Context
// - repo repository.Reader
// - repositoryResources resources.RepositoryResources
// - ref string
func (_e *MockCompareFn_Expecter) Execute(ctx interface{}, repo interface{}, repositoryResources interface{}, ref interface{}) *MockCompareFn_Execute_Call {
return &MockCompareFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, repositoryResources, ref)}
}
func (_c *MockCompareFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string)) *MockCompareFn_Execute_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(repository.Reader), args[2].(resources.RepositoryResources), args[3].(string))
})
return _c
}
func (_c *MockCompareFn_Execute_Call) Return(_a0 []ResourceFileChange, _a1 error) *MockCompareFn_Execute_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockCompareFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Reader, resources.RepositoryResources, string) ([]ResourceFileChange, error)) *MockCompareFn_Execute_Call {
_c.Call.Return(run)
return _c
}
// NewMockCompareFn creates a new instance of MockCompareFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockCompareFn(t interface {
mock.TestingT
Cleanup(func())
}) *MockCompareFn {
mock := &MockCompareFn{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -0,0 +1,133 @@
package sync
import (
"context"
"errors"
"fmt"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
func FullSync(
ctx context.Context,
repo repository.Reader,
compare CompareFn,
clients resources.ResourceClients,
currentRef string,
repositoryResources resources.RepositoryResources,
progress jobs.JobProgressRecorder,
) error {
cfg := repo.Config()
// Ensure the configured folder exists and is managed by the repository
rootFolder := resources.RootFolder(cfg)
if rootFolder != "" {
if err := repositoryResources.EnsureFolderExists(ctx, resources.Folder{
ID: rootFolder, // will not change if exists
Title: cfg.Spec.Title,
Path: "", // at the root of the repository
}, ""); err != nil {
return fmt.Errorf("create root folder: %w", err)
}
}
changes, err := compare(ctx, repo, repositoryResources, currentRef)
if err != nil {
return fmt.Errorf("compare changes: %w", err)
}
if len(changes) == 0 {
progress.SetFinalMessage(ctx, "no changes to sync")
return nil
}
return applyChanges(ctx, changes, clients, repositoryResources, progress)
}
func applyChanges(ctx context.Context, changes []ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error {
progress.SetTotal(ctx, len(changes))
for _, change := range changes {
if ctx.Err() != nil {
return ctx.Err()
}
if err := progress.TooManyErrors(); err != nil {
return err
}
if change.Action == repository.FileActionDeleted {
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
}
if change.Existing == nil || change.Existing.Name == "" {
result.Error = errors.New("missing existing reference")
progress.Record(ctx, result)
continue
}
result.Name = change.Existing.Name
result.Resource = change.Existing.Resource
result.Group = change.Existing.Group
versionlessGVR := schema.GroupVersionResource{
Group: change.Existing.Group,
Resource: change.Existing.Resource,
}
// TODO: should we use the clients or the resource manager instead?
client, _, err := clients.ForResource(versionlessGVR)
if err != nil {
result.Error = fmt.Errorf("get client for deleted object: %w", err)
progress.Record(ctx, result)
continue
}
result.Error = client.Delete(ctx, change.Existing.Name, metav1.DeleteOptions{})
progress.Record(ctx, result)
continue
}
// If folder ensure it exists
if safepath.IsDir(change.Path) {
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
Resource: resources.FolderResource.Resource,
Group: resources.FolderResource.Group,
}
folder, err := repositoryResources.EnsureFolderPathExist(ctx, change.Path)
if err != nil {
result.Error = err
progress.Record(ctx, result)
continue
}
result.Name = folder
progress.Record(ctx, result)
continue
}
name, gvk, err := repositoryResources.WriteResourceFromFile(ctx, change.Path, "")
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
Name: name,
Error: err,
Resource: gvk.Kind,
Group: gvk.Group,
}
progress.Record(ctx, result)
}
return nil
}
@@ -0,0 +1,93 @@
// Code generated by mockery v2.52.4. DO NOT EDIT.
package sync
import (
context "context"
jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
mock "github.com/stretchr/testify/mock"
repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
// MockFullSyncFn is an autogenerated mock type for the FullSyncFn type
type MockFullSyncFn struct {
mock.Mock
}
type MockFullSyncFn_Expecter struct {
mock *mock.Mock
}
func (_m *MockFullSyncFn) EXPECT() *MockFullSyncFn_Expecter {
return &MockFullSyncFn_Expecter{mock: &_m.Mock}
}
// Execute provides a mock function with given fields: ctx, repo, compare, clients, currentRef, repositoryResources, progress
func (_m *MockFullSyncFn) Execute(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error {
ret := _m.Called(ctx, repo, compare, clients, currentRef, repositoryResources, progress)
if len(ret) == 0 {
panic("no return value specified for Execute")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder) error); ok {
r0 = rf(ctx, repo, compare, clients, currentRef, repositoryResources, progress)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockFullSyncFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute'
type MockFullSyncFn_Execute_Call struct {
*mock.Call
}
// Execute is a helper method to define mock.On call
// - ctx context.Context
// - repo repository.Reader
// - compare CompareFn
// - clients resources.ResourceClients
// - currentRef string
// - repositoryResources resources.RepositoryResources
// - progress jobs.JobProgressRecorder
func (_e *MockFullSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, compare interface{}, clients interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}) *MockFullSyncFn_Execute_Call {
return &MockFullSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, compare, clients, currentRef, repositoryResources, progress)}
}
func (_c *MockFullSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder)) *MockFullSyncFn_Execute_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(repository.Reader), args[2].(CompareFn), args[3].(resources.ResourceClients), args[4].(string), args[5].(resources.RepositoryResources), args[6].(jobs.JobProgressRecorder))
})
return _c
}
func (_c *MockFullSyncFn_Execute_Call) Return(_a0 error) *MockFullSyncFn_Execute_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockFullSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder) error) *MockFullSyncFn_Execute_Call {
_c.Call.Return(run)
return _c
}
// NewMockFullSyncFn creates a new instance of MockFullSyncFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockFullSyncFn(t interface {
mock.TestingT
Cleanup(func())
}) *MockFullSyncFn {
mock := &MockFullSyncFn{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -0,0 +1,705 @@
package sync
import (
"context"
"errors"
"fmt"
"testing"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8testing "k8s.io/client-go/testing"
)
func TestFullSync_ContextCancelled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "Test Repo",
},
})
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{{}}, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
err := FullSync(ctx, repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
require.EqualError(t, err, "context canceled")
}
func TestFullSync_Error(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
})
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("some error"))
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
require.EqualError(t, err, "compare changes: some error")
}
func TestFullSync_NoChanges(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
})
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{}, nil)
progress.On("SetFinalMessage", mock.Anything, "no changes to sync").Return()
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
require.NoError(t, err)
}
func TestFullSync_SuccessfulFolderCreation(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "Test Repo",
Sync: provisioning.SyncOptions{
Target: provisioning.SyncTargetTypeFolder,
},
},
})
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{}, nil)
progress.On("SetFinalMessage", mock.Anything, "no changes to sync").Return()
repoResources.On("EnsureFolderExists", mock.Anything, resources.Folder{
ID: "test-repo",
Title: "Test Repo",
Path: "",
}, "").Return(nil)
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
require.NoError(t, err)
}
func TestFullSync_FolderCreationFailed(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "Test Repo",
Sync: provisioning.SyncOptions{
Target: provisioning.SyncTargetTypeFolder,
},
},
})
repoResources.On("EnsureFolderExists", mock.Anything, resources.Folder{
ID: "test-repo",
Title: "Test Repo",
Path: "",
}, "").Return(fmt.Errorf("folder creation failed"))
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
require.Error(t, err)
require.Contains(t, err.Error(), "create root folder: folder creation failed")
}
func TestFullSync_FolderCreationFailedWithInstanceTarget(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "Test Repo",
Sync: provisioning.SyncOptions{
Target: provisioning.SyncTargetTypeInstance,
},
},
})
// No folder creation should be attempted with instance target
// But we should still test the error path for completeness
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(nil, fmt.Errorf("compare error"))
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
require.Error(t, err)
require.Contains(t, err.Error(), "compare changes: compare error")
}
func TestFullSync_ApplyChanges(t *testing.T) {
tests := []struct {
name string
setupMocks func(*repository.MockRepository, *resources.MockRepositoryResources, *resources.MockResourceClients, *jobs.MockJobProgressRecorder, *MockCompareFn)
changes []ResourceFileChange
expectedError string
description string
}{
{
name: "too many errors",
description: "Should return an error when too many errors occur",
changes: []ResourceFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/one.json",
},
{
Action: repository.FileActionCreated,
Path: "dashboards/two.json",
},
{
Action: repository.FileActionCreated,
Path: "dashboards/three.json",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
// First call returns nil, second call returns error
progress.On("TooManyErrors").Return(nil).Once()
progress.On("TooManyErrors").Return(fmt.Errorf("too many errors")).Once()
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/one.json", "").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "dashboards/one.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
}).Return()
},
expectedError: "too many errors",
},
{
name: "successful apply with file creation",
description: "Should successfully apply changes when creating a new file",
changes: []ResourceFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
}).Return()
},
},
{
name: "failed apply with file creation",
description: "Should record an error when creating a new file",
changes: []ResourceFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write error"))
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
Error: fmt.Errorf("write error"),
}).Return()
},
},
{
name: "successful apply with file update",
description: "Should successfully apply changes when updating an existing file",
changes: []ResourceFileChange{
{
Action: repository.FileActionUpdated,
Path: "dashboards/test.json",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionUpdated,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
}).Return()
},
},
{
name: "failed apply with file update",
description: "Should record an error when updating a file",
changes: []ResourceFileChange{
{
Action: repository.FileActionUpdated,
Path: "dashboards/test.json",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write error"))
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionUpdated,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
Error: fmt.Errorf("write error"),
}).Return()
},
},
{
name: "successful apply with folder creation",
description: "Should successfully apply changes when creating a new folder",
changes: []ResourceFileChange{
{
Action: repository.FileActionCreated,
Path: "one/two/three/",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
repoResources.On("EnsureFolderPathExist", mock.Anything, "one/two/three/").Return("some-folder", nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "one/two/three/",
Name: "some-folder",
// FIXME: this is probably inconsistent across the codebase
Resource: "folders",
Group: "folder.grafana.app",
}).Return()
},
},
{
name: "failed apply folder creation",
description: "Should record an error when creating a new folder",
changes: []ResourceFileChange{
{
Action: repository.FileActionCreated,
Path: "one/two/three/",
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
repoResources.On(
"EnsureFolderPathExist",
mock.Anything,
"one/two/three/",
).Return("some-folder", errors.New("folder creation error"))
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "one/two/three/",
Name: "",
// FIXME: this is probably inconsistent across the codebase
Resource: "folders",
Group: "folder.grafana.app",
Error: errors.New("folder creation error"),
}).Return()
},
},
{
name: "successful apply with file deletion",
description: "Should successfully apply changes when deleting an existing file",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Existing: &provisioning.ResourceListItem{
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
},
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
scheme := runtime.NewScheme()
require.NoError(t, metav1.AddMetaToScheme(scheme))
listGVK := schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: "DashboardList",
}
scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{})
scheme.AddKnownTypeWithName(schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: resources.DashboardResource.Resource,
}, &metav1.PartialObjectMetadata{})
fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{
resources.DashboardResource: listGVK.Kind,
})
fakeDynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
clients.On("ForResource", schema.GroupVersionResource{
Group: "dashboards",
Resource: "Dashboard",
}).Return(fakeDynamicClient.Resource(resources.DashboardResource), schema.GroupVersionKind{
Kind: "Dashboard",
Group: "dashboards",
Version: "v1",
}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
Error: nil,
}).Return()
},
},
{
name: "file delete error",
description: "Should return an error when deleting a file",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Existing: &provisioning.ResourceListItem{
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
},
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
scheme := runtime.NewScheme()
require.NoError(t, metav1.AddMetaToScheme(scheme))
listGVK := schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: "DashboardList",
}
scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{})
scheme.AddKnownTypeWithName(schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: resources.DashboardResource.Resource,
}, &metav1.PartialObjectMetadata{})
fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{
resources.DashboardResource: listGVK.Kind,
})
fakeDynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("delete failed")
})
clients.On("ForResource", schema.GroupVersionResource{
Group: "dashboards",
Resource: "Dashboard",
}).Return(fakeDynamicClient.Resource(resources.DashboardResource), schema.GroupVersionKind{
Kind: "Dashboard",
Group: "dashboards",
Version: "v1",
}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
Error: fmt.Errorf("delete failed"),
}).Return()
},
},
{
name: "without existing for delete",
description: "Should record an error when deleting a file",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Existing: nil,
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Error: fmt.Errorf("missing existing reference"),
}).Return()
},
},
{
name: "without existing name for delete",
description: "Should record an error when deleting a file",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Existing: &provisioning.ResourceListItem{},
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Error: fmt.Errorf("missing existing reference"),
}).Return()
},
},
{
name: "error finding client for delete",
description: "Should record an error when deleting a file",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Existing: &provisioning.ResourceListItem{
Name: "test-dashboard",
Group: "dashboards",
Resource: "Dashboard",
},
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
clients.On("ForResource", schema.GroupVersionResource{
Group: "dashboards",
Resource: "Dashboard",
}).Return(nil, schema.GroupVersionKind{}, errors.New("didn't work"))
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Name: "test-dashboard",
Group: "dashboards",
Resource: "Dashboard",
Action: repository.FileActionDeleted,
Path: "dashboards/test.json",
Error: fmt.Errorf("get client for deleted object: %w", errors.New("didn't work")),
}).Return()
},
},
{
name: "successful apply with folder deletion",
description: "Should successfully apply changes when deleting an existing folder",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "to-be-deleted/",
Existing: &provisioning.ResourceListItem{
Name: "test-folder",
Resource: "Folder",
Group: "folders",
},
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
scheme := runtime.NewScheme()
require.NoError(t, metav1.AddMetaToScheme(scheme))
listGVK := schema.GroupVersionKind{
Group: resources.FolderResource.Group,
Version: resources.FolderResource.Version,
Kind: "FolderList",
}
scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{})
scheme.AddKnownTypeWithName(schema.GroupVersionKind{
Group: resources.FolderResource.Group,
Version: resources.FolderResource.Version,
Kind: resources.FolderResource.Resource,
}, &metav1.PartialObjectMetadata{})
fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{
resources.FolderResource: listGVK.Kind,
})
fakeDynamicClient.PrependReactor("delete", "folders", func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
clients.On("ForResource", schema.GroupVersionResource{
Group: "folders",
Resource: "Folder",
}).Return(fakeDynamicClient.Resource(resources.FolderResource), schema.GroupVersionKind{
Kind: "Folder",
Group: "folders",
Version: "v1",
}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "to-be-deleted/",
Name: "test-folder",
Resource: "Folder",
Group: "folders",
Error: nil,
}).Return()
},
},
{
name: "failed to apply with folder deletion",
description: "Should record an error when deleting a folder",
changes: []ResourceFileChange{
{
Action: repository.FileActionDeleted,
Path: "to-be-deleted/",
Existing: &provisioning.ResourceListItem{
Name: "test-folder",
Resource: "Folder",
Group: "folders",
},
},
},
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
progress.On("TooManyErrors").Return(nil)
scheme := runtime.NewScheme()
require.NoError(t, metav1.AddMetaToScheme(scheme))
listGVK := schema.GroupVersionKind{
Group: resources.FolderResource.Group,
Version: resources.FolderResource.Version,
Kind: "FolderList",
}
scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{})
scheme.AddKnownTypeWithName(schema.GroupVersionKind{
Group: resources.FolderResource.Group,
Version: resources.FolderResource.Version,
Kind: resources.FolderResource.Resource,
}, &metav1.PartialObjectMetadata{})
fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{
resources.FolderResource: listGVK.Kind,
})
fakeDynamicClient.PrependReactor("delete", "folders", func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("delete failed")
})
clients.On("ForResource", schema.GroupVersionResource{
Group: "folders",
Resource: "Folder",
}).Return(fakeDynamicClient.Resource(resources.FolderResource), schema.GroupVersionKind{
Kind: "Folder",
Group: "folders",
Version: "v1",
}, nil)
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "to-be-deleted/",
Name: "test-folder",
Resource: "Folder",
Group: "folders",
Error: fmt.Errorf("delete failed"),
}).Return()
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := repository.NewMockRepository(t)
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
tt.setupMocks(repo, repoResources, clients, progress, compareFn)
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.changes, nil)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "Test Repo",
},
})
progress.On("SetTotal", mock.Anything, len(tt.changes)).Return()
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress)
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError, tt.description)
} else {
require.NoError(t, err, tt.description)
}
})
}
}
@@ -0,0 +1,111 @@
package sync
import (
"context"
"fmt"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
)
// Convert git changes into resource file changes
func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error {
if previousRef == currentRef {
progress.SetFinalMessage(ctx, "same commit as last time")
return nil
}
diff, err := repo.CompareFiles(ctx, previousRef, currentRef)
if err != nil {
return fmt.Errorf("compare files error: %w", err)
}
if len(diff) < 1 {
progress.SetFinalMessage(ctx, "no changes detected between commits")
return nil
}
progress.SetTotal(ctx, len(diff))
progress.SetMessage(ctx, "replicating versioned changes")
for _, change := range diff {
if ctx.Err() != nil {
return ctx.Err()
}
if err := progress.TooManyErrors(); err != nil {
return err
}
if err := resources.IsPathSupported(change.Path); err != nil {
// Maintain the safe segment for empty folders
safeSegment := safepath.SafeSegment(change.Path)
if !safepath.IsDir(safeSegment) {
safeSegment = safepath.Dir(safeSegment)
}
if safeSegment != "" && resources.IsPathSupported(safeSegment) == nil {
folder, err := repositoryResources.EnsureFolderPathExist(ctx, safeSegment)
if err != nil {
return fmt.Errorf("unable to create empty file folder: %w", err)
}
progress.Record(ctx, jobs.JobResourceResult{
Path: safeSegment,
Action: repository.FileActionCreated,
Resource: resources.FolderResource.Resource,
Group: resources.FolderResource.Group,
Name: folder,
})
continue
}
progress.Record(ctx, jobs.JobResourceResult{
Path: change.Path,
Action: repository.FileActionIgnored,
})
continue
}
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
}
switch change.Action {
case repository.FileActionCreated, repository.FileActionUpdated:
name, gvk, err := repositoryResources.WriteResourceFromFile(ctx, change.Path, change.Ref)
if err != nil {
result.Error = err
}
result.Name = name
result.Resource = gvk.Kind
result.Group = gvk.Group
case repository.FileActionDeleted:
name, gvk, err := repositoryResources.RemoveResourceFromFile(ctx, change.Path, change.PreviousRef)
if err != nil {
result.Error = err
}
result.Name = name
result.Resource = gvk.Kind
result.Group = gvk.Group
case repository.FileActionRenamed:
name, gvk, err := repositoryResources.RenameResourceFile(ctx, change.Path, change.PreviousRef, change.Path, change.Ref)
if err != nil {
result.Error = err
}
result.Name = name
result.Resource = gvk.Kind
result.Group = gvk.Group
case repository.FileActionIgnored:
// do nothing
}
progress.Record(ctx, result)
}
progress.SetMessage(ctx, "versioned changes replicated")
return nil
}
@@ -0,0 +1,92 @@
// Code generated by mockery v2.52.4. DO NOT EDIT.
package sync
import (
context "context"
jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
mock "github.com/stretchr/testify/mock"
repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
// MockIncrementalSyncFn is an autogenerated mock type for the IncrementalSyncFn type
type MockIncrementalSyncFn struct {
mock.Mock
}
type MockIncrementalSyncFn_Expecter struct {
mock *mock.Mock
}
func (_m *MockIncrementalSyncFn) EXPECT() *MockIncrementalSyncFn_Expecter {
return &MockIncrementalSyncFn_Expecter{mock: &_m.Mock}
}
// Execute provides a mock function with given fields: ctx, repo, previousRef, currentRef, repositoryResources, progress
func (_m *MockIncrementalSyncFn) Execute(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error {
ret := _m.Called(ctx, repo, previousRef, currentRef, repositoryResources, progress)
if len(ret) == 0 {
panic("no return value specified for Execute")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder) error); ok {
r0 = rf(ctx, repo, previousRef, currentRef, repositoryResources, progress)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockIncrementalSyncFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute'
type MockIncrementalSyncFn_Execute_Call struct {
*mock.Call
}
// Execute is a helper method to define mock.On call
// - ctx context.Context
// - repo repository.Versioned
// - previousRef string
// - currentRef string
// - repositoryResources resources.RepositoryResources
// - progress jobs.JobProgressRecorder
func (_e *MockIncrementalSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, previousRef interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}) *MockIncrementalSyncFn_Execute_Call {
return &MockIncrementalSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, previousRef, currentRef, repositoryResources, progress)}
}
func (_c *MockIncrementalSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder)) *MockIncrementalSyncFn_Execute_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(repository.Versioned), args[2].(string), args[3].(string), args[4].(resources.RepositoryResources), args[5].(jobs.JobProgressRecorder))
})
return _c
}
func (_c *MockIncrementalSyncFn_Execute_Call) Return(_a0 error) *MockIncrementalSyncFn_Execute_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockIncrementalSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder) error) *MockIncrementalSyncFn_Execute_Call {
_c.Call.Return(run)
return _c
}
// NewMockIncrementalSyncFn creates a new instance of MockIncrementalSyncFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockIncrementalSyncFn(t interface {
mock.TestingT
Cleanup(func())
}) *MockIncrementalSyncFn {
mock := &MockIncrementalSyncFn{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -0,0 +1,396 @@
package sync
import (
"context"
"fmt"
"testing"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime/schema"
)
func TestIncrementalSync_ContextCancelled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
repo := repository.NewMockVersioned(t)
repoResources := resources.NewMockRepositoryResources(t)
progress := jobs.NewMockJobProgressRecorder(t)
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return([]repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Ref: "new-ref",
},
}, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
err := IncrementalSync(ctx, repo, "old-ref", "new-ref", repoResources, progress)
require.EqualError(t, err, "context canceled")
}
func TestIncrementalSync(t *testing.T) {
tests := []struct {
name string
setupMocks func(*repository.MockVersioned, *resources.MockRepositoryResources, *jobs.MockJobProgressRecorder)
previousRef string
currentRef string
expectedError string
expectedCalls int
expectedFiles []repository.VersionedFileChange
}{
{
name: "same commit as last time",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
progress.On("SetFinalMessage", mock.Anything, "same commit as last time").Return()
},
previousRef: "old-ref",
currentRef: "old-ref",
},
{
name: "no changes between commits",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return([]repository.VersionedFileChange{}, nil)
progress.On("SetFinalMessage", mock.Anything, "no changes detected between commits").Return()
},
previousRef: "old-ref",
currentRef: "new-ref",
},
{
name: "error comparing files",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(nil, fmt.Errorf("compare error"))
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedError: "compare files error: compare error",
},
{
name: "successful sync with file changes",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Ref: "new-ref",
},
{
Action: repository.FileActionUpdated,
Path: "alerts/alert.yaml",
Ref: "new-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 2).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
// Mock successful resource writes
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "new-ref").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
repoResources.On("WriteResourceFromFile", mock.Anything, "alerts/alert.yaml", "new-ref").
Return("test-alert", schema.GroupVersionKind{Kind: "Alert", Group: "alerts"}, nil)
// Mock progress recording
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Action == repository.FileActionCreated && result.Path == "dashboards/test.json"
})).Return()
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Action == repository.FileActionUpdated && result.Path == "alerts/alert.yaml"
})).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 2,
},
{
name: "unsupported file path with valid folder",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: "unsupported/path/file.txt",
Ref: "new-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
// Mock folder creation
repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/path/").
Return("test-folder", nil)
// Mock progress recording
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "unsupported/path/",
Resource: resources.FolderResource.Resource,
Group: resources.FolderResource.Group,
Name: "test-folder",
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "unsupported file path with invalid folder",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: ".unsupported/path/file.txt",
Ref: "new-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionIgnored,
Path: ".unsupported/path/file.txt",
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "file deletion",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/old.json",
PreviousRef: "old-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
// Mock resource deletion
repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/old.json", "old-ref").
Return("old-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
// Mock progress recording
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "dashboards/old.json",
Name: "old-dashboard",
Resource: "Dashboard",
Group: "dashboards",
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "file rename",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionRenamed,
Path: "dashboards/new.json",
PreviousPath: "dashboards/old.json",
Ref: "new-ref",
PreviousRef: "old-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
// Mock resource rename
repoResources.On("RenameResourceFile", mock.Anything, "dashboards/new.json", "old-ref", "dashboards/new.json", "new-ref").
Return("renamed-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
// Mock progress recording
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionRenamed,
Path: "dashboards/new.json",
Name: "renamed-dashboard",
Resource: "Dashboard",
Group: "dashboards",
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "file ignored",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionIgnored,
Path: "dashboards/ignored.json",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionIgnored,
Path: "dashboards/ignored.json",
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "error creating folder",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: "unsupported/path/file.txt",
Ref: "new-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
// Mock folder creation error
repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/path/").
Return("", fmt.Errorf("failed to create folder"))
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedError: "unable to create empty file folder: failed to create folder",
},
{
name: "error writing resource",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Ref: "new-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
// Mock resource write error
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "new-ref").
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write failed"))
// Mock progress recording with error
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Name: "test-dashboard",
Resource: "Dashboard",
Group: "dashboards",
Error: fmt.Errorf("write failed"),
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "error deleting resource",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionDeleted,
Path: "dashboards/old.json",
PreviousRef: "old-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
// Mock resource deletion error
repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/old.json", "old-ref").
Return("old-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("delete failed"))
// Mock progress recording with error
progress.On("Record", mock.Anything, jobs.JobResourceResult{
Action: repository.FileActionDeleted,
Path: "dashboards/old.json",
Name: "old-dashboard",
Resource: "Dashboard",
Group: "dashboards",
Error: fmt.Errorf("delete failed"),
}).Return()
progress.On("TooManyErrors").Return(nil)
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedCalls: 1,
},
{
name: "too many errors",
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
changes := []repository.VersionedFileChange{
{
Action: repository.FileActionCreated,
Path: "dashboards/test.json",
Ref: "new-ref",
},
}
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
progress.On("SetTotal", mock.Anything, 1).Return()
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
// Mock too many errors
progress.On("TooManyErrors").Return(fmt.Errorf("too many errors occurred"))
},
previousRef: "old-ref",
currentRef: "new-ref",
expectedError: "too many errors occurred",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := repository.NewMockVersioned(t)
repoResources := resources.NewMockRepositoryResources(t)
progress := jobs.NewMockJobProgressRecorder(t)
tt.setupMocks(repo, repoResources, progress)
err := IncrementalSync(context.Background(), repo, tt.previousRef, tt.currentRef, repoResources, progress)
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError)
} else {
require.NoError(t, err)
}
})
}
}
@@ -0,0 +1,85 @@
// Code generated by mockery v2.52.4. DO NOT EDIT.
package sync
import (
context "context"
v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
mock "github.com/stretchr/testify/mock"
)
// MockRepositoryPatchFn is an autogenerated mock type for the RepositoryPatchFn type
type MockRepositoryPatchFn struct {
mock.Mock
}
type MockRepositoryPatchFn_Expecter struct {
mock *mock.Mock
}
func (_m *MockRepositoryPatchFn) EXPECT() *MockRepositoryPatchFn_Expecter {
return &MockRepositoryPatchFn_Expecter{mock: &_m.Mock}
}
// Execute provides a mock function with given fields: ctx, repo, ops
func (_m *MockRepositoryPatchFn) Execute(ctx context.Context, repo *v0alpha1.Repository, ops []map[string]interface{}) error {
ret := _m.Called(ctx, repo, ops)
if len(ret) == 0 {
panic("no return value specified for Execute")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Repository, []map[string]interface{}) error); ok {
r0 = rf(ctx, repo, ops)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockRepositoryPatchFn_Execute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Execute'
type MockRepositoryPatchFn_Execute_Call struct {
*mock.Call
}
// Execute is a helper method to define mock.On call
// - ctx context.Context
// - repo *v0alpha1.Repository
// - ops []map[string]interface{}
func (_e *MockRepositoryPatchFn_Expecter) Execute(ctx interface{}, repo interface{}, ops interface{}) *MockRepositoryPatchFn_Execute_Call {
return &MockRepositoryPatchFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, ops)}
}
func (_c *MockRepositoryPatchFn_Execute_Call) Run(run func(ctx context.Context, repo *v0alpha1.Repository, ops []map[string]interface{})) *MockRepositoryPatchFn_Execute_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*v0alpha1.Repository), args[2].([]map[string]interface{}))
})
return _c
}
func (_c *MockRepositoryPatchFn_Execute_Call) Return(_a0 error) *MockRepositoryPatchFn_Execute_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockRepositoryPatchFn_Execute_Call) RunAndReturn(run func(context.Context, *v0alpha1.Repository, []map[string]interface{}) error) *MockRepositoryPatchFn_Execute_Call {
_c.Call.Return(run)
return _c
}
// NewMockRepositoryPatchFn creates a new instance of MockRepositoryPatchFn. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockRepositoryPatchFn(t interface {
mock.TestingT
Cleanup(func())
}) *MockRepositoryPatchFn {
mock := &MockRepositoryPatchFn{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -0,0 +1,62 @@
package sync
import (
"context"
"fmt"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
//go:generate mockery --name FullSyncFn --structname MockFullSyncFn --inpackage --filename full_sync_fn_mock.go --with-expecter
type FullSyncFn func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error
//go:generate mockery --name CompareFn --structname MockCompareFn --inpackage --filename compare_fn_mock.go --with-expecter
type CompareFn func(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error)
//go:generate mockery --name IncrementalSyncFn --structname MockIncrementalSyncFn --inpackage --filename incremental_sync_fn_mock.go --with-expecter
type IncrementalSyncFn func(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error
//go:generate mockery --name Syncer --structname MockSyncer --inpackage --filename syncer_mock.go --with-expecter
type Syncer interface {
Sync(ctx context.Context, repo repository.ReaderWriter, options provisioning.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder) (string, error)
}
type syncer struct {
compare CompareFn
fullSync FullSyncFn
incrementalSync IncrementalSyncFn
}
func NewSyncer(compare CompareFn, fullSync FullSyncFn, incrementalSync IncrementalSyncFn) Syncer {
return &syncer{
compare: compare,
fullSync: fullSync,
incrementalSync: incrementalSync,
}
}
func (r *syncer) Sync(ctx context.Context, repo repository.ReaderWriter, options provisioning.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder) (string, error) {
cfg := repo.Config()
var currentRef string
versionedRepo, ok := repo.(repository.Versioned)
if ok && versionedRepo != nil {
var err error
currentRef, err = versionedRepo.LatestRef(ctx)
if err != nil {
return "", fmt.Errorf("get latest ref: %w", err)
}
if cfg.Status.Sync.LastRef != "" && options.Incremental {
progress.SetMessage(ctx, "incremental sync")
return currentRef, r.incrementalSync(ctx, versionedRepo, cfg.Status.Sync.LastRef, currentRef, repositoryResources, progress)
}
}
progress.SetMessage(ctx, "full sync")
return currentRef, r.fullSync(ctx, repo, r.compare, clients, currentRef, repositoryResources, progress)
}
@@ -0,0 +1,180 @@
package sync
import (
"context"
"fmt"
"testing"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
mock "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type mockReaderWriter struct {
*repository.MockRepository
*repository.MockVersioned
}
// FIXME: understand how the MockRepository was generated as it seems
// stale and it's causing collisions for the embedded
func (m *mockReaderWriter) History(ctx context.Context, path, ref string) ([]provisioning.HistoryItem, error) {
return m.MockVersioned.History(ctx, path, ref)
}
func (m *mockReaderWriter) LatestRef(ctx context.Context) (string, error) {
return m.MockVersioned.LatestRef(ctx)
}
func (m *mockReaderWriter) CompareFiles(ctx context.Context, base, ref string) ([]repository.VersionedFileChange, error) {
return m.MockVersioned.CompareFiles(ctx, base, ref)
}
func TestSyncer_Sync(t *testing.T) {
tests := []struct {
name string
options provisioning.SyncJobOptions
setupMocks func(*mockReaderWriter, *resources.MockRepositoryResources, *resources.MockResourceClients, *jobs.MockJobProgressRecorder, *MockCompareFn, *MockFullSyncFn, *MockIncrementalSyncFn)
expectedRef string
expectedError string
expectedMessages []string
expectedFinalMsg string
}{
{
name: "successful full sync",
options: provisioning.SyncJobOptions{
Incremental: false,
},
setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) {
repo.MockRepository.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "Test Repo",
},
})
repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil)
progress.On("SetMessage", mock.Anything, "full sync").Return()
fullSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, mock.Anything, mock.Anything, "new-ref", mock.Anything, mock.Anything).Return(nil)
},
expectedMessages: []string{"full sync"},
},
{
name: "successful incremental sync",
options: provisioning.SyncJobOptions{
Incremental: true,
},
setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) {
repo.MockRepository.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "old-ref",
},
},
})
repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil)
progress.On("SetMessage", mock.Anything, "incremental sync").Return()
incrementalSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything).Return(nil)
},
expectedRef: "new-ref",
expectedMessages: []string{"incremental sync"},
},
{
name: "latest ref error",
options: provisioning.SyncJobOptions{
Incremental: true,
},
setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) {
repo.MockRepository.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "old-ref",
},
},
})
repo.MockVersioned.On("LatestRef", mock.Anything).Return("", fmt.Errorf("failed to get latest ref"))
},
expectedError: "get latest ref: failed to get latest ref",
},
{
name: "incremental sync error",
options: provisioning.SyncJobOptions{
Incremental: true,
},
setupMocks: func(repo *mockReaderWriter, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn, fullSyncFn *MockFullSyncFn, incrementalSyncFn *MockIncrementalSyncFn) {
repo.MockRepository.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "old-ref",
},
},
})
repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil)
progress.On("SetMessage", mock.Anything, "incremental sync").Return()
incrementalSyncFn.On("Execute", mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything).Return(fmt.Errorf("incremental sync failed"))
},
expectedRef: "new-ref",
expectedMessages: []string{"incremental sync"},
expectedError: "incremental sync failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repoResources := resources.NewMockRepositoryResources(t)
clients := resources.NewMockResourceClients(t)
progress := jobs.NewMockJobProgressRecorder(t)
compareFn := NewMockCompareFn(t)
fullSyncFn := NewMockFullSyncFn(t)
incrementalSyncFn := NewMockIncrementalSyncFn(t)
repo := &mockReaderWriter{
MockRepository: repository.NewMockRepository(t),
MockVersioned: repository.NewMockVersioned(t),
}
tt.setupMocks(repo, repoResources, clients, progress, compareFn, fullSyncFn, incrementalSyncFn)
syncer := NewSyncer(
compareFn.Execute,
fullSyncFn.Execute,
incrementalSyncFn.Execute,
)
ref, err := syncer.Sync(context.Background(), repo, tt.options, repoResources, clients, progress)
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError)
} else {
require.NoError(t, err)
if tt.expectedRef != "" {
require.Equal(t, tt.expectedRef, ref)
}
}
// Verify expected messages were set
if len(tt.expectedMessages) > 0 {
for _, msg := range tt.expectedMessages {
progress.AssertCalled(t, "SetMessage", mock.Anything, msg)
}
}
if tt.expectedFinalMsg != "" {
progress.AssertCalled(t, "SetFinalMessage", mock.Anything, tt.expectedFinalMsg)
}
})
}
}
@@ -0,0 +1,104 @@
// Code generated by mockery v2.52.4. DO NOT EDIT.
package sync
import (
context "context"
jobs "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
mock "github.com/stretchr/testify/mock"
repository "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
)
// MockSyncer is an autogenerated mock type for the Syncer type
type MockSyncer struct {
mock.Mock
}
type MockSyncer_Expecter struct {
mock *mock.Mock
}
func (_m *MockSyncer) EXPECT() *MockSyncer_Expecter {
return &MockSyncer_Expecter{mock: &_m.Mock}
}
// Sync provides a mock function with given fields: ctx, repo, options, repositoryResources, clients, progress
func (_m *MockSyncer) Sync(ctx context.Context, repo repository.ReaderWriter, options v0alpha1.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder) (string, error) {
ret := _m.Called(ctx, repo, options, repositoryResources, clients, progress)
if len(ret) == 0 {
panic("no return value specified for Sync")
}
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) (string, error)); ok {
return rf(ctx, repo, options, repositoryResources, clients, progress)
}
if rf, ok := ret.Get(0).(func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) string); ok {
r0 = rf(ctx, repo, options, repositoryResources, clients, progress)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) error); ok {
r1 = rf(ctx, repo, options, repositoryResources, clients, progress)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockSyncer_Sync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Sync'
type MockSyncer_Sync_Call struct {
*mock.Call
}
// Sync is a helper method to define mock.On call
// - ctx context.Context
// - repo repository.ReaderWriter
// - options v0alpha1.SyncJobOptions
// - repositoryResources resources.RepositoryResources
// - clients resources.ResourceClients
// - progress jobs.JobProgressRecorder
func (_e *MockSyncer_Expecter) Sync(ctx interface{}, repo interface{}, options interface{}, repositoryResources interface{}, clients interface{}, progress interface{}) *MockSyncer_Sync_Call {
return &MockSyncer_Sync_Call{Call: _e.mock.On("Sync", ctx, repo, options, repositoryResources, clients, progress)}
}
func (_c *MockSyncer_Sync_Call) Run(run func(ctx context.Context, repo repository.ReaderWriter, options v0alpha1.SyncJobOptions, repositoryResources resources.RepositoryResources, clients resources.ResourceClients, progress jobs.JobProgressRecorder)) *MockSyncer_Sync_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(repository.ReaderWriter), args[2].(v0alpha1.SyncJobOptions), args[3].(resources.RepositoryResources), args[4].(resources.ResourceClients), args[5].(jobs.JobProgressRecorder))
})
return _c
}
func (_c *MockSyncer_Sync_Call) Return(_a0 string, _a1 error) *MockSyncer_Sync_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockSyncer_Sync_Call) RunAndReturn(run func(context.Context, repository.ReaderWriter, v0alpha1.SyncJobOptions, resources.RepositoryResources, resources.ResourceClients, jobs.JobProgressRecorder) (string, error)) *MockSyncer_Sync_Call {
_c.Call.Return(run)
return _c
}
// NewMockSyncer creates a new instance of MockSyncer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockSyncer(t interface {
mock.TestingT
Cleanup(func())
}) *MockSyncer {
mock := &MockSyncer{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -2,56 +2,51 @@ package sync
import (
"context"
"encoding/json"
"errors"
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"github.com/grafana/grafana-app-sdk/logging"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
client "github.com/grafana/grafana/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/safepath"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
)
//go:generate mockery --name RepositoryPatchFn --structname MockRepositoryPatchFn --inpackage --filename repository_patch_fn_mock.go --with-expecter
type RepositoryPatchFn func(ctx context.Context, repo *provisioning.Repository, ops []map[string]interface{}) error
// SyncWorker synchronizes the external repo with grafana database
// this function updates the status for both the job and the referenced repository
type SyncWorker struct {
// Used to update the repository status with sync info
client client.ProvisioningV0alpha1Interface
// Lists the values saved in grafana database
lister resources.ResourceLister
// Parses fields saved in remore repository
parsers resources.ParserFactory
// Clients for the repository
clients resources.ClientFactory
// ResourceClients for the repository
repositoryResources resources.RepositoryResourcesFactory
// Check if the system is using unified storage
storageStatus dualwrite.Service
// Patch status for the repository
patchStatus RepositoryPatchFn
// Sync functions
syncer Syncer
}
func NewSyncWorker(
client client.ProvisioningV0alpha1Interface,
parsers resources.ParserFactory,
clients resources.ClientFactory,
lister resources.ResourceLister,
repositoryResources resources.RepositoryResourcesFactory,
storageStatus dualwrite.Service,
patchStatus RepositoryPatchFn,
syncer Syncer,
) *SyncWorker {
return &SyncWorker{
client: client,
parsers: parsers,
clients: clients,
lister: lister,
storageStatus: storageStatus,
clients: clients,
repositoryResources: repositoryResources,
patchStatus: patchStatus,
storageStatus: storageStatus,
syncer: syncer,
}
}
@@ -74,7 +69,8 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
syncStatus := job.Status.ToSyncStatus(job.Name)
// Preserve last ref as we use replace operation
syncStatus.LastRef = repo.Config().Status.Sync.LastRef
lastRef := repo.Config().Status.Sync.LastRef
syncStatus.LastRef = lastRef
// Update sync status at start using JSON patch
patchOperations := []map[string]interface{}{
@@ -90,19 +86,26 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
return fmt.Errorf("update repo with job status at start: %w", err)
}
progress.SetMessage(ctx, "execute sync job")
syncJob, err := r.createJob(ctx, rw, progress)
repositoryResources, err := r.repositoryResources.Client(ctx, rw)
if err != nil {
return fmt.Errorf("failed to create sync job: %w", err)
return fmt.Errorf("create repository resources client: %w", err)
}
syncError := syncJob.run(ctx, *job.Spec.Pull)
clients, err := r.clients.Clients(ctx, cfg.Namespace)
if err != nil {
return fmt.Errorf("get clients for %s: %w", cfg.Name, err)
}
progress.SetMessage(ctx, "execute sync job")
currentRef, syncError := r.syncer.Sync(ctx, rw, *job.Spec.Pull, repositoryResources, clients, progress)
jobStatus := progress.Complete(ctx, syncError)
syncStatus = jobStatus.ToSyncStatus(job.Name)
// Create sync status and set hash if successful
if syncStatus.State == provisioning.JobStateSuccess {
syncStatus.LastRef = progress.GetRef()
syncStatus.LastRef = currentRef
} else {
syncStatus.LastRef = lastRef
}
// Update final status using JSON patch
@@ -116,14 +119,20 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
}
// Only add stats patch if stats are not nil
if stats, err := r.lister.Stats(ctx, cfg.Namespace, cfg.Name); err != nil {
stats, err := repositoryResources.Stats(ctx)
switch {
case err != nil:
logger.Error("unable to read stats", "error", err)
} else if stats != nil && len(stats.Managed) == 1 {
case stats == nil:
logger.Error("stats are nil")
case len(stats.Managed) == 1:
patchOperations = append(patchOperations, map[string]interface{}{
"op": "replace",
"path": "/status/stats",
"value": stats.Managed[0].Stats,
})
default:
logger.Warn("unexpected number of managed stats", "count", len(stats.Managed))
}
// Only patch the specific fields we want to update, not the entire status
@@ -133,298 +142,3 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
return syncError
}
// start a job and run it
func (r *SyncWorker) createJob(ctx context.Context, repo repository.ReaderWriter, progress jobs.JobProgressRecorder) (*syncJob, error) {
cfg := repo.Config()
parser, err := r.parsers.GetParser(ctx, repo)
if err != nil {
return nil, fmt.Errorf("failed to get parser for %s: %w", cfg.Name, err)
}
clients, err := r.clients.Clients(ctx, cfg.Namespace)
if err != nil {
return nil, fmt.Errorf("failed to get clients for %s: %w", cfg.Name, err)
}
folderClient, err := clients.Folder()
if err != nil {
return nil, fmt.Errorf("unable to get folder client: %w", err)
}
folders := resources.NewFolderManager(repo, folderClient, resources.NewEmptyFolderTree())
job := &syncJob{
repository: repo,
progress: progress,
lister: r.lister,
folders: folders,
clients: clients,
resourceManager: resources.NewResourcesManager(repo, folders, parser, clients, nil),
}
return job, nil
}
func (r *SyncWorker) patchStatus(ctx context.Context, repo *provisioning.Repository, patchOperations []map[string]interface{}) error {
patch, err := json.Marshal(patchOperations)
if err != nil {
return fmt.Errorf("unable to marshal patch data: %w", err)
}
_, err = r.client.Repositories(repo.Namespace).
Patch(ctx, repo.Name, types.JSONPatchType, patch, metav1.PatchOptions{}, "status")
if err != nil {
return fmt.Errorf("unable to update repo with job status: %w", err)
}
return nil
}
// created once for each sync execution
type syncJob struct {
repository repository.Reader
progress jobs.JobProgressRecorder
lister resources.ResourceLister
clients resources.ResourceClients
folders *resources.FolderManager
resourceManager *resources.ResourcesManager
}
func (r *syncJob) run(ctx context.Context, options provisioning.SyncJobOptions) error {
// Ensure the configured folder exists and is managed by the repository
cfg := r.repository.Config()
rootFolder := resources.RootFolder(cfg)
if rootFolder != "" {
if err := r.folders.EnsureFolderExists(ctx, resources.Folder{
ID: rootFolder, // will not change if exists
Title: cfg.Spec.Title,
Path: "", // at the root of the repository
}, ""); err != nil {
return fmt.Errorf("unable to create root folder: %w", err)
}
}
var err error
var currentRef string
versionedRepo, _ := r.repository.(repository.Versioned)
if versionedRepo != nil {
currentRef, err = versionedRepo.LatestRef(ctx)
if err != nil {
return fmt.Errorf("getting latest ref: %w", err)
}
r.progress.SetRef(currentRef)
if cfg.Status.Sync.LastRef != "" && options.Incremental {
if currentRef == cfg.Status.Sync.LastRef {
r.progress.SetFinalMessage(ctx, "same commit as last sync")
return nil
}
return r.applyVersionedChanges(ctx, versionedRepo, cfg.Status.Sync.LastRef, currentRef)
}
}
// Read the complete change set
target, err := r.lister.List(ctx, cfg.Namespace, cfg.Name)
if err != nil {
return fmt.Errorf("error listing current: %w", err)
}
source, err := r.repository.ReadTree(ctx, currentRef)
if err != nil {
return fmt.Errorf("error reading tree: %w", err)
}
changes, err := Changes(source, target)
if err != nil {
return fmt.Errorf("error calculating changes: %w", err)
}
if len(changes) == 0 {
r.progress.SetFinalMessage(ctx, "no changes to sync")
return nil
}
r.folders.SetTree(resources.NewFolderTreeFromResourceList(target))
// Now apply the changes
return r.applyChanges(ctx, changes)
}
func (r *syncJob) applyChanges(ctx context.Context, changes []ResourceFileChange) error {
r.progress.SetTotal(ctx, len(changes))
r.progress.SetMessage(ctx, "replicating changes")
for _, change := range changes {
if ctx.Err() != nil {
return ctx.Err()
}
if err := r.progress.TooManyErrors(); err != nil {
return err
}
if change.Action == repository.FileActionDeleted {
result := jobs.JobResourceResult{
Name: change.Existing.Name,
Resource: change.Existing.Resource,
Group: change.Existing.Group,
Path: change.Path,
Action: change.Action,
}
if change.Existing == nil || change.Existing.Name == "" {
result.Error = errors.New("missing existing reference")
r.progress.Record(ctx, result)
continue
}
versionlessGVR := schema.GroupVersionResource{
Group: change.Existing.Group,
Resource: change.Existing.Resource,
}
// TODO: should we use the clients or the resource manager instead?
client, _, err := r.clients.ForResource(versionlessGVR)
if err != nil {
result.Error = fmt.Errorf("unable to get client for deleted object: %w", err)
r.progress.Record(ctx, result)
continue
}
result.Error = client.Delete(ctx, change.Existing.Name, metav1.DeleteOptions{})
r.progress.Record(ctx, result)
continue
}
// If folder ensure it exists
if safepath.IsDir(change.Path) {
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
}
folder, err := r.folders.EnsureFolderPathExist(ctx, change.Path)
if err != nil {
result.Error = fmt.Errorf("create folder: %w", err)
r.progress.Record(ctx, result)
continue
}
result.Name = folder
result.Resource = resources.FolderResource.Resource
result.Group = resources.FolderResource.Group
r.progress.Record(ctx, result)
continue
}
name, gvk, err := r.resourceManager.WriteResourceFromFile(ctx, change.Path, "")
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
Name: name,
Error: err,
Resource: gvk.Kind,
Group: gvk.Group,
}
r.progress.Record(ctx, result)
}
r.progress.SetMessage(ctx, "changes replicated")
return nil
}
// Convert git changes into resource file changes
func (r *syncJob) applyVersionedChanges(ctx context.Context, repo repository.Versioned, previousRef, currentRef string) error {
diff, err := repo.CompareFiles(ctx, previousRef, currentRef)
if err != nil {
return fmt.Errorf("compare files error: %w", err)
}
if len(diff) < 1 {
r.progress.SetFinalMessage(ctx, "no changes detected between commits")
return nil
}
r.progress.SetTotal(ctx, len(diff))
r.progress.SetMessage(ctx, "replicating versioned changes")
for _, change := range diff {
if ctx.Err() != nil {
return ctx.Err()
}
if err := r.progress.TooManyErrors(); err != nil {
return err
}
if err := resources.IsPathSupported(change.Path); err != nil {
// Maintain the safe segment for empty folders
safeSegment := safepath.SafeSegment(change.Path)
if !safepath.IsDir(safeSegment) {
safeSegment = safepath.Dir(safeSegment)
}
if safeSegment != "" && resources.IsPathSupported(safeSegment) == nil {
folder, err := r.folders.EnsureFolderPathExist(ctx, safeSegment)
if err != nil {
return fmt.Errorf("unable to create empty file folder: %w", err)
}
r.progress.Record(ctx, jobs.JobResourceResult{
Path: safeSegment,
Action: repository.FileActionCreated,
Resource: resources.FolderResource.Resource,
Group: resources.FolderResource.Group,
Name: folder,
})
continue
}
r.progress.Record(ctx, jobs.JobResourceResult{
Path: change.Path,
Action: repository.FileActionIgnored,
})
continue
}
result := jobs.JobResourceResult{
Path: change.Path,
Action: change.Action,
}
switch change.Action {
case repository.FileActionCreated, repository.FileActionUpdated:
name, gvk, err := r.resourceManager.WriteResourceFromFile(ctx, change.Path, change.Ref)
if err != nil {
result.Error = fmt.Errorf("write resource: %w", err)
}
result.Name = name
result.Resource = gvk.Kind
result.Group = gvk.Group
case repository.FileActionDeleted:
name, gvk, err := r.resourceManager.RemoveResourceFromFile(ctx, change.Path, change.PreviousRef)
if err != nil {
result.Error = fmt.Errorf("delete resource: %w", err)
}
result.Name = name
result.Resource = gvk.Kind
result.Group = gvk.Group
case repository.FileActionRenamed:
name, gvk, err := r.resourceManager.RenameResourceFile(ctx, change.Path, change.PreviousRef, change.Path, change.Ref)
if err != nil {
result.Error = fmt.Errorf("rename resource: %w", err)
}
result.Name = name
result.Resource = gvk.Kind
result.Group = gvk.Group
case repository.FileActionIgnored:
// do nothing
}
r.progress.Record(ctx, result)
}
r.progress.SetMessage(ctx, "versioned changes replicated")
return nil
}
@@ -0,0 +1,569 @@
package sync
import (
"context"
"errors"
"testing"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestSyncWorker_IsSupported(t *testing.T) {
tests := []struct {
name string
job provisioning.Job
expected bool
}{
{
name: "pull action is supported",
job: provisioning.Job{
Spec: provisioning.JobSpec{
Action: provisioning.JobActionPull,
},
},
expected: true,
},
{
name: "non-pull action is not supported",
job: provisioning.Job{
Spec: provisioning.JobSpec{
Action: provisioning.JobActionPush,
},
},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
worker := NewSyncWorker(nil, nil, nil, nil, nil)
result := worker.IsSupported(context.Background(), tt.job)
require.Equal(t, tt.expected, result)
})
}
}
func TestSyncWorker_ProcessNotReaderWriter(t *testing.T) {
repo := repository.NewMockReader(t)
repo.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "test-repo",
},
})
fakeDualwrite := dualwrite.NewMockService(t)
fakeDualwrite.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil)
err := worker.Process(context.Background(), repo, provisioning.Job{}, jobs.NewMockJobProgressRecorder(t))
require.EqualError(t, err, "sync job submitted for repository that does not support read-write -- this is a bug")
}
func TestSyncWorker_Process(t *testing.T) {
tests := []struct {
name string
setupMocks func(*resources.MockClientFactory, *resources.MockRepositoryResourcesFactory, *dualwrite.MockService, *MockRepositoryPatchFn, *MockSyncer, *mockReaderWriter, *jobs.MockJobProgressRecorder)
expectedError string
expectedStatus *provisioning.SyncStatus
}{
{
name: "legacy storage not migrated",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
rw.MockRepository.On("Config").Return(&provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "test-repo",
},
})
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(false, nil).Twice()
},
expectedError: "sync not supported until storage has migrated",
},
{
name: "failed initial status patching",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
// Setup repository config with existing LastRef
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "test-repo",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "existing-ref",
},
},
}
rw.MockRepository.On("Config").Return(repoConfig)
pr.On("SetMessage", mock.Anything, "update sync status at start").Return()
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 1 {
return false
}
if patch[0]["op"] != "replace" || patch[0]["path"] != "/status/sync" {
return false
}
if patch[0]["value"].(provisioning.SyncStatus).LastRef != "existing-ref" || patch[0]["value"].(provisioning.SyncStatus).JobID != "test-job" {
return false
}
return true
})).Return(errors.New("failed to patch status"))
},
expectedError: "update repo with job status at start: failed to patch status",
},
{
name: "failed getting repository resources",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
// Setup repository config
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
},
Spec: provisioning.RepositorySpec{
Title: "test-repo",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "existing-ref",
},
},
}
rw.MockRepository.On("Config").Return(repoConfig)
// Storage is migrated
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
// Initial status update succeeds
pr.On("SetMessage", mock.Anything, "update sync status at start").Return()
rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil)
// Repository resources creation fails
rrf.On("Client", mock.Anything, mock.Anything).Return(nil, errors.New("failed to create repository resources client"))
},
expectedError: "create repository resources client: failed to create repository resources client",
},
{
name: "failed getting clients for namespace",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
// Setup repository config
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
Spec: provisioning.RepositorySpec{
Title: "test-repo",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "existing-ref",
},
},
}
rw.MockRepository.On("Config").Return(repoConfig)
// Storage is migrated
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
// Initial status update succeeds
pr.On("SetMessage", mock.Anything, "update sync status at start").Return()
rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil)
// Repository resources creation succeeds
rrf.On("Client", mock.Anything, mock.Anything).Return(&resources.MockRepositoryResources{}, nil)
// Getting clients for namespace fails
cf.On("Clients", mock.Anything, "test-namespace").Return(nil, errors.New("failed to get clients"))
},
expectedError: "get clients for test-repo: failed to get clients",
},
{
name: "successful sync",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "existing-ref",
},
},
}
rw.MockRepository.On("Config").Return(repoConfig)
// Storage is migrated
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
// Initial status update
pr.On("SetMessage", mock.Anything, "update sync status at start").Return()
rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil)
// Setup resources and clients
mockRepoResources := resources.NewMockRepositoryResources(t)
mockRepoResources.On("Stats", mock.Anything).Return(nil, nil)
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, "test-namespace").Return(mockClients, nil)
// Sync execution succeeds
pr.On("SetMessage", mock.Anything, "execute sync job").Return()
s.On("Sync", mock.Anything, rw, mock.MatchedBy(func(opts provisioning.SyncJobOptions) bool {
return true // Add specific sync options validation if needed
}), mockRepoResources, mock.Anything, pr).Return("new-ref", nil)
// Final status updates
pr.On("Complete", mock.Anything, nil).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess})
pr.On("SetMessage", mock.Anything, "update status and stats").Return()
// Final patch should include new ref
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 1 {
return false
}
syncStatus := patch[0]["value"].(provisioning.SyncStatus)
return patch[0]["op"] == "replace" &&
patch[0]["path"] == "/status/sync" &&
syncStatus.LastRef == "new-ref" &&
syncStatus.State == provisioning.JobStateSuccess
})).Return(nil)
},
expectedError: "",
},
{
name: "failed sync",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
Status: provisioning.RepositoryStatus{
Sync: provisioning.SyncStatus{
LastRef: "existing-ref",
},
},
}
rw.MockRepository.On("Config").Return(repoConfig)
// Storage is migrated
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
// Initial status update
pr.On("SetMessage", mock.Anything, "update sync status at start").Return()
rpf.On("Execute", mock.Anything, repoConfig, mock.Anything).Return(nil)
// Setup resources and clients
mockRepoResources := resources.NewMockRepositoryResources(t)
mockRepoResources.On("Stats", mock.Anything).Return(nil, nil)
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, "test-namespace").Return(mockClients, nil)
// Sync execution fails
pr.On("SetMessage", mock.Anything, "execute sync job").Return()
syncError := errors.New("sync operation failed")
s.On("Sync", mock.Anything, rw, mock.MatchedBy(func(opts provisioning.SyncJobOptions) bool {
return true // Add specific sync options validation if needed
}), mockRepoResources, mock.Anything, pr).Return("", syncError)
// Final status updates
pr.On("Complete", mock.Anything, syncError).Return(provisioning.JobStatus{State: provisioning.JobStateError})
pr.On("SetMessage", mock.Anything, "update status and stats").Return()
// Final patch should preserve existing ref on failure
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 1 {
return false
}
syncStatus := patch[0]["value"].(provisioning.SyncStatus)
return patch[0]["op"] == "replace" &&
patch[0]["path"] == "/status/sync" &&
syncStatus.LastRef == "existing-ref" && // LastRef should not change on failure
syncStatus.State == provisioning.JobStateError
})).Return(nil)
},
expectedError: "sync operation failed",
},
{
name: "stats call fails",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
}
rw.MockRepository.On("Config").Return(repoConfig)
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
mockRepoResources := resources.NewMockRepositoryResources(t)
mockRepoResources.On("Stats", mock.Anything).Return(nil, errors.New("stats error"))
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Simple mocks for other calls
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil)
pr.On("SetMessage", mock.Anything, mock.Anything).Return()
pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess})
rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil)
s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil)
},
expectedError: "",
},
{
name: "stats returns nil stats and nil error",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
}
rw.MockRepository.On("Config").Return(repoConfig)
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
mockRepoResources := resources.NewMockRepositoryResources(t)
mockRepoResources.On("Stats", mock.Anything).Return(nil, nil)
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Verify only sync status is patched
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool {
return len(patch) == 1 && patch[0]["path"] == "/status/sync"
})).Return(nil)
// Simple mocks for other calls
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil)
pr.On("SetMessage", mock.Anything, mock.Anything).Return()
pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess})
s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil)
},
expectedError: "",
},
{
name: "stats returns one managed stats",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
}
rw.MockRepository.On("Config").Return(repoConfig)
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
mockRepoResources := resources.NewMockRepositoryResources(t)
stats := &provisioning.ResourceStats{
Managed: []provisioning.ManagerStats{
{
Stats: []provisioning.ResourceCount{
{
Group: "test",
Resource: "test",
Count: 42,
},
},
},
},
}
mockRepoResources.On("Stats", mock.Anything).Return(stats, nil)
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Verify both sync status and stats are patched
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 2 {
return false
}
if patch[0]["path"] != "/status/sync" {
return false
}
if patch[1]["path"] != "/status/stats" {
return false
}
value := patch[1]["value"].([]provisioning.ResourceCount)
if len(value) != 1 {
return false
}
if value[0].Group != "test" || value[0].Resource != "test" || value[0].Count != 42 {
return false
}
return true
})).Return(nil).Once()
// Simple mocks for other calls
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil)
pr.On("SetMessage", mock.Anything, mock.Anything).Return()
pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess})
s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil)
},
expectedError: "",
},
{
name: "stats returns multiple managed stats",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
}
rw.MockRepository.On("Config").Return(repoConfig)
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
mockRepoResources := resources.NewMockRepositoryResources(t)
stats := &provisioning.ResourceStats{
Managed: []provisioning.ManagerStats{
{
Stats: []provisioning.ResourceCount{
{
Group: "test1",
Resource: "test1",
Count: 42,
},
},
},
{
Stats: []provisioning.ResourceCount{
{
Group: "test2",
Resource: "test2",
Count: 24,
},
},
},
},
}
mockRepoResources.On("Stats", mock.Anything).Return(stats, nil)
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Verify only sync status is patched (multiple stats should be ignored)
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool {
return len(patch) == 1 && patch[0]["path"] == "/status/sync"
})).Return(nil)
// Simple mocks for other calls
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil)
pr.On("SetMessage", mock.Anything, mock.Anything).Return()
pr.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess})
s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil)
},
expectedError: "",
},
{
name: "failed final status patch",
setupMocks: func(cf *resources.MockClientFactory, rrf *resources.MockRepositoryResourcesFactory, ds *dualwrite.MockService, rpf *MockRepositoryPatchFn, s *MockSyncer, rw *mockReaderWriter, pr *jobs.MockJobProgressRecorder) {
repoConfig := &provisioning.Repository{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo",
Namespace: "test-namespace",
},
}
rw.MockRepository.On("Config").Return(repoConfig)
ds.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice()
// Initial status patch succeeds
rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
// Setup resources and clients
mockRepoResources := resources.NewMockRepositoryResources(t)
mockRepoResources.On("Stats", mock.Anything).Return(nil, nil)
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
mockClients := resources.NewMockResourceClients(t)
cf.On("Clients", mock.Anything, mock.Anything).Return(mockClients, nil)
// Sync succeeds
pr.On("SetMessage", mock.Anything, mock.Anything).Return()
s.On("Sync", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("new-ref", nil)
pr.On("Complete", mock.Anything, nil).Return(provisioning.JobStatus{State: provisioning.JobStateSuccess})
// Final status patch fails
rpf.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("failed to patch final status")).Once()
},
expectedError: "update repo with job final status: failed to patch final status",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create mocks
clientFactory := resources.NewMockClientFactory(t)
repoResourcesFactory := resources.NewMockRepositoryResourcesFactory(t)
dualwriteService := dualwrite.NewMockService(t)
repositoryPatchFn := NewMockRepositoryPatchFn(t)
syncer := NewMockSyncer(t)
readerWriter := &mockReaderWriter{
MockRepository: repository.NewMockRepository(t),
MockVersioned: repository.NewMockVersioned(t),
}
progressRecorder := jobs.NewMockJobProgressRecorder(t)
// Setup mocks
tt.setupMocks(clientFactory, repoResourcesFactory, dualwriteService, repositoryPatchFn, syncer, readerWriter, progressRecorder)
// Create worker
worker := NewSyncWorker(
clientFactory,
repoResourcesFactory,
dualwriteService,
repositoryPatchFn.Execute,
syncer,
)
// Create test job
job := provisioning.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "test-job",
},
Spec: provisioning.JobSpec{
Action: provisioning.JobActionPull,
Pull: &provisioning.SyncJobOptions{},
},
}
// Execute test
err := worker.Process(context.Background(), readerWriter, job, progressRecorder)
// Verify results
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError)
} else {
require.NoError(t, err)
}
// Verify mock expectations
repositoryPatchFn.AssertExpectations(t)
progressRecorder.AssertExpectations(t)
})
}
}
+9 -5
View File
@@ -125,6 +125,7 @@ func NewAPIBuilder(
clients := resources.NewClientFactory(configProvider)
parsers := resources.NewParserFactory(clients)
resourceLister := resources.NewResourceLister(unified, unified, legacyMigrator, storageStatus)
return &APIBuilder{
urlProvider: urlProvider,
@@ -135,10 +136,10 @@ func NewAPIBuilder(
ghFactory: ghFactory,
clients: clients,
parsers: parsers,
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients),
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister),
render: render,
clonedir: clonedir,
resourceLister: resources.NewResourceLister(unified, unified, legacyMigrator, storageStatus),
resourceLister: resourceLister,
legacyMigrator: legacyMigrator,
storageStatus: storageStatus,
unified: unified,
@@ -536,13 +537,16 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
repository.WrapWithCloneAndPushIfPossible,
)
statusPatcher := controller.NewRepositoryStatusPatcher(b.GetClient())
syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync)
syncWorker := sync.NewSyncWorker(
b.GetClient(),
b.parsers,
b.clients,
b.resourceLister,
b.repositoryResources,
b.storageStatus,
statusPatcher.Patch,
syncer,
)
migrationWorker := migrate.NewMigrationWorker(
b.legacyMigrator,
b.parsers,
@@ -184,6 +184,8 @@ type VersionedFileChange struct {
// Versioned is a repository that supports versioning.
// This interface may be extended to make the the original Repository interface more agnostic to the underlying storage system.
//
//go:generate mockery --name Versioned --structname MockVersioned --inpackage --filename versioned_mock.go --with-expecter
type Versioned interface {
// History of changes for a path
History(ctx context.Context, path, ref string) ([]provisioning.HistoryItem, error)
@@ -0,0 +1,213 @@
// Code generated by mockery v2.52.4. DO NOT EDIT.
package repository
import (
context "context"
v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
mock "github.com/stretchr/testify/mock"
)
// MockVersioned is an autogenerated mock type for the Versioned type
type MockVersioned struct {
mock.Mock
}
type MockVersioned_Expecter struct {
mock *mock.Mock
}
func (_m *MockVersioned) EXPECT() *MockVersioned_Expecter {
return &MockVersioned_Expecter{mock: &_m.Mock}
}
// CompareFiles provides a mock function with given fields: ctx, base, ref
func (_m *MockVersioned) CompareFiles(ctx context.Context, base string, ref string) ([]VersionedFileChange, error) {
ret := _m.Called(ctx, base, ref)
if len(ret) == 0 {
panic("no return value specified for CompareFiles")
}
var r0 []VersionedFileChange
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]VersionedFileChange, error)); ok {
return rf(ctx, base, ref)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string) []VersionedFileChange); ok {
r0 = rf(ctx, base, ref)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]VersionedFileChange)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, base, ref)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockVersioned_CompareFiles_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CompareFiles'
type MockVersioned_CompareFiles_Call struct {
*mock.Call
}
// CompareFiles is a helper method to define mock.On call
// - ctx context.Context
// - base string
// - ref string
func (_e *MockVersioned_Expecter) CompareFiles(ctx interface{}, base interface{}, ref interface{}) *MockVersioned_CompareFiles_Call {
return &MockVersioned_CompareFiles_Call{Call: _e.mock.On("CompareFiles", ctx, base, ref)}
}
func (_c *MockVersioned_CompareFiles_Call) Run(run func(ctx context.Context, base string, ref string)) *MockVersioned_CompareFiles_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(string))
})
return _c
}
func (_c *MockVersioned_CompareFiles_Call) Return(_a0 []VersionedFileChange, _a1 error) *MockVersioned_CompareFiles_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockVersioned_CompareFiles_Call) RunAndReturn(run func(context.Context, string, string) ([]VersionedFileChange, error)) *MockVersioned_CompareFiles_Call {
_c.Call.Return(run)
return _c
}
// History provides a mock function with given fields: ctx, path, ref
func (_m *MockVersioned) History(ctx context.Context, path string, ref string) ([]v0alpha1.HistoryItem, error) {
ret := _m.Called(ctx, path, ref)
if len(ret) == 0 {
panic("no return value specified for History")
}
var r0 []v0alpha1.HistoryItem
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]v0alpha1.HistoryItem, error)); ok {
return rf(ctx, path, ref)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string) []v0alpha1.HistoryItem); ok {
r0 = rf(ctx, path, ref)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]v0alpha1.HistoryItem)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, path, ref)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockVersioned_History_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'History'
type MockVersioned_History_Call struct {
*mock.Call
}
// History is a helper method to define mock.On call
// - ctx context.Context
// - path string
// - ref string
func (_e *MockVersioned_Expecter) History(ctx interface{}, path interface{}, ref interface{}) *MockVersioned_History_Call {
return &MockVersioned_History_Call{Call: _e.mock.On("History", ctx, path, ref)}
}
func (_c *MockVersioned_History_Call) Run(run func(ctx context.Context, path string, ref string)) *MockVersioned_History_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(string))
})
return _c
}
func (_c *MockVersioned_History_Call) Return(_a0 []v0alpha1.HistoryItem, _a1 error) *MockVersioned_History_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockVersioned_History_Call) RunAndReturn(run func(context.Context, string, string) ([]v0alpha1.HistoryItem, error)) *MockVersioned_History_Call {
_c.Call.Return(run)
return _c
}
// LatestRef provides a mock function with given fields: ctx
func (_m *MockVersioned) LatestRef(ctx context.Context) (string, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for LatestRef")
}
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) string); ok {
r0 = rf(ctx)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockVersioned_LatestRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LatestRef'
type MockVersioned_LatestRef_Call struct {
*mock.Call
}
// LatestRef is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockVersioned_Expecter) LatestRef(ctx interface{}) *MockVersioned_LatestRef_Call {
return &MockVersioned_LatestRef_Call{Call: _e.mock.On("LatestRef", ctx)}
}
func (_c *MockVersioned_LatestRef_Call) Run(run func(ctx context.Context)) *MockVersioned_LatestRef_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockVersioned_LatestRef_Call) Return(_a0 string, _a1 error) *MockVersioned_LatestRef_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockVersioned_LatestRef_Call) RunAndReturn(run func(context.Context) (string, error)) *MockVersioned_LatestRef_Call {
_c.Call.Return(run)
return _c
}
// NewMockVersioned creates a new instance of MockVersioned. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockVersioned(t interface {
mock.TestingT
Cleanup(func())
}) *MockVersioned {
mock := &MockVersioned{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
@@ -4,8 +4,10 @@ import (
"context"
"fmt"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
)
//go:generate mockery --name RepositoryResourcesFactory --structname MockRepositoryResourcesFactory --inpackage --filename repository_resources_factory_mock.go --with-expecter
@@ -15,21 +17,45 @@ type RepositoryResourcesFactory interface {
//go:generate mockery --name RepositoryResources --structname MockRepositoryResources --inpackage --filename repository_resources_mock.go --with-expecter
type RepositoryResources interface {
// Folders
SetTree(tree FolderTree)
EnsureFolderPathExist(ctx context.Context, filePath string) (parent string, err error)
EnsureFolderExists(ctx context.Context, folder Folder, parentID string) error
EnsureFolderTreeExists(ctx context.Context, ref, path string, tree FolderTree, fn func(folder Folder, created bool, err error) error) error
// File from Resource
CreateResourceFileFromObject(ctx context.Context, obj *unstructured.Unstructured, options WriteOptions) (string, error)
// Resource from file
WriteResourceFromFile(ctx context.Context, path, ref string) (string, schema.GroupVersionKind, error)
RemoveResourceFromFile(ctx context.Context, path, ref string) (string, schema.GroupVersionKind, error)
RenameResourceFile(ctx context.Context, path, previousRef, newPath, newRef string) (string, schema.GroupVersionKind, error)
// Stats
Stats(ctx context.Context) (*provisioning.ResourceStats, error)
List(ctx context.Context) (*provisioning.ResourceList, error)
}
type repositoryResourcesFactor struct {
parsers ParserFactory
clients ClientFactory
lister ResourceLister
}
type repositoryResources struct {
*FolderManager
*ResourcesManager
lister ResourceLister
namespace string
repoName string
}
func NewRepositoryResourcesFactory(parsers ParserFactory, clients ClientFactory) RepositoryResourcesFactory {
return &repositoryResourcesFactor{parsers, clients}
func (r *repositoryResources) Stats(ctx context.Context) (*provisioning.ResourceStats, error) {
return r.lister.Stats(ctx, r.namespace, r.repoName)
}
func (r *repositoryResources) List(ctx context.Context) (*provisioning.ResourceList, error) {
return r.lister.List(ctx, r.namespace, r.repoName)
}
func NewRepositoryResourcesFactory(parsers ParserFactory, clients ClientFactory, lister ResourceLister) RepositoryResourcesFactory {
return &repositoryResourcesFactor{parsers, clients, lister}
}
func (r *repositoryResourcesFactor) Client(ctx context.Context, repo repository.ReaderWriter) (RepositoryResources, error) {
@@ -50,5 +76,11 @@ func (r *repositoryResourcesFactor) Client(ctx context.Context, repo repository.
folders := NewFolderManager(repo, folderClient, NewEmptyFolderTree())
resources := NewResourcesManager(repo, folders, parser, clients, map[string]repository.CommitSignature{})
return &repositoryResources{folders, resources}, nil
return &repositoryResources{
FolderManager: folders,
ResourcesManager: resources,
lister: r.lister,
namespace: repo.Config().Namespace,
repoName: repo.Config().Name,
}, nil
}
@@ -6,7 +6,11 @@ import (
context "context"
mock "github.com/stretchr/testify/mock"
schema "k8s.io/apimachinery/pkg/runtime/schema"
unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
v0alpha1 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
)
// MockRepositoryResources is an autogenerated mock type for the RepositoryResources type
@@ -80,6 +84,111 @@ func (_c *MockRepositoryResources_CreateResourceFileFromObject_Call) RunAndRetur
return _c
}
// EnsureFolderExists provides a mock function with given fields: ctx, folder, parentID
func (_m *MockRepositoryResources) EnsureFolderExists(ctx context.Context, folder Folder, parentID string) error {
ret := _m.Called(ctx, folder, parentID)
if len(ret) == 0 {
panic("no return value specified for EnsureFolderExists")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, Folder, string) error); ok {
r0 = rf(ctx, folder, parentID)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockRepositoryResources_EnsureFolderExists_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnsureFolderExists'
type MockRepositoryResources_EnsureFolderExists_Call struct {
*mock.Call
}
// EnsureFolderExists is a helper method to define mock.On call
// - ctx context.Context
// - folder Folder
// - parentID string
func (_e *MockRepositoryResources_Expecter) EnsureFolderExists(ctx interface{}, folder interface{}, parentID interface{}) *MockRepositoryResources_EnsureFolderExists_Call {
return &MockRepositoryResources_EnsureFolderExists_Call{Call: _e.mock.On("EnsureFolderExists", ctx, folder, parentID)}
}
func (_c *MockRepositoryResources_EnsureFolderExists_Call) Run(run func(ctx context.Context, folder Folder, parentID string)) *MockRepositoryResources_EnsureFolderExists_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(Folder), args[2].(string))
})
return _c
}
func (_c *MockRepositoryResources_EnsureFolderExists_Call) Return(_a0 error) *MockRepositoryResources_EnsureFolderExists_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockRepositoryResources_EnsureFolderExists_Call) RunAndReturn(run func(context.Context, Folder, string) error) *MockRepositoryResources_EnsureFolderExists_Call {
_c.Call.Return(run)
return _c
}
// EnsureFolderPathExist provides a mock function with given fields: ctx, filePath
func (_m *MockRepositoryResources) EnsureFolderPathExist(ctx context.Context, filePath string) (string, error) {
ret := _m.Called(ctx, filePath)
if len(ret) == 0 {
panic("no return value specified for EnsureFolderPathExist")
}
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok {
return rf(ctx, filePath)
}
if rf, ok := ret.Get(0).(func(context.Context, string) string); ok {
r0 = rf(ctx, filePath)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, filePath)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepositoryResources_EnsureFolderPathExist_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnsureFolderPathExist'
type MockRepositoryResources_EnsureFolderPathExist_Call struct {
*mock.Call
}
// EnsureFolderPathExist is a helper method to define mock.On call
// - ctx context.Context
// - filePath string
func (_e *MockRepositoryResources_Expecter) EnsureFolderPathExist(ctx interface{}, filePath interface{}) *MockRepositoryResources_EnsureFolderPathExist_Call {
return &MockRepositoryResources_EnsureFolderPathExist_Call{Call: _e.mock.On("EnsureFolderPathExist", ctx, filePath)}
}
func (_c *MockRepositoryResources_EnsureFolderPathExist_Call) Run(run func(ctx context.Context, filePath string)) *MockRepositoryResources_EnsureFolderPathExist_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string))
})
return _c
}
func (_c *MockRepositoryResources_EnsureFolderPathExist_Call) Return(parent string, err error) *MockRepositoryResources_EnsureFolderPathExist_Call {
_c.Call.Return(parent, err)
return _c
}
func (_c *MockRepositoryResources_EnsureFolderPathExist_Call) RunAndReturn(run func(context.Context, string) (string, error)) *MockRepositoryResources_EnsureFolderPathExist_Call {
_c.Call.Return(run)
return _c
}
// EnsureFolderTreeExists provides a mock function with given fields: ctx, ref, path, tree, fn
func (_m *MockRepositoryResources) EnsureFolderTreeExists(ctx context.Context, ref string, path string, tree FolderTree, fn func(Folder, bool, error) error) error {
ret := _m.Called(ctx, ref, path, tree, fn)
@@ -130,6 +239,352 @@ func (_c *MockRepositoryResources_EnsureFolderTreeExists_Call) RunAndReturn(run
return _c
}
// List provides a mock function with given fields: ctx
func (_m *MockRepositoryResources) List(ctx context.Context) (*v0alpha1.ResourceList, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for List")
}
var r0 *v0alpha1.ResourceList
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (*v0alpha1.ResourceList, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) *v0alpha1.ResourceList); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v0alpha1.ResourceList)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepositoryResources_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List'
type MockRepositoryResources_List_Call struct {
*mock.Call
}
// List is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockRepositoryResources_Expecter) List(ctx interface{}) *MockRepositoryResources_List_Call {
return &MockRepositoryResources_List_Call{Call: _e.mock.On("List", ctx)}
}
func (_c *MockRepositoryResources_List_Call) Run(run func(ctx context.Context)) *MockRepositoryResources_List_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockRepositoryResources_List_Call) Return(_a0 *v0alpha1.ResourceList, _a1 error) *MockRepositoryResources_List_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockRepositoryResources_List_Call) RunAndReturn(run func(context.Context) (*v0alpha1.ResourceList, error)) *MockRepositoryResources_List_Call {
_c.Call.Return(run)
return _c
}
// RemoveResourceFromFile provides a mock function with given fields: ctx, path, ref
func (_m *MockRepositoryResources) RemoveResourceFromFile(ctx context.Context, path string, ref string) (string, schema.GroupVersionKind, error) {
ret := _m.Called(ctx, path, ref)
if len(ret) == 0 {
panic("no return value specified for RemoveResourceFromFile")
}
var r0 string
var r1 schema.GroupVersionKind
var r2 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) (string, schema.GroupVersionKind, error)); ok {
return rf(ctx, path, ref)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string) string); ok {
r0 = rf(ctx, path, ref)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string) schema.GroupVersionKind); ok {
r1 = rf(ctx, path, ref)
} else {
r1 = ret.Get(1).(schema.GroupVersionKind)
}
if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok {
r2 = rf(ctx, path, ref)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// MockRepositoryResources_RemoveResourceFromFile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveResourceFromFile'
type MockRepositoryResources_RemoveResourceFromFile_Call struct {
*mock.Call
}
// RemoveResourceFromFile is a helper method to define mock.On call
// - ctx context.Context
// - path string
// - ref string
func (_e *MockRepositoryResources_Expecter) RemoveResourceFromFile(ctx interface{}, path interface{}, ref interface{}) *MockRepositoryResources_RemoveResourceFromFile_Call {
return &MockRepositoryResources_RemoveResourceFromFile_Call{Call: _e.mock.On("RemoveResourceFromFile", ctx, path, ref)}
}
func (_c *MockRepositoryResources_RemoveResourceFromFile_Call) Run(run func(ctx context.Context, path string, ref string)) *MockRepositoryResources_RemoveResourceFromFile_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(string))
})
return _c
}
func (_c *MockRepositoryResources_RemoveResourceFromFile_Call) Return(_a0 string, _a1 schema.GroupVersionKind, _a2 error) *MockRepositoryResources_RemoveResourceFromFile_Call {
_c.Call.Return(_a0, _a1, _a2)
return _c
}
func (_c *MockRepositoryResources_RemoveResourceFromFile_Call) RunAndReturn(run func(context.Context, string, string) (string, schema.GroupVersionKind, error)) *MockRepositoryResources_RemoveResourceFromFile_Call {
_c.Call.Return(run)
return _c
}
// RenameResourceFile provides a mock function with given fields: ctx, path, previousRef, newPath, newRef
func (_m *MockRepositoryResources) RenameResourceFile(ctx context.Context, path string, previousRef string, newPath string, newRef string) (string, schema.GroupVersionKind, error) {
ret := _m.Called(ctx, path, previousRef, newPath, newRef)
if len(ret) == 0 {
panic("no return value specified for RenameResourceFile")
}
var r0 string
var r1 schema.GroupVersionKind
var r2 error
if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) (string, schema.GroupVersionKind, error)); ok {
return rf(ctx, path, previousRef, newPath, newRef)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) string); ok {
r0 = rf(ctx, path, previousRef, newPath, newRef)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string) schema.GroupVersionKind); ok {
r1 = rf(ctx, path, previousRef, newPath, newRef)
} else {
r1 = ret.Get(1).(schema.GroupVersionKind)
}
if rf, ok := ret.Get(2).(func(context.Context, string, string, string, string) error); ok {
r2 = rf(ctx, path, previousRef, newPath, newRef)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// MockRepositoryResources_RenameResourceFile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RenameResourceFile'
type MockRepositoryResources_RenameResourceFile_Call struct {
*mock.Call
}
// RenameResourceFile is a helper method to define mock.On call
// - ctx context.Context
// - path string
// - previousRef string
// - newPath string
// - newRef string
func (_e *MockRepositoryResources_Expecter) RenameResourceFile(ctx interface{}, path interface{}, previousRef interface{}, newPath interface{}, newRef interface{}) *MockRepositoryResources_RenameResourceFile_Call {
return &MockRepositoryResources_RenameResourceFile_Call{Call: _e.mock.On("RenameResourceFile", ctx, path, previousRef, newPath, newRef)}
}
func (_c *MockRepositoryResources_RenameResourceFile_Call) Run(run func(ctx context.Context, path string, previousRef string, newPath string, newRef string)) *MockRepositoryResources_RenameResourceFile_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(string))
})
return _c
}
func (_c *MockRepositoryResources_RenameResourceFile_Call) Return(_a0 string, _a1 schema.GroupVersionKind, _a2 error) *MockRepositoryResources_RenameResourceFile_Call {
_c.Call.Return(_a0, _a1, _a2)
return _c
}
func (_c *MockRepositoryResources_RenameResourceFile_Call) RunAndReturn(run func(context.Context, string, string, string, string) (string, schema.GroupVersionKind, error)) *MockRepositoryResources_RenameResourceFile_Call {
_c.Call.Return(run)
return _c
}
// SetTree provides a mock function with given fields: tree
func (_m *MockRepositoryResources) SetTree(tree FolderTree) {
_m.Called(tree)
}
// MockRepositoryResources_SetTree_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetTree'
type MockRepositoryResources_SetTree_Call struct {
*mock.Call
}
// SetTree is a helper method to define mock.On call
// - tree FolderTree
func (_e *MockRepositoryResources_Expecter) SetTree(tree interface{}) *MockRepositoryResources_SetTree_Call {
return &MockRepositoryResources_SetTree_Call{Call: _e.mock.On("SetTree", tree)}
}
func (_c *MockRepositoryResources_SetTree_Call) Run(run func(tree FolderTree)) *MockRepositoryResources_SetTree_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(FolderTree))
})
return _c
}
func (_c *MockRepositoryResources_SetTree_Call) Return() *MockRepositoryResources_SetTree_Call {
_c.Call.Return()
return _c
}
func (_c *MockRepositoryResources_SetTree_Call) RunAndReturn(run func(FolderTree)) *MockRepositoryResources_SetTree_Call {
_c.Run(run)
return _c
}
// Stats provides a mock function with given fields: ctx
func (_m *MockRepositoryResources) Stats(ctx context.Context) (*v0alpha1.ResourceStats, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for Stats")
}
var r0 *v0alpha1.ResourceStats
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (*v0alpha1.ResourceStats, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) *v0alpha1.ResourceStats); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v0alpha1.ResourceStats)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRepositoryResources_Stats_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Stats'
type MockRepositoryResources_Stats_Call struct {
*mock.Call
}
// Stats is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockRepositoryResources_Expecter) Stats(ctx interface{}) *MockRepositoryResources_Stats_Call {
return &MockRepositoryResources_Stats_Call{Call: _e.mock.On("Stats", ctx)}
}
func (_c *MockRepositoryResources_Stats_Call) Run(run func(ctx context.Context)) *MockRepositoryResources_Stats_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockRepositoryResources_Stats_Call) Return(_a0 *v0alpha1.ResourceStats, _a1 error) *MockRepositoryResources_Stats_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockRepositoryResources_Stats_Call) RunAndReturn(run func(context.Context) (*v0alpha1.ResourceStats, error)) *MockRepositoryResources_Stats_Call {
_c.Call.Return(run)
return _c
}
// WriteResourceFromFile provides a mock function with given fields: ctx, path, ref
func (_m *MockRepositoryResources) WriteResourceFromFile(ctx context.Context, path string, ref string) (string, schema.GroupVersionKind, error) {
ret := _m.Called(ctx, path, ref)
if len(ret) == 0 {
panic("no return value specified for WriteResourceFromFile")
}
var r0 string
var r1 schema.GroupVersionKind
var r2 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) (string, schema.GroupVersionKind, error)); ok {
return rf(ctx, path, ref)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string) string); ok {
r0 = rf(ctx, path, ref)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string) schema.GroupVersionKind); ok {
r1 = rf(ctx, path, ref)
} else {
r1 = ret.Get(1).(schema.GroupVersionKind)
}
if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok {
r2 = rf(ctx, path, ref)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// MockRepositoryResources_WriteResourceFromFile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WriteResourceFromFile'
type MockRepositoryResources_WriteResourceFromFile_Call struct {
*mock.Call
}
// WriteResourceFromFile is a helper method to define mock.On call
// - ctx context.Context
// - path string
// - ref string
func (_e *MockRepositoryResources_Expecter) WriteResourceFromFile(ctx interface{}, path interface{}, ref interface{}) *MockRepositoryResources_WriteResourceFromFile_Call {
return &MockRepositoryResources_WriteResourceFromFile_Call{Call: _e.mock.On("WriteResourceFromFile", ctx, path, ref)}
}
func (_c *MockRepositoryResources_WriteResourceFromFile_Call) Run(run func(ctx context.Context, path string, ref string)) *MockRepositoryResources_WriteResourceFromFile_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(string))
})
return _c
}
func (_c *MockRepositoryResources_WriteResourceFromFile_Call) Return(_a0 string, _a1 schema.GroupVersionKind, _a2 error) *MockRepositoryResources_WriteResourceFromFile_Call {
_c.Call.Return(_a0, _a1, _a2)
return _c
}
func (_c *MockRepositoryResources_WriteResourceFromFile_Call) RunAndReturn(run func(context.Context, string, string) (string, schema.GroupVersionKind, error)) *MockRepositoryResources_WriteResourceFromFile_Call {
_c.Call.Return(run)
return _c
}
// NewMockRepositoryResources creates a new instance of MockRepositoryResources. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockRepositoryResources(t interface {
@@ -0,0 +1,76 @@
package dualwrite
import (
"context"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime/schema"
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1alpha1"
folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
)
func TestIsReadingLegacyDashboardsAndFolders(t *testing.T) {
tests := []struct {
name string
setupMockSvc func(*MockService)
expectedResult bool
}{
{
name: "both folders and dashboards are read from unified storage",
setupMockSvc: func(svc *MockService) {
svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(true, nil)
svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{
Group: dashboard.GROUP,
Resource: dashboard.DASHBOARD_RESOURCE,
}).Return(true, nil)
},
expectedResult: false,
},
{
name: "only folders are read from unified storage",
setupMockSvc: func(svc *MockService) {
svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(true, nil)
svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{
Group: dashboard.GROUP,
Resource: dashboard.DASHBOARD_RESOURCE,
}).Return(false, nil)
},
expectedResult: true,
},
{
name: "only dashboards are read from unified storage",
setupMockSvc: func(svc *MockService) {
svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(false, nil)
svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{
Group: dashboard.GROUP,
Resource: dashboard.DASHBOARD_RESOURCE,
}).Return(true, nil)
},
expectedResult: true,
},
{
name: "neither folders nor dashboards are read from unified storage",
setupMockSvc: func(svc *MockService) {
svc.On("ReadFromUnified", mock.Anything, folders.FolderResourceInfo.GroupResource()).Return(false, nil)
svc.On("ReadFromUnified", mock.Anything, schema.GroupResource{
Group: dashboard.GROUP,
Resource: dashboard.DASHBOARD_RESOURCE,
}).Return(false, nil)
},
expectedResult: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockSvc := NewMockService(t)
tt.setupMockSvc(mockSvc)
result := IsReadingLegacyDashboardsAndFolders(context.Background(), mockSvc)
require.Equal(t, tt.expectedResult, result)
})
}
}