Provisioning finalisers fix 2 (#111679)
* adding some logs to better understand what might be happening * only focus this PR on improve logging in finalizer handling * debug log before calling finalizers * working on finalizers * removing last todos, adding unit tests * better use SupportedFinalizers name * addressing comments * wip: fix tests and add delete error in status * chore: codegen * chore: codegen openapi * Merge remote-tracking branch 'origin/main' into provisioning-finalisers-fix-2 * update frontend client * fix: errors in testing * fix: breaking test --------- Co-authored-by: Daniele Ferru <daniele.ferru@grafana.com> Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Costa Alexoglou
co-authored by
Daniele Ferru
Ryan McKinley
Daniele Ferru
Ryan McKinley
parent
893523dd7c
commit
1b766b9c9f
@@ -0,0 +1,85 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
repository "github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockFinalizerProcessor is an autogenerated mock type for the finalizerProcessor type
|
||||
type MockFinalizerProcessor struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockFinalizerProcessor_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockFinalizerProcessor) EXPECT() *MockFinalizerProcessor_Expecter {
|
||||
return &MockFinalizerProcessor_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// process provides a mock function with given fields: ctx, repo, finalizers
|
||||
func (_m *MockFinalizerProcessor) process(ctx context.Context, repo repository.Repository, finalizers []string) error {
|
||||
ret := _m.Called(ctx, repo, finalizers)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for process")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, repository.Repository, []string) error); ok {
|
||||
r0 = rf(ctx, repo, finalizers)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockFinalizerProcessor_process_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'process'
|
||||
type MockFinalizerProcessor_process_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// process is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - repo repository.Repository
|
||||
// - finalizers []string
|
||||
func (_e *MockFinalizerProcessor_Expecter) process(ctx interface{}, repo interface{}, finalizers interface{}) *MockFinalizerProcessor_process_Call {
|
||||
return &MockFinalizerProcessor_process_Call{Call: _e.mock.On("process", ctx, repo, finalizers)}
|
||||
}
|
||||
|
||||
func (_c *MockFinalizerProcessor_process_Call) Run(run func(ctx context.Context, repo repository.Repository, finalizers []string)) *MockFinalizerProcessor_process_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(repository.Repository), args[2].([]string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockFinalizerProcessor_process_Call) Return(_a0 error) *MockFinalizerProcessor_process_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockFinalizerProcessor_process_Call) RunAndReturn(run func(context.Context, repository.Repository, []string) error) *MockFinalizerProcessor_process_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockFinalizerProcessor creates a new instance of MockFinalizerProcessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockFinalizerProcessor(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockFinalizerProcessor {
|
||||
mock := &MockFinalizerProcessor{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package controller
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -32,8 +34,18 @@ func (f *finalizer) process(ctx context.Context,
|
||||
finalizers []string,
|
||||
) error {
|
||||
logger := logging.FromContext(ctx)
|
||||
logger.Info("process finalizers", "finalizers", finalizers)
|
||||
|
||||
for _, finalizer := range finalizers {
|
||||
orderedFinalizers := [3]string{
|
||||
repository.CleanFinalizer,
|
||||
repository.ReleaseOrphanResourcesFinalizer,
|
||||
repository.RemoveOrphanResourcesFinalizer}
|
||||
|
||||
for _, finalizer := range orderedFinalizers {
|
||||
if !slices.Contains(finalizers, finalizer) {
|
||||
continue
|
||||
}
|
||||
logger.Info("running finalizer", "finalizer", finalizer)
|
||||
var err error
|
||||
var count int
|
||||
start := time.Now()
|
||||
@@ -42,44 +54,33 @@ func (f *finalizer) process(ctx context.Context,
|
||||
switch finalizer {
|
||||
case repository.CleanFinalizer:
|
||||
// NOTE: the controller loop will never get run unless a finalizer is set
|
||||
logger.Info("running cleanup finalizer")
|
||||
hooks, ok := repo.(repository.Hooks)
|
||||
if ok {
|
||||
if err = hooks.OnDelete(ctx); err != nil {
|
||||
logger.Warn("Error running deletion hooks", "err", err)
|
||||
err = fmt.Errorf("execute deletion hooks: %w", err)
|
||||
outcome = metricutils.ErrorOutcome
|
||||
}
|
||||
}
|
||||
|
||||
case repository.ReleaseOrphanResourcesFinalizer:
|
||||
count, err = f.processExistingItems(ctx, repo.Config(),
|
||||
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
patchAnnotations, err := getPatchedAnnotations(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.Patch(
|
||||
ctx, item.Name, types.JSONPatchType, patchAnnotations, v1.PatchOptions{},
|
||||
)
|
||||
return err
|
||||
})
|
||||
logger.Info("releasing orphan resources")
|
||||
count, err = f.processExistingItems(ctx, repo.Config(), f.releaseResources(ctx, logger))
|
||||
if err != nil {
|
||||
err = fmt.Errorf("release resources: %w", err)
|
||||
outcome = metricutils.ErrorOutcome
|
||||
logger.Warn("Error processing release orphan resources finalizer", "err", err)
|
||||
}
|
||||
|
||||
case repository.RemoveOrphanResourcesFinalizer:
|
||||
count, err = f.processExistingItems(ctx, repo.Config(),
|
||||
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
return client.Delete(ctx, item.Name, v1.DeleteOptions{})
|
||||
})
|
||||
logger.Info("removing orphan resources")
|
||||
count, err = f.processExistingItems(ctx, repo.Config(), f.removeResources(ctx, logger))
|
||||
if err != nil {
|
||||
err = fmt.Errorf("remove resources: %w", err)
|
||||
outcome = metricutils.ErrorOutcome
|
||||
logger.Warn("Error processing remove orphan resources finalizer", "err", err)
|
||||
}
|
||||
|
||||
default:
|
||||
logger.Warn("skipping unknown finalizer", "finalizer", finalizer)
|
||||
logger.Error("skipping unknown finalizer", "finalizer", finalizer)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -106,36 +107,74 @@ func (f *finalizer) processExistingItems(
|
||||
|
||||
items, err := f.lister.List(ctx, repo.Namespace, repo.Name)
|
||||
if err != nil {
|
||||
logger.Warn("error listing resources", "error", err)
|
||||
logger.Error("error listing resources", "error", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Safe deletion order
|
||||
sortResourceListForDeletion(items)
|
||||
count := 0
|
||||
errors := 0
|
||||
|
||||
for _, item := range items.Items {
|
||||
res, _, err := clients.ForResource(ctx, schema.GroupVersionResource{
|
||||
Group: item.Group,
|
||||
Resource: item.Resource,
|
||||
})
|
||||
logger.Error("error getting client for resource", "resource", item.Resource, "error", err)
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
|
||||
err = cb(res, &item)
|
||||
if err != nil {
|
||||
logger.Warn("error processing item", "name", item.Name, "error", err)
|
||||
errors++
|
||||
logger.Error("error processing item", "name", item.Name, "error", err)
|
||||
return count, fmt.Errorf("processing item: %w", err)
|
||||
} else {
|
||||
count++
|
||||
}
|
||||
}
|
||||
logger.Info("processed orphan items", "items", count, "errors", errors)
|
||||
logger.Info("processed orphan items", "items", count)
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (f *finalizer) releaseResources(
|
||||
ctx context.Context, logger logging.Logger,
|
||||
) func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
return func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
logger.Info("release resource",
|
||||
"name", item.Name,
|
||||
"group", item.Group,
|
||||
"resource", item.Resource,
|
||||
)
|
||||
|
||||
patchAnnotations, err := getPatchedAnnotations(item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get patched annotations: %w", err)
|
||||
}
|
||||
|
||||
_, err = client.Patch(
|
||||
ctx, item.Name, types.JSONPatchType, patchAnnotations, v1.PatchOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("patch resource to release ownership: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *finalizer) removeResources(
|
||||
ctx context.Context, logger logging.Logger,
|
||||
) func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
return func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
logger.Info("remove resource",
|
||||
"name", item.Name,
|
||||
"group", item.Group,
|
||||
"resource", item.Resource,
|
||||
)
|
||||
return client.Delete(ctx, item.Name, v1.DeleteOptions{})
|
||||
}
|
||||
}
|
||||
|
||||
type jsonPatchOperation struct {
|
||||
Op string `json:"op"`
|
||||
Path string `json:"path"`
|
||||
|
||||
@@ -1,12 +1,508 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/dynamic"
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
)
|
||||
|
||||
var (
|
||||
_ dynamic.ResourceInterface = (*mockDynamicClient)(nil)
|
||||
_ repository.Repository = (*mockRepo)(nil)
|
||||
_ repository.Hooks = (*mockRepo)(nil)
|
||||
)
|
||||
|
||||
type mockDynamicClient struct {
|
||||
deleteFunc func(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error
|
||||
patchFunc func(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error)
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Create(ctx context.Context, obj *unstructured.Unstructured, options metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Update(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions) (*unstructured.Unstructured, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Delete(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error {
|
||||
if m.deleteFunc != nil {
|
||||
return m.deleteFunc(ctx, name, options, subresources...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) DeleteCollection(ctx context.Context, options metav1.DeleteOptions, listOptions metav1.ListOptions) error {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Get(ctx context.Context, name string, options metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) List(ctx context.Context, opts metav1.ListOptions) (*unstructured.UnstructuredList, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
if m.patchFunc != nil {
|
||||
return m.patchFunc(ctx, name, pt, data, options, subresources...)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) Apply(ctx context.Context, name string, obj *unstructured.Unstructured, options metav1.ApplyOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockDynamicClient) ApplyStatus(ctx context.Context, name string, obj *unstructured.Unstructured, options metav1.ApplyOptions) (*unstructured.Unstructured, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
type mockRepo struct {
|
||||
name string
|
||||
namespace string
|
||||
onDeleteFunc func(ctx context.Context) error
|
||||
}
|
||||
|
||||
func (m mockRepo) OnCreate(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepo) OnUpdate(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepo) OnDelete(ctx context.Context) error {
|
||||
if m.onDeleteFunc != nil {
|
||||
return m.onDeleteFunc(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m mockRepo) Config() *provisioning.Repository {
|
||||
return &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: m.name,
|
||||
Namespace: m.namespace,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (m mockRepo) Validate() field.ErrorList {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepo) Test(ctx context.Context) (*provisioning.TestResults, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func TestFinalizer_process(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
lister resources.ResourceLister
|
||||
clientFactory resources.ClientFactory
|
||||
repo repository.Repository
|
||||
finalizers []string
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "No finalizers",
|
||||
lister: nil,
|
||||
clientFactory: nil,
|
||||
repo: nil,
|
||||
finalizers: []string{},
|
||||
},
|
||||
{
|
||||
name: "Successfully releases resources and cleanup hooks",
|
||||
lister: func() resources.ResourceLister {
|
||||
resourceLister := resources.NewMockResourceLister(t)
|
||||
|
||||
resourceLister.
|
||||
On("List", context.Background(), "default", "my-repo").
|
||||
Once().
|
||||
Return(&provisioning.ResourceList{
|
||||
Items: []provisioning.ResourceListItem{
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Name: "my-dashboard",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
return resourceLister
|
||||
}(),
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
client := &mockDynamicClient{
|
||||
patchFunc: func(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
return &unstructured.Unstructured{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(clients, nil)
|
||||
|
||||
clients.
|
||||
On("ForResource", context.Background(), schema.GroupVersionResource{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
}).
|
||||
Once().
|
||||
Return(client, schema.GroupVersionKind{}, nil)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
onDeleteFunc: func(ctx context.Context) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.ReleaseOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Successfully removes resources and cleanup hooks",
|
||||
lister: func() resources.ResourceLister {
|
||||
resourceLister := resources.NewMockResourceLister(t)
|
||||
|
||||
resourceLister.
|
||||
On("List", context.Background(), "default", "my-repo").
|
||||
Once().
|
||||
Return(&provisioning.ResourceList{
|
||||
Items: []provisioning.ResourceListItem{
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Name: "my-dashboard",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
return resourceLister
|
||||
}(),
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
client := &mockDynamicClient{
|
||||
deleteFunc: func(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(clients, nil)
|
||||
|
||||
clients.
|
||||
On("ForResource", context.Background(), schema.GroupVersionResource{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
}).
|
||||
Once().
|
||||
Return(client, schema.GroupVersionKind{}, nil)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
onDeleteFunc: func(ctx context.Context) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Issue getting the namespace clients",
|
||||
lister: nil,
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(nil, assert.AnError)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
expectedErr: "remove resources: " + assert.AnError.Error(),
|
||||
},
|
||||
{
|
||||
name: "Issue listing items",
|
||||
lister: func() resources.ResourceLister {
|
||||
resourceLister := resources.NewMockResourceLister(t)
|
||||
|
||||
resourceLister.
|
||||
On("List", context.Background(), "default", "my-repo").
|
||||
Once().
|
||||
Return(nil, assert.AnError)
|
||||
|
||||
return resourceLister
|
||||
}(),
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(clients, nil)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
expectedErr: "remove resources: " + assert.AnError.Error(),
|
||||
},
|
||||
{
|
||||
name: "Issue getting client for resource",
|
||||
lister: func() resources.ResourceLister {
|
||||
resourceLister := resources.NewMockResourceLister(t)
|
||||
|
||||
resourceLister.
|
||||
On("List", context.Background(), "default", "my-repo").
|
||||
Once().
|
||||
Return(&provisioning.ResourceList{
|
||||
Items: []provisioning.ResourceListItem{
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Name: "my-dashboard",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
return resourceLister
|
||||
}(),
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(clients, nil)
|
||||
|
||||
clients.
|
||||
On("ForResource", context.Background(), schema.GroupVersionResource{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
}).
|
||||
Once().
|
||||
Return(nil, schema.GroupVersionKind{}, assert.AnError)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
expectedErr: "remove resources: " + assert.AnError.Error(),
|
||||
},
|
||||
{
|
||||
name: "Error deleting items",
|
||||
lister: func() resources.ResourceLister {
|
||||
resourceLister := resources.NewMockResourceLister(t)
|
||||
|
||||
resourceLister.
|
||||
On("List", context.Background(), "default", "my-repo").
|
||||
Once().
|
||||
Return(&provisioning.ResourceList{
|
||||
Items: []provisioning.ResourceListItem{
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Name: "my-dashboard",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
return resourceLister
|
||||
}(),
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
client := &mockDynamicClient{
|
||||
deleteFunc: func(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error {
|
||||
return assert.AnError
|
||||
},
|
||||
}
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(clients, nil)
|
||||
|
||||
clients.
|
||||
On("ForResource", context.Background(), schema.GroupVersionResource{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
}).
|
||||
Once().
|
||||
Return(client, schema.GroupVersionKind{}, nil)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
onDeleteFunc: func(ctx context.Context) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
expectedErr: "remove resources",
|
||||
},
|
||||
{
|
||||
name: "Error releasing items",
|
||||
lister: func() resources.ResourceLister {
|
||||
resourceLister := resources.NewMockResourceLister(t)
|
||||
|
||||
resourceLister.
|
||||
On("List", context.Background(), "default", "my-repo").
|
||||
Once().
|
||||
Return(&provisioning.ResourceList{
|
||||
Items: []provisioning.ResourceListItem{
|
||||
{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
Name: "my-dashboard",
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
return resourceLister
|
||||
}(),
|
||||
clientFactory: func() resources.ClientFactory {
|
||||
clientFactory := resources.NewMockClientFactory(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
client := &mockDynamicClient{
|
||||
patchFunc: func(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
|
||||
return nil, assert.AnError
|
||||
},
|
||||
}
|
||||
|
||||
clientFactory.
|
||||
On("Clients", context.Background(), "default").
|
||||
Once().
|
||||
Return(clients, nil)
|
||||
|
||||
clients.
|
||||
On("ForResource", context.Background(), schema.GroupVersionResource{
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
}).
|
||||
Once().
|
||||
Return(client, schema.GroupVersionKind{}, nil)
|
||||
|
||||
return clientFactory
|
||||
}(),
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
onDeleteFunc: func(ctx context.Context) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.ReleaseOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
expectedErr: "release resources",
|
||||
},
|
||||
{
|
||||
name: "Error deleting hooks",
|
||||
lister: nil,
|
||||
clientFactory: nil,
|
||||
repo: mockRepo{
|
||||
name: "my-repo",
|
||||
namespace: "default",
|
||||
onDeleteFunc: func(ctx context.Context) error {
|
||||
return assert.AnError
|
||||
},
|
||||
},
|
||||
finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
repository.CleanFinalizer,
|
||||
},
|
||||
expectedErr: "execute deletion hooks: " + assert.AnError.Error(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
metrics := registerFinalizerMetrics(prometheus.NewRegistry())
|
||||
f := &finalizer{
|
||||
lister: tc.lister,
|
||||
clientFactory: tc.clientFactory,
|
||||
metrics: &metrics,
|
||||
}
|
||||
err := f.process(context.Background(), tc.repo, tc.finalizers)
|
||||
if tc.expectedErr == "" {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortResourceListForDeletion(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
||||
@@ -41,6 +41,11 @@ type queueItem struct {
|
||||
attempts int
|
||||
}
|
||||
|
||||
//go:generate mockery --name finalizerProcessor --structname MockFinalizerProcessor --inpackage --filename finalizer_mock.go --with-expecter
|
||||
type finalizerProcessor interface {
|
||||
process(ctx context.Context, repo repository.Repository, finalizers []string) error
|
||||
}
|
||||
|
||||
// RepositoryController controls how and when CRD is established.
|
||||
type RepositoryController struct {
|
||||
client client.ProvisioningV0alpha1Interface
|
||||
@@ -50,7 +55,7 @@ type RepositoryController struct {
|
||||
dualwrite dualwrite.Service
|
||||
|
||||
jobs jobs.Queue
|
||||
finalizer *finalizer
|
||||
finalizer finalizerProcessor
|
||||
statusPatcher StatusPatcher
|
||||
|
||||
repoFactory repository.Factory
|
||||
@@ -223,12 +228,15 @@ func (rc *RepositoryController) handleDelete(ctx context.Context, obj *provision
|
||||
if len(obj.Finalizers) > 0 {
|
||||
repo, err := rc.repoFactory.Build(ctx, obj)
|
||||
if err != nil {
|
||||
logger.Warn("unable to get repository for cleanup")
|
||||
} else {
|
||||
err := rc.finalizer.process(ctx, repo, obj.Finalizers)
|
||||
if err != nil {
|
||||
logger.Warn("error running finalizer", "err", err)
|
||||
return fmt.Errorf("create repository from configuration: %w", err)
|
||||
}
|
||||
|
||||
err = rc.finalizer.process(ctx, repo, obj.Finalizers)
|
||||
if err != nil {
|
||||
if statusErr := rc.updateDeleteStatus(ctx, obj, fmt.Errorf("remove finalizers: %w", err)); statusErr != nil {
|
||||
logger.Error("failed to update repository status after finalizer removal error", "error", statusErr)
|
||||
}
|
||||
return fmt.Errorf("process finalizers: %w", err)
|
||||
}
|
||||
|
||||
// remove the finalizers
|
||||
@@ -238,12 +246,27 @@ func (rc *RepositoryController) handleDelete(ctx context.Context, obj *provision
|
||||
]`), v1.PatchOptions{
|
||||
FieldManager: "provisioning-controller",
|
||||
})
|
||||
return err // delete will be called again
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove finalizers: %w", err)
|
||||
}
|
||||
return nil
|
||||
} else {
|
||||
logger.Info("no finalizers to process")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rc *RepositoryController) updateDeleteStatus(ctx context.Context, obj *provisioning.Repository, err error) error {
|
||||
logger := logging.FromContext(ctx)
|
||||
logger.Info("updating repository status with deletion error", "error", err.Error())
|
||||
return rc.statusPatcher.Patch(ctx, obj, map[string]interface{}{
|
||||
"op": "replace",
|
||||
"path": "/status/deleteError",
|
||||
"value": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func (rc *RepositoryController) shouldResync(obj *provisioning.Repository) bool {
|
||||
// don't trigger resync if a sync was never started
|
||||
if obj.Status.Sync.Finished == 0 && obj.Status.Sync.State == "" {
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/controller/mocks"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
provisioningv0alpha1 "github.com/grafana/grafana/apps/provisioning/pkg/generated/applyconfiguration/provisioning/v0alpha1"
|
||||
client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
)
|
||||
|
||||
type mockProvisioningV0alpha1Interface struct {
|
||||
repositoriesFunc func(namespace string) client.RepositoryInterface
|
||||
}
|
||||
|
||||
func (m mockProvisioningV0alpha1Interface) RESTClient() rest.Interface {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockProvisioningV0alpha1Interface) HistoricJobs(namespace string) client.HistoricJobInterface {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockProvisioningV0alpha1Interface) Jobs(namespace string) client.JobInterface {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockProvisioningV0alpha1Interface) Repositories(namespace string) client.RepositoryInterface {
|
||||
if m.repositoriesFunc != nil {
|
||||
return m.repositoriesFunc(namespace)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type mockRepoInterface struct {
|
||||
patchFunc func(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *provisioning.Repository, err error)
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Create(ctx context.Context, repository *provisioning.Repository, opts metav1.CreateOptions) (*provisioning.Repository, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Update(ctx context.Context, repository *provisioning.Repository, opts metav1.UpdateOptions) (*provisioning.Repository, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) UpdateStatus(ctx context.Context, repository *provisioning.Repository, opts metav1.UpdateOptions) (*provisioning.Repository, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Get(ctx context.Context, name string, opts metav1.GetOptions) (*provisioning.Repository, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) List(ctx context.Context, opts metav1.ListOptions) (*provisioning.RepositoryList, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *provisioning.Repository, err error) {
|
||||
if m.patchFunc != nil {
|
||||
return m.patchFunc(ctx, name, pt, data, opts, subresources...)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) Apply(ctx context.Context, repository *provisioningv0alpha1.RepositoryApplyConfiguration, opts metav1.ApplyOptions) (result *provisioning.Repository, err error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
func (m mockRepoInterface) ApplyStatus(ctx context.Context, repository *provisioningv0alpha1.RepositoryApplyConfiguration, opts metav1.ApplyOptions) (result *provisioning.Repository, err error) {
|
||||
panic("not needed for testing")
|
||||
}
|
||||
|
||||
var (
|
||||
_ client.ProvisioningV0alpha1Interface = (*mockProvisioningV0alpha1Interface)(nil)
|
||||
_ client.RepositoryInterface = (*mockRepoInterface)(nil)
|
||||
)
|
||||
|
||||
func TestRepositoryController_handleDelete(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
repoFactory repository.Factory
|
||||
finalizer finalizerProcessor
|
||||
client client.ProvisioningV0alpha1Interface
|
||||
statusPatcher StatusPatcher
|
||||
repo *provisioning.Repository
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "No finalizers",
|
||||
repoFactory: nil,
|
||||
finalizer: nil,
|
||||
client: nil,
|
||||
statusPatcher: nil,
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Finalizers: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Finalizers deleted successfully",
|
||||
repoFactory: func() repository.Factory {
|
||||
f := repository.NewMockFactory(t)
|
||||
|
||||
f.
|
||||
On("Build", context.Background(), mock.Anything).
|
||||
Once().
|
||||
Return(nil, nil)
|
||||
|
||||
return f
|
||||
}(),
|
||||
finalizer: func() finalizerProcessor {
|
||||
f := NewMockFinalizerProcessor(t)
|
||||
|
||||
f.
|
||||
On("process", context.Background(), nil, []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
}).
|
||||
Once().
|
||||
Return(nil)
|
||||
|
||||
return f
|
||||
}(),
|
||||
client: func() client.ProvisioningV0alpha1Interface {
|
||||
repo := &mockRepoInterface{
|
||||
patchFunc: func(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *provisioning.Repository, err error) {
|
||||
return &provisioning.Repository{}, nil
|
||||
},
|
||||
}
|
||||
c := &mockProvisioningV0alpha1Interface{
|
||||
repositoriesFunc: func(namespace string) client.RepositoryInterface {
|
||||
return repo
|
||||
},
|
||||
}
|
||||
|
||||
return c
|
||||
}(),
|
||||
statusPatcher: nil,
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Error when building repository",
|
||||
repoFactory: func() repository.Factory {
|
||||
f := repository.NewMockFactory(t)
|
||||
|
||||
f.
|
||||
On("Build", context.Background(), mock.Anything).
|
||||
Once().
|
||||
Return(nil, assert.AnError)
|
||||
|
||||
return f
|
||||
}(),
|
||||
finalizer: nil,
|
||||
client: nil,
|
||||
statusPatcher: nil,
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedErr: "create repository from configuration: " + assert.AnError.Error(),
|
||||
},
|
||||
{
|
||||
name: "Error when processing finalizer",
|
||||
repoFactory: func() repository.Factory {
|
||||
f := repository.NewMockFactory(t)
|
||||
|
||||
f.
|
||||
On("Build", context.Background(), mock.Anything).
|
||||
Once().
|
||||
Return(nil, nil)
|
||||
|
||||
return f
|
||||
}(),
|
||||
finalizer: func() finalizerProcessor {
|
||||
f := NewMockFinalizerProcessor(t)
|
||||
|
||||
f.
|
||||
On("process", context.Background(), nil, []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
}).
|
||||
Once().
|
||||
Return(assert.AnError)
|
||||
|
||||
return f
|
||||
}(),
|
||||
statusPatcher: func() StatusPatcher {
|
||||
s := mocks.NewStatusPatcher(t)
|
||||
|
||||
s.
|
||||
On("Patch", context.Background(), mock.AnythingOfType("*v0alpha1.Repository"), mock.AnythingOfType("map[string]interface {}")).
|
||||
Once().
|
||||
Return(nil) // Return nil error for the status patch
|
||||
|
||||
return s
|
||||
}(),
|
||||
client: nil,
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedErr: "process finalizers: " + assert.AnError.Error(),
|
||||
},
|
||||
{
|
||||
name: "Error when patching finalizers",
|
||||
repoFactory: func() repository.Factory {
|
||||
f := repository.NewMockFactory(t)
|
||||
|
||||
f.
|
||||
On("Build", context.Background(), mock.Anything).
|
||||
Once().
|
||||
Return(nil, nil)
|
||||
|
||||
return f
|
||||
}(),
|
||||
finalizer: func() finalizerProcessor {
|
||||
f := NewMockFinalizerProcessor(t)
|
||||
|
||||
f.
|
||||
On("process", context.Background(), nil, []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
}).
|
||||
Once().
|
||||
Return(nil)
|
||||
|
||||
return f
|
||||
}(),
|
||||
client: func() client.ProvisioningV0alpha1Interface {
|
||||
repo := &mockRepoInterface{
|
||||
patchFunc: func(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *provisioning.Repository, err error) {
|
||||
return &provisioning.Repository{}, assert.AnError
|
||||
},
|
||||
}
|
||||
c := &mockProvisioningV0alpha1Interface{
|
||||
repositoriesFunc: func(namespace string) client.RepositoryInterface {
|
||||
return repo
|
||||
},
|
||||
}
|
||||
|
||||
return c
|
||||
}(),
|
||||
statusPatcher: nil,
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Finalizers: []string{
|
||||
repository.RemoveOrphanResourcesFinalizer,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedErr: "remove finalizers: " + assert.AnError.Error(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := &RepositoryController{
|
||||
repoFactory: tc.repoFactory,
|
||||
finalizer: tc.finalizer,
|
||||
client: tc.client,
|
||||
statusPatcher: tc.statusPatcher,
|
||||
}
|
||||
|
||||
err := c.handleDelete(context.Background(), tc.repo)
|
||||
if tc.expectedErr != "" {
|
||||
assert.Error(t, err)
|
||||
assert.ErrorContains(t, err, tc.expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user