diff --git a/pkg/registry/apis/provisioning/jobs/export/folders_test.go b/pkg/registry/apis/provisioning/jobs/export/folders_test.go index ac2c1aa13e4..bec0214d7ef 100644 --- a/pkg/registry/apis/provisioning/jobs/export/folders_test.go +++ b/pkg/registry/apis/provisioning/jobs/export/folders_test.go @@ -510,3 +510,10 @@ func (m *mockDynamicInterface) List(ctx context.Context, opts metav1.ListOptions func (m *mockDynamicInterface) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error { return m.deleteError } + +func (m *mockDynamicInterface) Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { + if len(m.items) == 0 { + return nil, fmt.Errorf("no items found") + } + return &m.items[0], nil +} diff --git a/pkg/registry/apis/provisioning/jobs/export/resources.go b/pkg/registry/apis/provisioning/jobs/export/resources.go index a401615c427..c469f4def8c 100644 --- a/pkg/registry/apis/provisioning/jobs/export/resources.go +++ b/pkg/registry/apis/provisioning/jobs/export/resources.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/dynamic" @@ -14,6 +16,11 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) +// FIXME: This is used to make sure we save dashboards in the apiVersion they were original saved in +// When requesting v0 or v2 dashboards over the v1 api -- the backend tries (and fails!) to convert values +// The response status indicates the original stored version, so we can then request it in an un-converted form +type conversionShim = func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error) + func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { progress.SetMessage(ctx, "start resource export") for _, kind := range resources.SupportedProvisioningResources { @@ -28,7 +35,39 @@ func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, return fmt.Errorf("get client for %s: %w", kind.Resource, err) } - if err := exportResource(ctx, options, client, repositoryResources, progress); err != nil { + // When requesting v2 (or v0) dashboards over the v1 api, we want to keep the original apiVersion if conversion fails + var shim conversionShim + if kind.GroupResource() == resources.DashboardResource.GroupResource() { + var v2client dynamic.ResourceInterface + shim = func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error) { + failed, _, _ := unstructured.NestedBool(item.Object, "status", "conversion", "failed") + if failed { + storedVersion, _, _ := unstructured.NestedString(item.Object, "status", "conversion", "storedVersion") + + // For v2 we need to request the original version + if strings.HasPrefix(storedVersion, "v2") { + if v2client == nil { + v2client, _, err = clients.ForResource(resources.DashboardResourceV2) + if err != nil { + return nil, err + } + } + return v2client.Get(ctx, item.GetName(), metav1.GetOptions{}) + } + + // For v0 we can simply fallback -- the full model is saved, but + if strings.HasPrefix(storedVersion, "v0") { + item.SetAPIVersion(fmt.Sprintf("%s/%s", kind.Group, storedVersion)) + return item, nil + } + + return nil, fmt.Errorf("unsupported dashboard version: %s", storedVersion) + } + return item, nil + } + } + + if err := exportResource(ctx, options, client, shim, repositoryResources, progress); err != nil { return fmt.Errorf("export %s: %w", kind.Resource, err) } } @@ -36,20 +75,32 @@ func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, return nil } -func exportResource(ctx context.Context, options provisioning.ExportJobOptions, client dynamic.ResourceInterface, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { - return resources.ForEach(ctx, client, func(item *unstructured.Unstructured) error { - fileName, err := repositoryResources.WriteResourceFileFromObject(ctx, item, resources.WriteOptions{ - Path: options.Path, - Ref: options.Branch, - }) - +func exportResource(ctx context.Context, + options provisioning.ExportJobOptions, + client dynamic.ResourceInterface, + shim conversionShim, + repositoryResources resources.RepositoryResources, + progress jobs.JobProgressRecorder, +) error { + // FIXME: using k8s list will force evrything into one version -- we really want the original saved version + // this will work well enough for now, but needs to be revisted as we have a bigger mix of active versions + return resources.ForEach(ctx, client, func(item *unstructured.Unstructured) (err error) { gvk := item.GroupVersionKind() result := jobs.JobResourceResult{ Name: item.GetName(), Resource: gvk.Kind, Group: gvk.Group, Action: repository.FileActionCreated, - Path: fileName, + } + + if shim != nil { + item, err = shim(ctx, item) + } + if err == nil { + result.Path, err = repositoryResources.WriteResourceFileFromObject(ctx, item, resources.WriteOptions{ + Path: options.Path, + Ref: options.Branch, + }) } if errors.Is(err, resources.ErrAlreadyInRepository) { diff --git a/pkg/registry/apis/provisioning/jobs/export/resources_test.go b/pkg/registry/apis/provisioning/jobs/export/resources_test.go index d1a75443316..64f7d256866 100644 --- a/pkg/registry/apis/provisioning/jobs/export/resources_test.go +++ b/pkg/registry/apis/provisioning/jobs/export/resources_test.go @@ -7,12 +7,8 @@ import ( mock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - dynamicfake "k8s.io/client-go/dynamic/fake" - k8testing "k8s.io/client-go/testing" provisioningV0 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" @@ -20,44 +16,35 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) -func TestExportResources(t *testing.T) { +func TestExportResources_Dashboards(t *testing.T) { tests := []struct { name string - reactorFunc func(action k8testing.Action) (bool, runtime.Object, error) + mockItems []unstructured.Unstructured expectedError string setupProgress func(progress *jobs.MockJobProgressRecorder) - setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) + setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) }{ { name: "successful dashboard export", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - // Return dashboard list - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-1", - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-2", - }, + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "dashboard-1", }, }, - }, nil + }, + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "dashboard-2", + }, + }, + }, }, expectedError: "", setupProgress: func(progress *jobs.MockJobProgressRecorder) { @@ -72,8 +59,8 @@ func TestExportResources(t *testing.T) { progress.On("TooManyErrors").Return(nil) progress.On("TooManyErrors").Return(nil) }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) options := resources.WriteOptions{ Path: "grafana", Ref: "feature/branch", @@ -89,62 +76,38 @@ func TestExportResources(t *testing.T) { }, }, { - name: "client error", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, nil, fmt.Errorf("shouldn't happen") - }, + name: "client error", + mockItems: nil, expectedError: "get client for dashboards: didn't work", setupProgress: func(progress *jobs.MockJobProgressRecorder) { progress.On("SetMessage", mock.Anything, "start resource export").Return() progress.On("SetMessage", mock.Anything, "export dashboards").Return() }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, fmt.Errorf("didn't work")) - }, - }, - { - name: "dashboard list error", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, nil, fmt.Errorf("failed to list dashboards") - }, - expectedError: "export dashboards: error executing list: failed to list dashboards", - setupProgress: func(progress *jobs.MockJobProgressRecorder) { - progress.On("SetMessage", mock.Anything, "start resource export").Return() - progress.On("SetMessage", mock.Anything, "export dashboards").Return() - }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, fmt.Errorf("didn't work")) }, }, { name: "dashboard export with errors", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-1", - }, - }, - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-2", - }, + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "dashboard-1", }, }, - }, nil + }, + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "dashboard-2", + }, + }, + }, }, expectedError: "", setupProgress: func(progress *jobs.MockJobProgressRecorder) { @@ -159,8 +122,8 @@ func TestExportResources(t *testing.T) { progress.On("TooManyErrors").Return(nil) progress.On("TooManyErrors").Return(nil) }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) options := resources.WriteOptions{ Path: "grafana", Ref: "feature/branch", @@ -177,24 +140,16 @@ func TestExportResources(t *testing.T) { }, { name: "dashboard export too many errors", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "dashboard-1", - }, + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "dashboard-1", }, }, - }, nil + }, }, expectedError: "export dashboards: too many errors encountered", setupProgress: func(progress *jobs.MockJobProgressRecorder) { @@ -205,8 +160,8 @@ func TestExportResources(t *testing.T) { })).Return() progress.On("TooManyErrors").Return(fmt.Errorf("too many errors encountered")) }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) options := resources.WriteOptions{ Path: "grafana", Ref: "feature/branch", @@ -219,24 +174,16 @@ func TestExportResources(t *testing.T) { }, { name: "ignores existing dashboards", - reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) { - return true, &metav1.PartialObjectMetadataList{ - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "DashboardList", - }, - Items: []metav1.PartialObjectMetadata{ - { - TypeMeta: metav1.TypeMeta{ - APIVersion: resources.DashboardResource.GroupVersion().String(), - Kind: "Dashboard", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "existing-dashboard", - }, + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "existing-dashboard", }, }, - }, nil + }, }, expectedError: "", setupProgress: func(progress *jobs.MockJobProgressRecorder) { @@ -247,50 +194,238 @@ func TestExportResources(t *testing.T) { })).Return() progress.On("TooManyErrors").Return(nil) }, - setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) { - resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil) + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) options := resources.WriteOptions{ Path: "grafana", Ref: "feature/branch", } - // Return true to indicate the file already exists, and provide the updated path repoResources.On("WriteResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { return obj.GetName() == "existing-dashboard" }), options).Return("", resources.ErrAlreadyInRepository) }, }, + { + name: "uses saved dashboard version", + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "existing-dashboard", + }, + "spec": map[string]interface{}{ + "hello": "world", + }, + "status": map[string]interface{}{ + "conversion": map[string]interface{}{ + "failed": true, + "storedVersion": "v0xyz", + }, + }, + }, + }, + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "existing-dashboard" && result.Action == repository.FileActionIgnored + })).Return() + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) + options := resources.WriteOptions{ + Path: "grafana", + Ref: "feature/branch", + } + + repoResources.On("WriteResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool { + // Verify that the object has the expected status.conversion.storedVersion field + status, exists, err := unstructured.NestedMap(obj.Object, "status") + if !exists || err != nil { + return false + } + + conversion, exists, err := unstructured.NestedMap(status, "conversion") + if !exists || err != nil { + return false + } + + storedVersion, exists, err := unstructured.NestedString(conversion, "storedVersion") + if !exists || err != nil { + return false + } + + if storedVersion != "v0xyz" { + return false + } + + return obj.GetName() == "existing-dashboard" + }), options).Return("", fmt.Errorf("XXX")) + }, + }, + { + name: "dashboard with failed conversion but no stored version", + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "dashboard-no-stored-version", + }, + "status": map[string]interface{}{ + "conversion": map[string]interface{}{ + "failed": true, + // No storedVersion field + }, + }, + }, + }, + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "dashboard-no-stored-version" && + result.Action == repository.FileActionIgnored && + result.Error != nil + })).Return() + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) + // The value is not saved + }, + }, + { + name: "handles v2 dashboard version", + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "v2-dashboard", + }, + "status": map[string]interface{}{ + "conversion": map[string]interface{}{ + "failed": true, + "storedVersion": "v2", + }, + }, + }, + }, + }, + expectedError: "", + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + return result.Name == "v2-dashboard" && result.Action == repository.FileActionCreated + })).Return() + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + // Setup v1 client + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) + + // Setup v2 client + + // Mock v2 client Get call + v2Dashboard := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "dashboard.grafana.app/v2alpha1", + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "v2-dashboard", + }, + "spec": map[string]interface{}{ + "version": 2, + "title": "V2 Dashboard", + }, + }, + } + v2Client := &mockDynamicInterface{items: []unstructured.Unstructured{*v2Dashboard}} + resourceClients.On("ForResource", resources.DashboardResourceV2).Return(v2Client, gvk, nil) + + options := resources.WriteOptions{ + Path: "grafana", + Ref: "feature/branch", + } + repoResources.On("WriteResourceFileFromObject", mock.Anything, v2Dashboard, options).Return("v2-dashboard.json", nil) + }, + }, + { + name: "handles v2 client creation error", + mockItems: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": resources.DashboardResource.GroupVersion().String(), + "kind": "Dashboard", + "metadata": map[string]interface{}{ + "name": "v2-dashboard-error", + }, + "status": map[string]interface{}{ + "conversion": map[string]interface{}{ + "failed": true, + "storedVersion": "v2", + }, + }, + }, + }, + }, + setupProgress: func(progress *jobs.MockJobProgressRecorder) { + progress.On("SetMessage", mock.Anything, "start resource export").Return() + progress.On("SetMessage", mock.Anything, "export dashboards").Return() + progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { + if result.Name != "v2-dashboard-error" { + return false + } + if result.Action != repository.FileActionIgnored { + return false + } + if result.Error == nil { + return false + } + + if result.Error.Error() != "v2 client error" { + return false + } + + return true + })).Return() + progress.On("TooManyErrors").Return(nil) + }, + setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) { + resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil) + resourceClients.On("ForResource", resources.DashboardResourceV2).Return(nil, gvk, fmt.Errorf("v2 client error")) + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, metav1.AddMetaToScheme(scheme)) - listGVK := schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: "DashboardList", + mockClient := &mockDynamicInterface{ + items: tt.mockItems, } - scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{}) - scheme.AddKnownTypeWithName(schema.GroupVersionKind{ - Group: resources.DashboardResource.Group, - Version: resources.DashboardResource.Version, - Kind: resources.DashboardResource.Resource, - }, &metav1.PartialObjectMetadata{}) - - fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - resources.DashboardResource: listGVK.Kind, - }) - resourceClients := resources.NewMockResourceClients(t) - fakeDynamicClient.PrependReactor("list", "dashboards", tt.reactorFunc) - mockProgress := jobs.NewMockJobProgressRecorder(t) tt.setupProgress(mockProgress) repoResources := resources.NewMockRepositoryResources(t) - tt.setupResources(repoResources, resourceClients, fakeDynamicClient, listGVK) + tt.setupResources(repoResources, resourceClients, mockClient, schema.GroupVersionKind{ + Group: resources.DashboardResource.Group, + Version: resources.DashboardResource.Version, + Kind: "DashboardList", + }) options := provisioningV0.ExportJobOptions{ Path: "grafana", diff --git a/pkg/registry/apis/provisioning/resources/client.go b/pkg/registry/apis/provisioning/resources/client.go index fcdabcb0705..b27f45b6892 100644 --- a/pkg/registry/apis/provisioning/resources/client.go +++ b/pkg/registry/apis/provisioning/resources/client.go @@ -10,7 +10,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" - dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" + dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" + dashboardV2 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2alpha1" folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" iam "github.com/grafana/grafana/pkg/apis/iam/v0alpha1" "github.com/grafana/grafana/pkg/services/apiserver" @@ -18,9 +19,10 @@ import ( ) var ( - UserResource = iam.UserResourceInfo.GroupVersionResource() - FolderResource = folders.FolderResourceInfo.GroupVersionResource() - DashboardResource = dashboard.DashboardResourceInfo.GroupVersionResource() + UserResource = iam.UserResourceInfo.GroupVersionResource() + FolderResource = folders.FolderResourceInfo.GroupVersionResource() + DashboardResource = dashboardV1.DashboardResourceInfo.GroupVersionResource() + DashboardResourceV2 = dashboardV2.DashboardResourceInfo.GroupVersionResource() // SupportedProvisioningResources is the list of resources that can fully managed from the UI SupportedProvisioningResources = []schema.GroupVersionResource{FolderResource, DashboardResource} diff --git a/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v0.yaml b/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v0.yaml new file mode 100644 index 00000000000..2d4cf82b060 --- /dev/null +++ b/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v0.yaml @@ -0,0 +1,8 @@ +apiVersion: dashboard.grafana.app/v0alpha1 +kind: Dashboard +metadata: + name: test-v0 +spec: + title: Test dashboard. Created at v0 + uid: test-v0 # will be removed by mutation hook + version: 1234567 # will be removed by mutation hook diff --git a/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v1.yaml b/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v1.yaml new file mode 100644 index 00000000000..18dee5c28d9 --- /dev/null +++ b/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v1.yaml @@ -0,0 +1,9 @@ +apiVersion: dashboard.grafana.app/v1beta1 +kind: Dashboard +metadata: + name: test-v1 +spec: + title: Test dashboard. Created at v1 + uid: test-v1 # will be removed by mutation hook + version: 1234567 # will be removed by mutation hook + schemaVersion: 41 diff --git a/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v2.yaml b/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v2.yaml new file mode 100644 index 00000000000..540b9dcc919 --- /dev/null +++ b/pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v2.yaml @@ -0,0 +1,10 @@ +apiVersion: dashboard.grafana.app/v2alpha1 +kind: Dashboard +metadata: + name: test-v2 +spec: + title: Test dashboard. Created at v2 + layout: + kind: GridLayout + spec: + items: [] diff --git a/pkg/tests/apis/provisioning/exportunifiedtorepository/root_dashboard.json b/pkg/tests/apis/provisioning/exportunifiedtorepository/root_dashboard.json deleted file mode 100644 index b2008d618b0..00000000000 --- a/pkg/tests/apis/provisioning/exportunifiedtorepository/root_dashboard.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "apiVersion": "dashboard.grafana.app/v1beta1", - "kind": "Dashboard", - "metadata": { - "name": "root_dashboard", - "namespace": "default" - }, - "spec": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations \u0026 Alerts", - "type": "dashboard" - } - ] - }, - "description": "This is on the root level of the repository.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlPu", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "Y axis on the right side", - "axisPlacement": "right", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 7, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "decimals": 3, - "fieldMinMax": false, - "links": [ - { - "oneClick": true, - "targetBlank": true, - "title": "Test test", - "url": "https://github.com/grafana/grafana" - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "kvoltamp" - }, - "overrides": [] - }, - "gridPos": { - "h": 24, - "w": 22, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-pre", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Test panel", - "type": "timeseries" - } - ], - "preload": false, - "schemaVersion": 41, - "tags": [ - "grafana", - "dashboard", - "root", - "test" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "Pacific/Kiritimati", - "title": "Test dashboard on root level", - "uid": "", - "version": 0, - "weekStart": "saturday" - } -} diff --git a/pkg/tests/apis/provisioning/helper_test.go b/pkg/tests/apis/provisioning/helper_test.go index e7226447fd1..bf63e2358fc 100644 --- a/pkg/tests/apis/provisioning/helper_test.go +++ b/pkg/tests/apis/provisioning/helper_test.go @@ -23,7 +23,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" - dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" + dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1" + dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" + dashboardV2 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2alpha1" folder "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" @@ -46,7 +48,9 @@ type provisioningTestHelper struct { Repositories *apis.K8sResourceClient Jobs *apis.K8sResourceClient Folders *apis.K8sResourceClient - Dashboards *apis.K8sResourceClient + DashboardsV0 *apis.K8sResourceClient + DashboardsV1 *apis.K8sResourceClient + DashboardsV2 *apis.K8sResourceClient AdminREST *rest.RESTClient EditorREST *rest.RESTClient ViewerREST *rest.RESTClient @@ -250,10 +254,20 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper Namespace: "default", // actually org1 GVR: folder.FolderResourceInfo.GroupVersionResource(), }) - dashboards := helper.GetResourceClient(apis.ResourceClientArgs{ + dashboardsV0 := helper.GetResourceClient(apis.ResourceClientArgs{ User: helper.Org1.Admin, Namespace: "default", // actually org1 - GVR: dashboard.DashboardResourceInfo.GroupVersionResource(), + GVR: dashboardV0.DashboardResourceInfo.GroupVersionResource(), + }) + dashboardsV1 := helper.GetResourceClient(apis.ResourceClientArgs{ + User: helper.Org1.Admin, + Namespace: "default", // actually org1 + GVR: dashboardV1.DashboardResourceInfo.GroupVersionResource(), + }) + dashboardsV2 := helper.GetResourceClient(apis.ResourceClientArgs{ + User: helper.Org1.Admin, + Namespace: "default", // actually org1 + GVR: dashboardV2.DashboardResourceInfo.GroupVersionResource(), }) // Repo client, but less guard rails. Useful for subresources. We'll need this later... @@ -276,7 +290,7 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper return nil } - require.NoError(t, deleteAll(dashboards), "deleting all dashboards") + require.NoError(t, deleteAll(dashboardsV1), "deleting all dashboards") // v0+v1+v2 require.NoError(t, deleteAll(folders), "deleting all folders") require.NoError(t, deleteAll(repositories), "deleting all repositories") @@ -290,7 +304,9 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper ViewerREST: viewerClient, Jobs: jobs, Folders: folders, - Dashboards: dashboards, + DashboardsV0: dashboardsV0, + DashboardsV1: dashboardsV1, + DashboardsV2: dashboardsV2, } } diff --git a/pkg/tests/apis/provisioning/provisioning_test.go b/pkg/tests/apis/provisioning/provisioning_test.go index b70137b823a..22ec10b7073 100644 --- a/pkg/tests/apis/provisioning/provisioning_test.go +++ b/pkg/tests/apis/provisioning/provisioning_test.go @@ -147,7 +147,7 @@ func TestIntegrationProvisioning_FailInvalidSchema(t *testing.T) { require.Equal(t, status.Message, "Dry run failed: Dashboard.dashboard.grafana.app \"invalid-schema-uid\" is invalid: [spec.panels.0.repeatDirection: Invalid value: conflicting values \"h\" and \"this is not an allowed value\", spec.panels.0.repeatDirection: Invalid value: conflicting values \"v\" and \"this is not an allowed value\"]") const invalidSchemaUid = "invalid-schema-uid" - _, err = helper.Dashboards.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{}) + _, err = helper.DashboardsV1.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{}) require.Error(t, err, "invalid dashboard shouldn't exist") require.True(t, apierrors.IsNotFound(err)) @@ -195,7 +195,7 @@ func TestIntegrationProvisioning_FailInvalidSchema(t *testing.T) { require.Equal(t, job.Status.Errors[0], "Dashboard.dashboard.grafana.app \"invalid-schema-uid\" is invalid: [spec.panels.0.repeatDirection: Invalid value: conflicting values \"h\" and \"this is not an allowed value\", spec.panels.0.repeatDirection: Invalid value: conflicting values \"v\" and \"this is not an allowed value\"]") }, time.Second*10, time.Millisecond*10, "Expected provisioning job to conclude with the status failed") - _, err = helper.Dashboards.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{}) + _, err = helper.DashboardsV1.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{}) require.Error(t, err, "invalid dashboard shouldn't have been created") require.True(t, apierrors.IsNotFound(err)) @@ -272,7 +272,7 @@ func TestIntegrationProvisioning_CreatingGitHubRepository(t *testing.T) { // By now, we should have synced, meaning we have data to read in the local Grafana instance! - found, err := helper.Dashboards.Resource.List(ctx, metav1.ListOptions{}) + found, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{}) require.NoError(t, err, "can list values") names := []string{} @@ -286,7 +286,7 @@ func TestIntegrationProvisioning_CreatingGitHubRepository(t *testing.T) { require.NoError(t, err, "should delete values") assert.EventuallyWithT(t, func(collect *assert.CollectT) { - found, err := helper.Dashboards.Resource.List(ctx, metav1.ListOptions{}) + found, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{}) require.NoError(t, err, "can list values") require.Equal(collect, 0, len(found.Items), "expected dashboards to be deleted") }, time.Second*10, time.Millisecond*10, "Expected dashboards to be deleted") @@ -396,7 +396,7 @@ func TestIntegrationProvisioning_RunLocalRepository(t *testing.T) { require.Equal(t, allPanels, name, "save the name from the request") // Get the file from the grafana database - obj, err := helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{}) + obj, err := helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{}) require.NoError(t, err, "the value should be saved in grafana") val, _, _ := unstructured.NestedString(obj.Object, "metadata", "annotations", utils.AnnoKeyManagerKind) require.Equal(t, string(utils.ManagerKindRepo), val, "should have repo annotations") @@ -539,33 +539,33 @@ func TestIntegrationProvisioning_ImportAllPanelsFromLocalRepository(t *testing.T // But the dashboard shouldn't exist yet const allPanels = "n1jR8vnnz" - _, err = helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{}) + _, err = helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{}) require.Error(t, err, "no all-panels dashboard should exist") require.True(t, apierrors.IsNotFound(err)) // Now, we import it, such that it may exist helper.SyncAndWait(t, repo, nil) - _, err = helper.Dashboards.Resource.List(ctx, metav1.ListOptions{}) + _, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{}) require.NoError(t, err, "can list values") - obj, err = helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{}) + obj, err = helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{}) require.NoError(t, err, "all-panels dashboard should exist") require.Equal(t, repo, obj.GetAnnotations()[utils.AnnoKeyManagerIdentity]) // Try writing the value directly err = unstructured.SetNestedField(obj.Object, []any{"aaa", "bbb"}, "spec", "tags") require.NoError(t, err, "set tags") - obj, err = helper.Dashboards.Resource.Update(ctx, obj, metav1.UpdateOptions{}) + obj, err = helper.DashboardsV1.Resource.Update(ctx, obj, metav1.UpdateOptions{}) require.NoError(t, err) v, _, _ := unstructured.NestedString(obj.Object, "metadata", "annotations", utils.AnnoKeyUpdatedBy) require.Equal(t, "access-policy:provisioning", v) // Should not be able to directly delete the managed resource - err = helper.Dashboards.Resource.Delete(ctx, allPanels, metav1.DeleteOptions{}) + err = helper.DashboardsV1.Resource.Delete(ctx, allPanels, metav1.DeleteOptions{}) require.NoError(t, err, "user can delete") - _, err = helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{}) + _, err = helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{}) require.Error(t, err, "should delete the internal resource") require.True(t, apierrors.IsNotFound(err)) } @@ -578,10 +578,18 @@ func TestProvisioning_ExportUnifiedToRepository(t *testing.T) { helper := runGrafana(t) ctx := context.Background() - // Set up dashboards first, then the repository, and finally export. - dashboard := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/root_dashboard.json") - _, err := helper.Dashboards.Resource.Create(ctx, dashboard, metav1.CreateOptions{}) - require.NoError(t, err, "should be able to create prerequisite dashboard") + // Write dashboards at + dashboard := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v0.yaml") + _, err := helper.DashboardsV0.Resource.Create(ctx, dashboard, metav1.CreateOptions{}) + require.NoError(t, err, "should be able to create v0 dashboard") + + dashboard = helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v1.yaml") + _, err = helper.DashboardsV1.Resource.Create(ctx, dashboard, metav1.CreateOptions{}) + require.NoError(t, err, "should be able to create v1 dashboard") + + dashboard = helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v2.yaml") + _, err = helper.DashboardsV2.Resource.Create(ctx, dashboard, metav1.CreateOptions{}) + require.NoError(t, err, "should be able to create v2 dashboard") // Now for the repository. const repo = "local-repository" @@ -608,7 +616,38 @@ func TestProvisioning_ExportUnifiedToRepository(t *testing.T) { // And time to assert. helper.AwaitJobs(t, repo) - fpath := filepath.Join(helper.ProvisioningPath, slugify.Slugify(mustNestedString(dashboard.Object, "spec", "title"))+".json") - _, err = os.Stat(fpath) - require.NoError(t, err, "exported file was not created at path %s", fpath) + type props struct { + title string + apiVersion string + name string + } + + // Check that each file was exported with its stored version + for _, test := range []props{ + {title: "Test dashboard. Created at v0", apiVersion: "dashboard.grafana.app/v0alpha1", name: "test-v0"}, + {title: "Test dashboard. Created at v1", apiVersion: "dashboard.grafana.app/v1beta1", name: "test-v1"}, + {title: "Test dashboard. Created at v2", apiVersion: "dashboard.grafana.app/v2alpha1", name: "test-v2"}, + } { + fpath := filepath.Join(helper.ProvisioningPath, slugify.Slugify(test.title)+".json") + //nolint:gosec // we are ok with reading files in testdata + body, err := os.ReadFile(fpath) + require.NoError(t, err, "exported file was not created at path %s", fpath) + obj := map[string]any{} + err = json.Unmarshal(body, &obj) + require.NoError(t, err, "exported file not json %s", fpath) + + val, _, err := unstructured.NestedString(obj, "apiVersion") + require.NoError(t, err) + require.Equal(t, test.apiVersion, val) + + val, _, err = unstructured.NestedString(obj, "spec", "title") + require.NoError(t, err) + require.Equal(t, test.title, val) + + val, _, err = unstructured.NestedString(obj, "metadata", "name") + require.NoError(t, err) + require.Equal(t, test.name, val) + + require.Nil(t, obj["status"], "should not have a status element") + } }