public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provisioning: Export dashboards with conversion errors (#104369)

Browse Source
This commit is contained in:
Ryan McKinley
2025-04-24 14:16:25 +01:00
committed by GitHub
parent e385237daf
commit 2e51096eb4
10 changed files with 453 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/registry/apis/provisioning/jobs/export/folders_test.go
+7
View File
@@ -510,3 +510,10 @@ func (m *mockDynamicInterface) List(ctx context.Context, opts metav1.ListOptions
func (m *mockDynamicInterface) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error {
return m.deleteError
}
func (m *mockDynamicInterface) Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
if len(m.items) == 0 {
return nil, fmt.Errorf("no items found")
}
return &m.items[0], nil
}
pkg/registry/apis/provisioning/jobs/export/resources.go
+60 -9
View File
@@ -4,7 +4,9 @@ import (
"context"
"errors"
"fmt"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/dynamic"
@@ -14,6 +16,11 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
// FIXME: This is used to make sure we save dashboards in the apiVersion they were original saved in
// When requesting v0 or v2 dashboards over the v1 api -- the backend tries (and fails!) to convert values
// The response status indicates the original stored version, so we can then request it in an un-converted form
type conversionShim = func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error)
func ExportResources(ctx context.Context, options provisioning.ExportJobOptions, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error {
progress.SetMessage(ctx, "start resource export")
for _, kind := range resources.SupportedProvisioningResources {
@@ -28,7 +35,39 @@ func ExportResources(ctx context.Context, options provisioning.ExportJobOptions,
return fmt.Errorf("get client for %s: %w", kind.Resource, err)
}
if err := exportResource(ctx, options, client, repositoryResources, progress); err != nil {
// When requesting v2 (or v0) dashboards over the v1 api, we want to keep the original apiVersion if conversion fails
var shim conversionShim
if kind.GroupResource() == resources.DashboardResource.GroupResource() {
var v2client dynamic.ResourceInterface
shim = func(ctx context.Context, item *unstructured.Unstructured) (*unstructured.Unstructured, error) {
failed, _, _ := unstructured.NestedBool(item.Object, "status", "conversion", "failed")
if failed {
storedVersion, _, _ := unstructured.NestedString(item.Object, "status", "conversion", "storedVersion")
// For v2 we need to request the original version
if strings.HasPrefix(storedVersion, "v2") {
if v2client == nil {
v2client, _, err = clients.ForResource(resources.DashboardResourceV2)
if err != nil {
return nil, err
}
}
return v2client.Get(ctx, item.GetName(), metav1.GetOptions{})
}
// For v0 we can simply fallback -- the full model is saved, but
if strings.HasPrefix(storedVersion, "v0") {
item.SetAPIVersion(fmt.Sprintf("%s/%s", kind.Group, storedVersion))
return item, nil
}
return nil, fmt.Errorf("unsupported dashboard version: %s", storedVersion)
}
return item, nil
}
}
if err := exportResource(ctx, options, client, shim, repositoryResources, progress); err != nil {
return fmt.Errorf("export %s: %w", kind.Resource, err)
}
}
@@ -36,20 +75,32 @@ func ExportResources(ctx context.Context, options provisioning.ExportJobOptions,
return nil
}
func exportResource(ctx context.Context, options provisioning.ExportJobOptions, client dynamic.ResourceInterface, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error {
return resources.ForEach(ctx, client, func(item *unstructured.Unstructured) error {
fileName, err := repositoryResources.WriteResourceFileFromObject(ctx, item, resources.WriteOptions{
Path: options.Path,
Ref: options.Branch,
})
func exportResource(ctx context.Context,
options provisioning.ExportJobOptions,
client dynamic.ResourceInterface,
shim conversionShim,
repositoryResources resources.RepositoryResources,
progress jobs.JobProgressRecorder,
) error {
// FIXME: using k8s list will force evrything into one version -- we really want the original saved version
// this will work well enough for now, but needs to be revisted as we have a bigger mix of active versions
return resources.ForEach(ctx, client, func(item *unstructured.Unstructured) (err error) {
gvk := item.GroupVersionKind()
result := jobs.JobResourceResult{
Name: item.GetName(),
Resource: gvk.Kind,
Group: gvk.Group,
Action: repository.FileActionCreated,
Path: fileName,
}
if shim != nil {
item, err = shim(ctx, item)
}
if err == nil {
result.Path, err = repositoryResources.WriteResourceFileFromObject(ctx, item, resources.WriteOptions{
Path: options.Path,
Ref: options.Branch,
})
}
if errors.Is(err, resources.ErrAlreadyInRepository) {
pkg/registry/apis/provisioning/jobs/export/resources_test.go
+274 -139
View File
@@ -7,12 +7,8 @@ import (
mock "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8testing "k8s.io/client-go/testing"
provisioningV0 "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
@@ -20,44 +16,35 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
)
func TestExportResources(t *testing.T) {
func TestExportResources_Dashboards(t *testing.T) {
tests := []struct {
name string
reactorFunc func(action k8testing.Action) (bool, runtime.Object, error)
mockItems []unstructured.Unstructured
expectedError string
setupProgress func(progress *jobs.MockJobProgressRecorder)
setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind)
setupResources func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind)
}{
{
name: "successful dashboard export",
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
// Return dashboard list
return true, &metav1.PartialObjectMetadataList{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "DashboardList",
},
Items: []metav1.PartialObjectMetadata{
{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "Dashboard",
},
ObjectMeta: metav1.ObjectMeta{
Name: "dashboard-1",
},
},
{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "Dashboard",
},
ObjectMeta: metav1.ObjectMeta{
Name: "dashboard-2",
},
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "dashboard-1",
},
},
}, nil
},
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "dashboard-2",
},
},
},
},
expectedError: "",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
@@ -72,8 +59,8 @@ func TestExportResources(t *testing.T) {
progress.On("TooManyErrors").Return(nil)
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil)
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
options := resources.WriteOptions{
Path: "grafana",
Ref: "feature/branch",
@@ -89,62 +76,38 @@ func TestExportResources(t *testing.T) {
},
},
{
name: "client error",
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("shouldn't happen")
},
name: "client error",
mockItems: nil,
expectedError: "get client for dashboards: didn't work",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
progress.On("SetMessage", mock.Anything, "start resource export").Return()
progress.On("SetMessage", mock.Anything, "export dashboards").Return()
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, fmt.Errorf("didn't work"))
},
},
{
name: "dashboard list error",
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("failed to list dashboards")
},
expectedError: "export dashboards: error executing list: failed to list dashboards",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
progress.On("SetMessage", mock.Anything, "start resource export").Return()
progress.On("SetMessage", mock.Anything, "export dashboards").Return()
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil)
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, fmt.Errorf("didn't work"))
},
},
{
name: "dashboard export with errors",
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, &metav1.PartialObjectMetadataList{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "DashboardList",
},
Items: []metav1.PartialObjectMetadata{
{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "Dashboard",
},
ObjectMeta: metav1.ObjectMeta{
Name: "dashboard-1",
},
},
{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "Dashboard",
},
ObjectMeta: metav1.ObjectMeta{
Name: "dashboard-2",
},
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "dashboard-1",
},
},
}, nil
},
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "dashboard-2",
},
},
},
},
expectedError: "",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
@@ -159,8 +122,8 @@ func TestExportResources(t *testing.T) {
progress.On("TooManyErrors").Return(nil)
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil)
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
options := resources.WriteOptions{
Path: "grafana",
Ref: "feature/branch",
@@ -177,24 +140,16 @@ func TestExportResources(t *testing.T) {
},
{
name: "dashboard export too many errors",
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, &metav1.PartialObjectMetadataList{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "DashboardList",
},
Items: []metav1.PartialObjectMetadata{
{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "Dashboard",
},
ObjectMeta: metav1.ObjectMeta{
Name: "dashboard-1",
},
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "dashboard-1",
},
},
}, nil
},
},
expectedError: "export dashboards: too many errors encountered",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
@@ -205,8 +160,8 @@ func TestExportResources(t *testing.T) {
})).Return()
progress.On("TooManyErrors").Return(fmt.Errorf("too many errors encountered"))
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil)
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
options := resources.WriteOptions{
Path: "grafana",
Ref: "feature/branch",
@@ -219,24 +174,16 @@ func TestExportResources(t *testing.T) {
},
{
name: "ignores existing dashboards",
reactorFunc: func(action k8testing.Action) (bool, runtime.Object, error) {
return true, &metav1.PartialObjectMetadataList{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "DashboardList",
},
Items: []metav1.PartialObjectMetadata{
{
TypeMeta: metav1.TypeMeta{
APIVersion: resources.DashboardResource.GroupVersion().String(),
Kind: "Dashboard",
},
ObjectMeta: metav1.ObjectMeta{
Name: "existing-dashboard",
},
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "existing-dashboard",
},
},
}, nil
},
},
expectedError: "",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
@@ -247,50 +194,238 @@ func TestExportResources(t *testing.T) {
})).Return()
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, dynamicClient *dynamicfake.FakeDynamicClient, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(dynamicClient.Resource(resources.DashboardResource), gvk, nil)
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
options := resources.WriteOptions{
Path: "grafana",
Ref: "feature/branch",
}
// Return true to indicate the file already exists, and provide the updated path
repoResources.On("WriteResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool {
return obj.GetName() == "existing-dashboard"
}), options).Return("", resources.ErrAlreadyInRepository)
},
},
{
name: "uses saved dashboard version",
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "existing-dashboard",
},
"spec": map[string]interface{}{
"hello": "world",
},
"status": map[string]interface{}{
"conversion": map[string]interface{}{
"failed": true,
"storedVersion": "v0xyz",
},
},
},
},
},
expectedError: "",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
progress.On("SetMessage", mock.Anything, "start resource export").Return()
progress.On("SetMessage", mock.Anything, "export dashboards").Return()
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Name == "existing-dashboard" && result.Action == repository.FileActionIgnored
})).Return()
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
options := resources.WriteOptions{
Path: "grafana",
Ref: "feature/branch",
}
repoResources.On("WriteResourceFileFromObject", mock.Anything, mock.MatchedBy(func(obj *unstructured.Unstructured) bool {
// Verify that the object has the expected status.conversion.storedVersion field
status, exists, err := unstructured.NestedMap(obj.Object, "status")
if !exists || err != nil {
return false
}
conversion, exists, err := unstructured.NestedMap(status, "conversion")
if !exists || err != nil {
return false
}
storedVersion, exists, err := unstructured.NestedString(conversion, "storedVersion")
if !exists || err != nil {
return false
}
if storedVersion != "v0xyz" {
return false
}
return obj.GetName() == "existing-dashboard"
}), options).Return("", fmt.Errorf("XXX"))
},
},
{
name: "dashboard with failed conversion but no stored version",
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "dashboard-no-stored-version",
},
"status": map[string]interface{}{
"conversion": map[string]interface{}{
"failed": true,
// No storedVersion field
},
},
},
},
},
expectedError: "",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
progress.On("SetMessage", mock.Anything, "start resource export").Return()
progress.On("SetMessage", mock.Anything, "export dashboards").Return()
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Name == "dashboard-no-stored-version" &&
result.Action == repository.FileActionIgnored &&
result.Error != nil
})).Return()
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
// The value is not saved
},
},
{
name: "handles v2 dashboard version",
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "v2-dashboard",
},
"status": map[string]interface{}{
"conversion": map[string]interface{}{
"failed": true,
"storedVersion": "v2",
},
},
},
},
},
expectedError: "",
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
progress.On("SetMessage", mock.Anything, "start resource export").Return()
progress.On("SetMessage", mock.Anything, "export dashboards").Return()
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
return result.Name == "v2-dashboard" && result.Action == repository.FileActionCreated
})).Return()
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
// Setup v1 client
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
// Setup v2 client
// Mock v2 client Get call
v2Dashboard := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "dashboard.grafana.app/v2alpha1",
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "v2-dashboard",
},
"spec": map[string]interface{}{
"version": 2,
"title": "V2 Dashboard",
},
},
}
v2Client := &mockDynamicInterface{items: []unstructured.Unstructured{*v2Dashboard}}
resourceClients.On("ForResource", resources.DashboardResourceV2).Return(v2Client, gvk, nil)
options := resources.WriteOptions{
Path: "grafana",
Ref: "feature/branch",
}
repoResources.On("WriteResourceFileFromObject", mock.Anything, v2Dashboard, options).Return("v2-dashboard.json", nil)
},
},
{
name: "handles v2 client creation error",
mockItems: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"apiVersion": resources.DashboardResource.GroupVersion().String(),
"kind": "Dashboard",
"metadata": map[string]interface{}{
"name": "v2-dashboard-error",
},
"status": map[string]interface{}{
"conversion": map[string]interface{}{
"failed": true,
"storedVersion": "v2",
},
},
},
},
},
setupProgress: func(progress *jobs.MockJobProgressRecorder) {
progress.On("SetMessage", mock.Anything, "start resource export").Return()
progress.On("SetMessage", mock.Anything, "export dashboards").Return()
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
if result.Name != "v2-dashboard-error" {
return false
}
if result.Action != repository.FileActionIgnored {
return false
}
if result.Error == nil {
return false
}
if result.Error.Error() != "v2 client error" {
return false
}
return true
})).Return()
progress.On("TooManyErrors").Return(nil)
},
setupResources: func(repoResources *resources.MockRepositoryResources, resourceClients *resources.MockResourceClients, mockClient *mockDynamicInterface, gvk schema.GroupVersionKind) {
resourceClients.On("ForResource", resources.DashboardResource).Return(mockClient, gvk, nil)
resourceClients.On("ForResource", resources.DashboardResourceV2).Return(nil, gvk, fmt.Errorf("v2 client error"))
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
scheme := runtime.NewScheme()
require.NoError(t, metav1.AddMetaToScheme(scheme))
listGVK := schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: "DashboardList",
mockClient := &mockDynamicInterface{
items: tt.mockItems,
}
scheme.AddKnownTypeWithName(listGVK, &metav1.PartialObjectMetadataList{})
scheme.AddKnownTypeWithName(schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: resources.DashboardResource.Resource,
}, &metav1.PartialObjectMetadata{})
fakeDynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{
resources.DashboardResource: listGVK.Kind,
})
resourceClients := resources.NewMockResourceClients(t)
fakeDynamicClient.PrependReactor("list", "dashboards", tt.reactorFunc)
mockProgress := jobs.NewMockJobProgressRecorder(t)
tt.setupProgress(mockProgress)
repoResources := resources.NewMockRepositoryResources(t)
tt.setupResources(repoResources, resourceClients, fakeDynamicClient, listGVK)
tt.setupResources(repoResources, resourceClients, mockClient, schema.GroupVersionKind{
Group: resources.DashboardResource.Group,
Version: resources.DashboardResource.Version,
Kind: "DashboardList",
})
options := provisioningV0.ExportJobOptions{
Path: "grafana",
pkg/registry/apis/provisioning/resources/client.go
+6 -4
View File
@@ -10,7 +10,8 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
dashboardV2 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2alpha1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
iam "github.com/grafana/grafana/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/services/apiserver"
@@ -18,9 +19,10 @@ import (
)
var (
UserResource = iam.UserResourceInfo.GroupVersionResource()
FolderResource = folders.FolderResourceInfo.GroupVersionResource()
DashboardResource = dashboard.DashboardResourceInfo.GroupVersionResource()
UserResource = iam.UserResourceInfo.GroupVersionResource()
FolderResource = folders.FolderResourceInfo.GroupVersionResource()
DashboardResource = dashboardV1.DashboardResourceInfo.GroupVersionResource()
DashboardResourceV2 = dashboardV2.DashboardResourceInfo.GroupVersionResource()
// SupportedProvisioningResources is the list of resources that can fully managed from the UI
SupportedProvisioningResources = []schema.GroupVersionResource{FolderResource, DashboardResource}
pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v0.yaml
+8
View File
@@ -0,0 +1,8 @@
apiVersion: dashboard.grafana.app/v0alpha1
kind: Dashboard
metadata:
name: test-v0
spec:
title: Test dashboard. Created at v0
uid: test-v0 # will be removed by mutation hook
version: 1234567 # will be removed by mutation hook
pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v1.yaml
+9
View File
@@ -0,0 +1,9 @@
apiVersion: dashboard.grafana.app/v1beta1
kind: Dashboard
metadata:
name: test-v1
spec:
title: Test dashboard. Created at v1
uid: test-v1 # will be removed by mutation hook
version: 1234567 # will be removed by mutation hook
schemaVersion: 41
pkg/tests/apis/provisioning/exportunifiedtorepository/dashboard-test-v2.yaml
+10
View File
@@ -0,0 +1,10 @@
apiVersion: dashboard.grafana.app/v2alpha1
kind: Dashboard
metadata:
name: test-v2
spec:
title: Test dashboard. Created at v2
layout:
kind: GridLayout
spec:
items: []
pkg/tests/apis/provisioning/exportunifiedtorepository/root_dashboard.json
-162
View File
@@ -1,162 +0,0 @@
{
"apiVersion": "dashboard.grafana.app/v1beta1",
"kind": "Dashboard",
"metadata": {
"name": "root_dashboard",
"namespace": "default"
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "This is on the root level of the repository.",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-BlPu",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": true,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisGridShow": true,
"axisLabel": "Y axis on the right side",
"axisPlacement": "right",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 7,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "dashed+area"
}
},
"decimals": 3,
"fieldMinMax": false,
"links": [
{
"oneClick": true,
"targetBlank": true,
"title": "Test test",
"url": "https://github.com/grafana/grafana"
}
],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "kvoltamp"
},
"overrides": []
},
"gridPos": {
"h": 24,
"w": 22,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.6.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"queryType": "randomWalk",
"refId": "A"
}
],
"title": "Test panel",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 41,
"tags": [
"grafana",
"dashboard",
"root",
"test"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "Pacific/Kiritimati",
"title": "Test dashboard on root level",
"uid": "",
"version": 0,
"weekStart": "saturday"
}
}
pkg/tests/apis/provisioning/helper_test.go
+22 -6
View File
@@ -23,7 +23,9 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1"
dashboardV1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
dashboardV2 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2alpha1"
folder "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
@@ -46,7 +48,9 @@ type provisioningTestHelper struct {
Repositories *apis.K8sResourceClient
Jobs *apis.K8sResourceClient
Folders *apis.K8sResourceClient
Dashboards *apis.K8sResourceClient
DashboardsV0 *apis.K8sResourceClient
DashboardsV1 *apis.K8sResourceClient
DashboardsV2 *apis.K8sResourceClient
AdminREST *rest.RESTClient
EditorREST *rest.RESTClient
ViewerREST *rest.RESTClient
@@ -250,10 +254,20 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper
Namespace: "default", // actually org1
GVR: folder.FolderResourceInfo.GroupVersionResource(),
})
dashboards := helper.GetResourceClient(apis.ResourceClientArgs{
dashboardsV0 := helper.GetResourceClient(apis.ResourceClientArgs{
User: helper.Org1.Admin,
Namespace: "default", // actually org1
GVR: dashboard.DashboardResourceInfo.GroupVersionResource(),
GVR: dashboardV0.DashboardResourceInfo.GroupVersionResource(),
})
dashboardsV1 := helper.GetResourceClient(apis.ResourceClientArgs{
User: helper.Org1.Admin,
Namespace: "default", // actually org1
GVR: dashboardV1.DashboardResourceInfo.GroupVersionResource(),
})
dashboardsV2 := helper.GetResourceClient(apis.ResourceClientArgs{
User: helper.Org1.Admin,
Namespace: "default", // actually org1
GVR: dashboardV2.DashboardResourceInfo.GroupVersionResource(),
})
// Repo client, but less guard rails. Useful for subresources. We'll need this later...
@@ -276,7 +290,7 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper
return nil
}
require.NoError(t, deleteAll(dashboards), "deleting all dashboards")
require.NoError(t, deleteAll(dashboardsV1), "deleting all dashboards") // v0+v1+v2
require.NoError(t, deleteAll(folders), "deleting all folders")
require.NoError(t, deleteAll(repositories), "deleting all repositories")
@@ -290,7 +304,9 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper
ViewerREST: viewerClient,
Jobs: jobs,
Folders: folders,
Dashboards: dashboards,
DashboardsV0: dashboardsV0,
DashboardsV1: dashboardsV1,
DashboardsV2: dashboardsV2,
}
}
pkg/tests/apis/provisioning/provisioning_test.go
+57 -18
View File
@@ -147,7 +147,7 @@ func TestIntegrationProvisioning_FailInvalidSchema(t *testing.T) {
require.Equal(t, status.Message, "Dry run failed: Dashboard.dashboard.grafana.app \"invalid-schema-uid\" is invalid: [spec.panels.0.repeatDirection: Invalid value: conflicting values \"h\" and \"this is not an allowed value\", spec.panels.0.repeatDirection: Invalid value: conflicting values \"v\" and \"this is not an allowed value\"]")
const invalidSchemaUid = "invalid-schema-uid"
_, err = helper.Dashboards.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{})
_, err = helper.DashboardsV1.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{})
require.Error(t, err, "invalid dashboard shouldn't exist")
require.True(t, apierrors.IsNotFound(err))
@@ -195,7 +195,7 @@ func TestIntegrationProvisioning_FailInvalidSchema(t *testing.T) {
require.Equal(t, job.Status.Errors[0], "Dashboard.dashboard.grafana.app \"invalid-schema-uid\" is invalid: [spec.panels.0.repeatDirection: Invalid value: conflicting values \"h\" and \"this is not an allowed value\", spec.panels.0.repeatDirection: Invalid value: conflicting values \"v\" and \"this is not an allowed value\"]")
}, time.Second*10, time.Millisecond*10, "Expected provisioning job to conclude with the status failed")
_, err = helper.Dashboards.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{})
_, err = helper.DashboardsV1.Resource.Get(ctx, invalidSchemaUid, metav1.GetOptions{})
require.Error(t, err, "invalid dashboard shouldn't have been created")
require.True(t, apierrors.IsNotFound(err))
@@ -272,7 +272,7 @@ func TestIntegrationProvisioning_CreatingGitHubRepository(t *testing.T) {
// By now, we should have synced, meaning we have data to read in the local Grafana instance!
found, err := helper.Dashboards.Resource.List(ctx, metav1.ListOptions{})
found, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err, "can list values")
names := []string{}
@@ -286,7 +286,7 @@ func TestIntegrationProvisioning_CreatingGitHubRepository(t *testing.T) {
require.NoError(t, err, "should delete values")
assert.EventuallyWithT(t, func(collect *assert.CollectT) {
found, err := helper.Dashboards.Resource.List(ctx, metav1.ListOptions{})
found, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err, "can list values")
require.Equal(collect, 0, len(found.Items), "expected dashboards to be deleted")
}, time.Second*10, time.Millisecond*10, "Expected dashboards to be deleted")
@@ -396,7 +396,7 @@ func TestIntegrationProvisioning_RunLocalRepository(t *testing.T) {
require.Equal(t, allPanels, name, "save the name from the request")
// Get the file from the grafana database
obj, err := helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{})
obj, err := helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{})
require.NoError(t, err, "the value should be saved in grafana")
val, _, _ := unstructured.NestedString(obj.Object, "metadata", "annotations", utils.AnnoKeyManagerKind)
require.Equal(t, string(utils.ManagerKindRepo), val, "should have repo annotations")
@@ -539,33 +539,33 @@ func TestIntegrationProvisioning_ImportAllPanelsFromLocalRepository(t *testing.T
// But the dashboard shouldn't exist yet
const allPanels = "n1jR8vnnz"
_, err = helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{})
_, err = helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{})
require.Error(t, err, "no all-panels dashboard should exist")
require.True(t, apierrors.IsNotFound(err))
// Now, we import it, such that it may exist
helper.SyncAndWait(t, repo, nil)
_, err = helper.Dashboards.Resource.List(ctx, metav1.ListOptions{})
_, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err, "can list values")
obj, err = helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{})
obj, err = helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{})
require.NoError(t, err, "all-panels dashboard should exist")
require.Equal(t, repo, obj.GetAnnotations()[utils.AnnoKeyManagerIdentity])
// Try writing the value directly
err = unstructured.SetNestedField(obj.Object, []any{"aaa", "bbb"}, "spec", "tags")
require.NoError(t, err, "set tags")
obj, err = helper.Dashboards.Resource.Update(ctx, obj, metav1.UpdateOptions{})
obj, err = helper.DashboardsV1.Resource.Update(ctx, obj, metav1.UpdateOptions{})
require.NoError(t, err)
v, _, _ := unstructured.NestedString(obj.Object, "metadata", "annotations", utils.AnnoKeyUpdatedBy)
require.Equal(t, "access-policy:provisioning", v)
// Should not be able to directly delete the managed resource
err = helper.Dashboards.Resource.Delete(ctx, allPanels, metav1.DeleteOptions{})
err = helper.DashboardsV1.Resource.Delete(ctx, allPanels, metav1.DeleteOptions{})
require.NoError(t, err, "user can delete")
_, err = helper.Dashboards.Resource.Get(ctx, allPanels, metav1.GetOptions{})
_, err = helper.DashboardsV1.Resource.Get(ctx, allPanels, metav1.GetOptions{})
require.Error(t, err, "should delete the internal resource")
require.True(t, apierrors.IsNotFound(err))
}
@@ -578,10 +578,18 @@ func TestProvisioning_ExportUnifiedToRepository(t *testing.T) {
helper := runGrafana(t)
ctx := context.Background()
// Set up dashboards first, then the repository, and finally export.
dashboard := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/root_dashboard.json")
_, err := helper.Dashboards.Resource.Create(ctx, dashboard, metav1.CreateOptions{})
require.NoError(t, err, "should be able to create prerequisite dashboard")
// Write dashboards at
dashboard := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v0.yaml")
_, err := helper.DashboardsV0.Resource.Create(ctx, dashboard, metav1.CreateOptions{})
require.NoError(t, err, "should be able to create v0 dashboard")
dashboard = helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v1.yaml")
_, err = helper.DashboardsV1.Resource.Create(ctx, dashboard, metav1.CreateOptions{})
require.NoError(t, err, "should be able to create v1 dashboard")
dashboard = helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v2.yaml")
_, err = helper.DashboardsV2.Resource.Create(ctx, dashboard, metav1.CreateOptions{})
require.NoError(t, err, "should be able to create v2 dashboard")
// Now for the repository.
const repo = "local-repository"
@@ -608,7 +616,38 @@ func TestProvisioning_ExportUnifiedToRepository(t *testing.T) {
// And time to assert.
helper.AwaitJobs(t, repo)
fpath := filepath.Join(helper.ProvisioningPath, slugify.Slugify(mustNestedString(dashboard.Object, "spec", "title"))+".json")
_, err = os.Stat(fpath)
require.NoError(t, err, "exported file was not created at path %s", fpath)
type props struct {
title string
apiVersion string
name string
}
// Check that each file was exported with its stored version
for _, test := range []props{
{title: "Test dashboard. Created at v0", apiVersion: "dashboard.grafana.app/v0alpha1", name: "test-v0"},
{title: "Test dashboard. Created at v1", apiVersion: "dashboard.grafana.app/v1beta1", name: "test-v1"},
{title: "Test dashboard. Created at v2", apiVersion: "dashboard.grafana.app/v2alpha1", name: "test-v2"},
} {
fpath := filepath.Join(helper.ProvisioningPath, slugify.Slugify(test.title)+".json")
//nolint:gosec // we are ok with reading files in testdata
body, err := os.ReadFile(fpath)
require.NoError(t, err, "exported file was not created at path %s", fpath)
obj := map[string]any{}
err = json.Unmarshal(body, &obj)
require.NoError(t, err, "exported file not json %s", fpath)
val, _, err := unstructured.NestedString(obj, "apiVersion")
require.NoError(t, err)
require.Equal(t, test.apiVersion, val)
val, _, err = unstructured.NestedString(obj, "spec", "title")
require.NoError(t, err)
require.Equal(t, test.title, val)
val, _, err = unstructured.NestedString(obj, "metadata", "name")
require.NoError(t, err)
require.Equal(t, test.name, val)
require.Nil(t, obj["status"], "should not have a status element")
}
}