Provisioning: bulk delete resources by name (#108833)
* Add resource reference to spec * Add FindResourcePath * Fix formatting * Use ForKind client * Add unit test for new method * Format code * Add integration tests * Fix unit tests * Fix formatting * Find out preferred version based on group and kind * Handle trailing slash for folders * Format code * Fix linting * Add integration test for folder bulk deletion * Format code * Format discovery file * Deduplicate paths for deletion
This commit is contained in:
Roberto Jiménez Sánchez
@@ -927,6 +927,340 @@ func TestIntegrationProvisioning_DeleteJob(t *testing.T) {
|
||||
require.Equal(t, 0, len(dashboards.Items), "should have 0 dashboards after deleting all")
|
||||
})
|
||||
|
||||
t.Run("delete by resource reference", func(t *testing.T) {
|
||||
// Create modified test files with unique UIDs for ResourceRef testing
|
||||
// Read and modify the testdata files to have unique UIDs that don't conflict with existing resources
|
||||
allPanelsContent := helper.LoadFile("testdata/all-panels.json")
|
||||
textOptionsContent := helper.LoadFile("testdata/text-options.json")
|
||||
timelineDemoContent := helper.LoadFile("testdata/timeline-demo.json")
|
||||
|
||||
// Modify UIDs to be unique for ResourceRef tests
|
||||
allPanelsModified := strings.Replace(string(allPanelsContent), `"uid": "n1jR8vnnz"`, `"uid": "resourceref1"`, 1)
|
||||
textOptionsModified := strings.Replace(string(textOptionsContent), `"uid": "WZ7AhQiVz"`, `"uid": "resourceref2"`, 1)
|
||||
timelineDemoModified := strings.Replace(string(timelineDemoContent), `"uid": "mIJjFy8Kz"`, `"uid": "resourceref3"`, 1)
|
||||
|
||||
// Create temporary files and copy them to the provisioning path
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile1 := filepath.Join(tmpDir, "resource-test-1.json")
|
||||
tmpFile2 := filepath.Join(tmpDir, "resource-test-2.json")
|
||||
tmpFile3 := filepath.Join(tmpDir, "resource-test-3.json")
|
||||
|
||||
require.NoError(t, os.WriteFile(tmpFile1, []byte(allPanelsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpFile2, []byte(textOptionsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpFile3, []byte(timelineDemoModified), 0644))
|
||||
|
||||
// Copy the temporary files to the provisioning path
|
||||
helper.CopyToProvisioningPath(t, tmpFile1, "resource-test-1.json") // UID: resourceref1
|
||||
helper.CopyToProvisioningPath(t, tmpFile2, "resource-test-2.json") // UID: resourceref2
|
||||
helper.CopyToProvisioningPath(t, tmpFile3, "nested/resource-test-3.json") // UID: resourceref3
|
||||
|
||||
// Trigger sync to populate the new resources
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Verify the new resources are created
|
||||
dashboards, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(dashboards.Items), 3, "should have at least 3 dashboards after adding test resources")
|
||||
|
||||
// Debug: print the actual dashboard names/UIDs to verify they match our expectations
|
||||
for i, dashboard := range dashboards.Items {
|
||||
t.Logf("Dashboard %d: name=%s, UID=%s", i+1, dashboard.GetName(), dashboard.GetUID())
|
||||
}
|
||||
|
||||
t.Run("delete single dashboard by resource reference", func(t *testing.T) {
|
||||
// Create delete job for single dashboard using ResourceRef
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "resourceref1", // UID from modified all-panels.json
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job with ResourceRef")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify corresponding file is deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "resource-test-1.json")
|
||||
require.Error(t, err, "file should be deleted from repository")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
|
||||
// Verify dashboard is removed from Grafana (check count like other successful tests)
|
||||
dashboards, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(dashboards.Items), "should have 2 dashboards after deleting 1 from 3")
|
||||
|
||||
// Verify other resources still exist
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "resource-test-2.json")
|
||||
require.NoError(t, err, "other files should still exist")
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "nested", "resource-test-3.json")
|
||||
require.NoError(t, err, "nested files should still exist")
|
||||
})
|
||||
|
||||
t.Run("delete multiple resources by reference", func(t *testing.T) {
|
||||
// Create delete job for multiple resources using ResourceRef
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "resourceref2", // UID from modified text-options.json
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
{
|
||||
Name: "resourceref3", // UID from modified timeline-demo.json
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job with multiple ResourceRefs")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify both dashboards are removed from Grafana
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref2", metav1.GetOptions{})
|
||||
require.Error(t, err, "text-options dashboard should be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref3", metav1.GetOptions{})
|
||||
require.Error(t, err, "timeline-demo dashboard should be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
// Verify corresponding files are deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "resource-test-2.json")
|
||||
require.Error(t, err, "resource-test-2.json should be deleted")
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "nested", "resource-test-3.json")
|
||||
require.Error(t, err, "nested/resource-test-3.json should be deleted")
|
||||
|
||||
// Verify specific dashboards are removed from Grafana
|
||||
dashboards, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(dashboards.Items), "should have 0 dashboards after deleting 2 more (2 -> 0)")
|
||||
})
|
||||
|
||||
t.Run("mixed deletion - paths and resources", func(t *testing.T) {
|
||||
// Setup fresh resources for mixed test - reuse the modified content with unique UIDs
|
||||
tmpMixed1 := filepath.Join(tmpDir, "mixed-test-1.json")
|
||||
tmpMixed2 := filepath.Join(tmpDir, "mixed-test-2.json")
|
||||
tmpMixed3 := filepath.Join(tmpDir, "mixed-test-3.json")
|
||||
|
||||
require.NoError(t, os.WriteFile(tmpMixed1, []byte(allPanelsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpMixed2, []byte(textOptionsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpMixed3, []byte(timelineDemoModified), 0644))
|
||||
|
||||
helper.CopyToProvisioningPath(t, tmpMixed1, "mixed-test-1.json") // UID: resourceref1
|
||||
helper.CopyToProvisioningPath(t, tmpMixed2, "mixed-test-2.json") // UID: resourceref2
|
||||
helper.CopyToProvisioningPath(t, tmpMixed3, "mixed-test-3.json") // UID: resourceref3
|
||||
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Create delete job that combines both paths and resource references
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Paths: []string{"mixed-test-1.json"}, // Delete by path
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "resourceref2", // Delete by resource reference
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create mixed delete job")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify both targeted resources are deleted from Grafana
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref1", metav1.GetOptions{})
|
||||
require.Error(t, err, "dashboard deleted by path should be removed")
|
||||
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref2", metav1.GetOptions{})
|
||||
require.Error(t, err, "dashboard deleted by resource ref should be removed")
|
||||
|
||||
// Verify the untargeted resource still exists
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref3", metav1.GetOptions{})
|
||||
require.NoError(t, err, "untargeted dashboard should still exist")
|
||||
|
||||
// Verify files are properly deleted/preserved in repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "mixed-test-1.json")
|
||||
require.Error(t, err, "file deleted by path should be removed")
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "mixed-test-2.json")
|
||||
require.Error(t, err, "file for resource deleted by ref should be removed")
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "mixed-test-3.json")
|
||||
require.NoError(t, err, "untargeted file should still exist")
|
||||
})
|
||||
|
||||
t.Run("delete folder by resource reference", func(t *testing.T) {
|
||||
// Create a dashboard inside a folder to automatically create the folder structure
|
||||
// This follows the same pattern as other tests in this file
|
||||
testDashboard := strings.Replace(string(allPanelsContent), `"uid": "n1jR8vnnz"`, `"uid": "folder-dash"`, 1)
|
||||
|
||||
// Write the modified dashboard to a temporary file first
|
||||
tmpFolderDash := filepath.Join(tmpDir, "folder-dashboard.json")
|
||||
require.NoError(t, os.WriteFile(tmpFolderDash, []byte(testDashboard), 0644))
|
||||
|
||||
// Copy it to the folder structure using the helper
|
||||
helper.CopyToProvisioningPath(t, tmpFolderDash, "test-folder/dashboard-in-folder.json")
|
||||
|
||||
// Sync to create the folder and its contents
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Verify folder was created in Grafana as a Folder resource
|
||||
folders, err := helper.Folders.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
var testFolder *unstructured.Unstructured
|
||||
for _, folder := range folders.Items {
|
||||
// Folder names are generated with suffixes, so check if it starts with "test-folder"
|
||||
if strings.HasPrefix(folder.GetName(), "test-folder") {
|
||||
testFolder = &folder
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, testFolder, "test-folder should exist as a Folder resource")
|
||||
testFolderName := testFolder.GetName()
|
||||
|
||||
// Verify dashboard inside the folder exists
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "test-folder", "dashboard-in-folder.json")
|
||||
require.NoError(t, err, "dashboard inside folder should exist")
|
||||
|
||||
// Create delete job for the folder using ResourceRef (use the actual generated name)
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: testFolderName, // Use the actual generated folder name
|
||||
Kind: "Folder",
|
||||
Group: "folder.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job for folder")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify folder is deleted from Grafana
|
||||
_, err = helper.Folders.Resource.Get(ctx, testFolderName, metav1.GetOptions{})
|
||||
require.Error(t, err, "folder should be deleted from Grafana")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
|
||||
// Verify folder contents are also deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "test-folder", "dashboard-in-folder.json")
|
||||
require.Error(t, err, "dashboard inside deleted folder should also be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
})
|
||||
|
||||
t.Run("delete non-existent resource by reference", func(t *testing.T) {
|
||||
// Create delete job for non-existent resource
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "non-existent-uid",
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job")
|
||||
|
||||
// Wait for job to complete - should record error but continue
|
||||
require.EventuallyWithT(t, func(collect *assert.CollectT) {
|
||||
list := &unstructured.UnstructuredList{}
|
||||
err := helper.AdminREST.Get().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Do(ctx).Into(list)
|
||||
assert.NoError(collect, err, "should be able to list jobs")
|
||||
assert.NotEmpty(collect, list.Items, "expect at least one job")
|
||||
|
||||
// Find the most recent delete job
|
||||
var deleteJob *unstructured.Unstructured
|
||||
for _, elem := range list.Items {
|
||||
assert.Equal(collect, repo, elem.GetLabels()["provisioning.grafana.app/repository"], "should have repo label")
|
||||
|
||||
action := mustNestedString(elem.Object, "spec", "action")
|
||||
if action == "delete" {
|
||||
// Get the most recent one (they should be ordered by creation time)
|
||||
deleteJob = &elem
|
||||
}
|
||||
}
|
||||
if !assert.NotNil(collect, deleteJob, "should find a delete job") {
|
||||
return
|
||||
}
|
||||
|
||||
state := mustNestedString(deleteJob.Object, "status", "state")
|
||||
// The job should complete but record errors for individual resource resolution failures
|
||||
if state == "error" || state == "completed" || state == "success" {
|
||||
// Any of these states is acceptable - the key is that resource resolution errors are recorded
|
||||
// and don't fail the entire job due to error-tolerant implementation
|
||||
return
|
||||
}
|
||||
assert.Fail(collect, "job should complete or error, but got state: %s", state)
|
||||
}, time.Second*10, time.Millisecond*100, "Expected delete job to handle non-existent resource")
|
||||
})
|
||||
|
||||
// Repository cleanup is handled by the main test function
|
||||
})
|
||||
|
||||
t.Run("delete non-existent file", func(t *testing.T) {
|
||||
// Create delete job for non-existent file
|
||||
result := helper.AdminREST.Post().
|
||||
|
||||
Reference in New Issue
Block a user