Provisioning: bulk delete resources by name (#108833)
* Add resource reference to spec * Add FindResourcePath * Fix formatting * Use ForKind client * Add unit test for new method * Format code * Add integration tests * Fix unit tests * Fix formatting * Find out preferred version based on group and kind * Handle trailing slash for folders * Format code * Fix linting * Add integration test for folder bulk deletion * Format code * Format discovery file * Deduplicate paths for deletion
This commit is contained in:
Roberto Jiménez Sánchez
@@ -2645,6 +2645,18 @@
|
||||
"ref": {
|
||||
"description": "Ref to the branch or commit hash to delete from",
|
||||
"type": "string"
|
||||
},
|
||||
"resources": {
|
||||
"description": "Resources to delete This option has been created because currently the frontend does not use standarized app platform APIs. For performance and API consistency reasons, the preferred option is it to use the paths.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"default": {},
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ResourceRef"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -3816,6 +3828,23 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ResourceRef": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"group": {
|
||||
"description": "Group is the group of the resource, such as \"dashboard.grafana.app\".",
|
||||
"type": "string"
|
||||
},
|
||||
"kind": {
|
||||
"description": "Kind is the type of resource, for example, \"Dashboard\".",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"description": "Name is the name of the resource, such as a dashboard UID.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ResourceRepositoryInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
|
||||
@@ -927,6 +927,340 @@ func TestIntegrationProvisioning_DeleteJob(t *testing.T) {
|
||||
require.Equal(t, 0, len(dashboards.Items), "should have 0 dashboards after deleting all")
|
||||
})
|
||||
|
||||
t.Run("delete by resource reference", func(t *testing.T) {
|
||||
// Create modified test files with unique UIDs for ResourceRef testing
|
||||
// Read and modify the testdata files to have unique UIDs that don't conflict with existing resources
|
||||
allPanelsContent := helper.LoadFile("testdata/all-panels.json")
|
||||
textOptionsContent := helper.LoadFile("testdata/text-options.json")
|
||||
timelineDemoContent := helper.LoadFile("testdata/timeline-demo.json")
|
||||
|
||||
// Modify UIDs to be unique for ResourceRef tests
|
||||
allPanelsModified := strings.Replace(string(allPanelsContent), `"uid": "n1jR8vnnz"`, `"uid": "resourceref1"`, 1)
|
||||
textOptionsModified := strings.Replace(string(textOptionsContent), `"uid": "WZ7AhQiVz"`, `"uid": "resourceref2"`, 1)
|
||||
timelineDemoModified := strings.Replace(string(timelineDemoContent), `"uid": "mIJjFy8Kz"`, `"uid": "resourceref3"`, 1)
|
||||
|
||||
// Create temporary files and copy them to the provisioning path
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile1 := filepath.Join(tmpDir, "resource-test-1.json")
|
||||
tmpFile2 := filepath.Join(tmpDir, "resource-test-2.json")
|
||||
tmpFile3 := filepath.Join(tmpDir, "resource-test-3.json")
|
||||
|
||||
require.NoError(t, os.WriteFile(tmpFile1, []byte(allPanelsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpFile2, []byte(textOptionsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpFile3, []byte(timelineDemoModified), 0644))
|
||||
|
||||
// Copy the temporary files to the provisioning path
|
||||
helper.CopyToProvisioningPath(t, tmpFile1, "resource-test-1.json") // UID: resourceref1
|
||||
helper.CopyToProvisioningPath(t, tmpFile2, "resource-test-2.json") // UID: resourceref2
|
||||
helper.CopyToProvisioningPath(t, tmpFile3, "nested/resource-test-3.json") // UID: resourceref3
|
||||
|
||||
// Trigger sync to populate the new resources
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Verify the new resources are created
|
||||
dashboards, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(dashboards.Items), 3, "should have at least 3 dashboards after adding test resources")
|
||||
|
||||
// Debug: print the actual dashboard names/UIDs to verify they match our expectations
|
||||
for i, dashboard := range dashboards.Items {
|
||||
t.Logf("Dashboard %d: name=%s, UID=%s", i+1, dashboard.GetName(), dashboard.GetUID())
|
||||
}
|
||||
|
||||
t.Run("delete single dashboard by resource reference", func(t *testing.T) {
|
||||
// Create delete job for single dashboard using ResourceRef
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "resourceref1", // UID from modified all-panels.json
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job with ResourceRef")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify corresponding file is deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "resource-test-1.json")
|
||||
require.Error(t, err, "file should be deleted from repository")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
|
||||
// Verify dashboard is removed from Grafana (check count like other successful tests)
|
||||
dashboards, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(dashboards.Items), "should have 2 dashboards after deleting 1 from 3")
|
||||
|
||||
// Verify other resources still exist
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "resource-test-2.json")
|
||||
require.NoError(t, err, "other files should still exist")
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "nested", "resource-test-3.json")
|
||||
require.NoError(t, err, "nested files should still exist")
|
||||
})
|
||||
|
||||
t.Run("delete multiple resources by reference", func(t *testing.T) {
|
||||
// Create delete job for multiple resources using ResourceRef
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "resourceref2", // UID from modified text-options.json
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
{
|
||||
Name: "resourceref3", // UID from modified timeline-demo.json
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job with multiple ResourceRefs")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify both dashboards are removed from Grafana
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref2", metav1.GetOptions{})
|
||||
require.Error(t, err, "text-options dashboard should be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref3", metav1.GetOptions{})
|
||||
require.Error(t, err, "timeline-demo dashboard should be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
// Verify corresponding files are deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "resource-test-2.json")
|
||||
require.Error(t, err, "resource-test-2.json should be deleted")
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "nested", "resource-test-3.json")
|
||||
require.Error(t, err, "nested/resource-test-3.json should be deleted")
|
||||
|
||||
// Verify specific dashboards are removed from Grafana
|
||||
dashboards, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(dashboards.Items), "should have 0 dashboards after deleting 2 more (2 -> 0)")
|
||||
})
|
||||
|
||||
t.Run("mixed deletion - paths and resources", func(t *testing.T) {
|
||||
// Setup fresh resources for mixed test - reuse the modified content with unique UIDs
|
||||
tmpMixed1 := filepath.Join(tmpDir, "mixed-test-1.json")
|
||||
tmpMixed2 := filepath.Join(tmpDir, "mixed-test-2.json")
|
||||
tmpMixed3 := filepath.Join(tmpDir, "mixed-test-3.json")
|
||||
|
||||
require.NoError(t, os.WriteFile(tmpMixed1, []byte(allPanelsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpMixed2, []byte(textOptionsModified), 0644))
|
||||
require.NoError(t, os.WriteFile(tmpMixed3, []byte(timelineDemoModified), 0644))
|
||||
|
||||
helper.CopyToProvisioningPath(t, tmpMixed1, "mixed-test-1.json") // UID: resourceref1
|
||||
helper.CopyToProvisioningPath(t, tmpMixed2, "mixed-test-2.json") // UID: resourceref2
|
||||
helper.CopyToProvisioningPath(t, tmpMixed3, "mixed-test-3.json") // UID: resourceref3
|
||||
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Create delete job that combines both paths and resource references
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Paths: []string{"mixed-test-1.json"}, // Delete by path
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "resourceref2", // Delete by resource reference
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create mixed delete job")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify both targeted resources are deleted from Grafana
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref1", metav1.GetOptions{})
|
||||
require.Error(t, err, "dashboard deleted by path should be removed")
|
||||
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref2", metav1.GetOptions{})
|
||||
require.Error(t, err, "dashboard deleted by resource ref should be removed")
|
||||
|
||||
// Verify the untargeted resource still exists
|
||||
_, err = helper.DashboardsV1.Resource.Get(ctx, "resourceref3", metav1.GetOptions{})
|
||||
require.NoError(t, err, "untargeted dashboard should still exist")
|
||||
|
||||
// Verify files are properly deleted/preserved in repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "mixed-test-1.json")
|
||||
require.Error(t, err, "file deleted by path should be removed")
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "mixed-test-2.json")
|
||||
require.Error(t, err, "file for resource deleted by ref should be removed")
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "mixed-test-3.json")
|
||||
require.NoError(t, err, "untargeted file should still exist")
|
||||
})
|
||||
|
||||
t.Run("delete folder by resource reference", func(t *testing.T) {
|
||||
// Create a dashboard inside a folder to automatically create the folder structure
|
||||
// This follows the same pattern as other tests in this file
|
||||
testDashboard := strings.Replace(string(allPanelsContent), `"uid": "n1jR8vnnz"`, `"uid": "folder-dash"`, 1)
|
||||
|
||||
// Write the modified dashboard to a temporary file first
|
||||
tmpFolderDash := filepath.Join(tmpDir, "folder-dashboard.json")
|
||||
require.NoError(t, os.WriteFile(tmpFolderDash, []byte(testDashboard), 0644))
|
||||
|
||||
// Copy it to the folder structure using the helper
|
||||
helper.CopyToProvisioningPath(t, tmpFolderDash, "test-folder/dashboard-in-folder.json")
|
||||
|
||||
// Sync to create the folder and its contents
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Verify folder was created in Grafana as a Folder resource
|
||||
folders, err := helper.Folders.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
var testFolder *unstructured.Unstructured
|
||||
for _, folder := range folders.Items {
|
||||
// Folder names are generated with suffixes, so check if it starts with "test-folder"
|
||||
if strings.HasPrefix(folder.GetName(), "test-folder") {
|
||||
testFolder = &folder
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, testFolder, "test-folder should exist as a Folder resource")
|
||||
testFolderName := testFolder.GetName()
|
||||
|
||||
// Verify dashboard inside the folder exists
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "test-folder", "dashboard-in-folder.json")
|
||||
require.NoError(t, err, "dashboard inside folder should exist")
|
||||
|
||||
// Create delete job for the folder using ResourceRef (use the actual generated name)
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: testFolderName, // Use the actual generated folder name
|
||||
Kind: "Folder",
|
||||
Group: "folder.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job for folder")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify folder is deleted from Grafana
|
||||
_, err = helper.Folders.Resource.Get(ctx, testFolderName, metav1.GetOptions{})
|
||||
require.Error(t, err, "folder should be deleted from Grafana")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
|
||||
// Verify folder contents are also deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "test-folder", "dashboard-in-folder.json")
|
||||
require.Error(t, err, "dashboard inside deleted folder should also be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
})
|
||||
|
||||
t.Run("delete non-existent resource by reference", func(t *testing.T) {
|
||||
// Create delete job for non-existent resource
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Resources: []provisioning.ResourceRef{
|
||||
{
|
||||
Name: "non-existent-uid",
|
||||
Kind: "Dashboard",
|
||||
Group: "dashboard.grafana.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job")
|
||||
|
||||
// Wait for job to complete - should record error but continue
|
||||
require.EventuallyWithT(t, func(collect *assert.CollectT) {
|
||||
list := &unstructured.UnstructuredList{}
|
||||
err := helper.AdminREST.Get().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Do(ctx).Into(list)
|
||||
assert.NoError(collect, err, "should be able to list jobs")
|
||||
assert.NotEmpty(collect, list.Items, "expect at least one job")
|
||||
|
||||
// Find the most recent delete job
|
||||
var deleteJob *unstructured.Unstructured
|
||||
for _, elem := range list.Items {
|
||||
assert.Equal(collect, repo, elem.GetLabels()["provisioning.grafana.app/repository"], "should have repo label")
|
||||
|
||||
action := mustNestedString(elem.Object, "spec", "action")
|
||||
if action == "delete" {
|
||||
// Get the most recent one (they should be ordered by creation time)
|
||||
deleteJob = &elem
|
||||
}
|
||||
}
|
||||
if !assert.NotNil(collect, deleteJob, "should find a delete job") {
|
||||
return
|
||||
}
|
||||
|
||||
state := mustNestedString(deleteJob.Object, "status", "state")
|
||||
// The job should complete but record errors for individual resource resolution failures
|
||||
if state == "error" || state == "completed" || state == "success" {
|
||||
// Any of these states is acceptable - the key is that resource resolution errors are recorded
|
||||
// and don't fail the entire job due to error-tolerant implementation
|
||||
return
|
||||
}
|
||||
assert.Fail(collect, "job should complete or error, but got state: %s", state)
|
||||
}, time.Second*10, time.Millisecond*100, "Expected delete job to handle non-existent resource")
|
||||
})
|
||||
|
||||
// Repository cleanup is handled by the main test function
|
||||
})
|
||||
|
||||
t.Run("delete non-existent file", func(t *testing.T) {
|
||||
// Create delete job for non-existent file
|
||||
result := helper.AdminREST.Post().
|
||||
|
||||
Reference in New Issue
Block a user