Provisioning: Add bulk delete job (#108580)
* Add delete job type * Regenerate spec * Add first implementation of worker * Move interface and mock to repository package * Add unit tests * Add integration tests * Fix linting and spec * Regenerate client * Format file * go fmt * fix --------- Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>
This commit is contained in:
Roberto Jiménez Sánchez
co-authored by
Stephanie Hingtgen
Stephanie Hingtgen
parent
a82a7f1340
commit
20cea80795
@@ -2631,6 +2631,23 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.DeleteJobOptions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"paths": {
|
||||
"description": "Paths to be deleted. Examples: - dashboard.json (for a file) - a/b/c/other-dashboard.json (for a file) - nested/deep/ (for a directory) FIXME: we should validate this in admission hooks",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"default": ""
|
||||
}
|
||||
},
|
||||
"ref": {
|
||||
"description": "Ref to the branch or commit hash to delete from",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.ErrorDetails": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -2653,7 +2670,7 @@
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"branch": {
|
||||
"description": "Target branch for export (only git)",
|
||||
"description": "FIXME: we should validate this in admission hooks Target branch for export (only git)",
|
||||
"type": "string"
|
||||
},
|
||||
"folder": {
|
||||
@@ -2665,7 +2682,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"path": {
|
||||
"description": "Prefix in target file system",
|
||||
"description": "FIXME: we should validate this in admission hooks Prefix in target file system",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
@@ -3049,15 +3066,24 @@
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"description": "Possible enum values:\n - `\"migrate\"` acts like JobActionExport, then JobActionPull. It also tries to preserve the history.\n - `\"pr\"` adds additional useful information to a PR, such as comments with preview links and rendered images.\n - `\"pull\"` replicates the remote branch in the local copy of the repository.\n - `\"push\"` replicates the local copy of the repository in the remote branch.",
|
||||
"description": "Possible enum values:\n - `\"delete\"` deletes files in the remote repository\n - `\"migrate\"` acts like JobActionExport, then JobActionPull. It also tries to preserve the history.\n - `\"pr\"` adds additional useful information to a PR, such as comments with preview links and rendered images.\n - `\"pull\"` replicates the remote branch in the local copy of the repository.\n - `\"push\"` replicates the local copy of the repository in the remote branch.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"delete",
|
||||
"migrate",
|
||||
"pr",
|
||||
"pull",
|
||||
"push"
|
||||
]
|
||||
},
|
||||
"delete": {
|
||||
"description": "Delete when the action is `delete`",
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/com.github.grafana.grafana.pkg.apis.provisioning.v0alpha1.DeleteJobOptions"
|
||||
}
|
||||
]
|
||||
},
|
||||
"migrate": {
|
||||
"description": "Required when the action is `migrate`",
|
||||
"allOf": [
|
||||
|
||||
@@ -822,7 +822,7 @@ func TestIntegrationProvisioning_DeleteResources(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegrationProvisioning_MoveResources(t *testing.T) {
|
||||
func TestIntegrationProvisioning_DeleteJob(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test")
|
||||
}
|
||||
@@ -830,6 +830,158 @@ func TestIntegrationProvisioning_MoveResources(t *testing.T) {
|
||||
helper := runGrafana(t)
|
||||
ctx := context.Background()
|
||||
|
||||
const repo = "delete-job-test-repo"
|
||||
localTmp := helper.RenderObject(t, "testdata/local-write.json.tmpl", map[string]any{
|
||||
"Name": repo,
|
||||
"SyncEnabled": true,
|
||||
"SyncTarget": "instance",
|
||||
})
|
||||
_, err := helper.Repositories.Resource.Create(ctx, localTmp, metav1.CreateOptions{})
|
||||
require.NoError(t, err)
|
||||
// Copy multiple test files to the repository
|
||||
helper.CopyToProvisioningPath(t, "testdata/all-panels.json", "dashboard1.json")
|
||||
helper.CopyToProvisioningPath(t, "testdata/text-options.json", "dashboard2.json")
|
||||
helper.CopyToProvisioningPath(t, "testdata/timeline-demo.json", "folder/dashboard3.json")
|
||||
|
||||
// Trigger and wait for initial sync to populate resources
|
||||
helper.SyncAndWait(t, repo, nil)
|
||||
|
||||
// Verify initial state - should have 3 dashboards and 1 folder
|
||||
dashboards, err := helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, len(dashboards.Items), "should have 3 dashboards after sync")
|
||||
|
||||
folders, err := helper.Folders.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(folders.Items), "should have 1 folder after sync")
|
||||
|
||||
t.Run("delete single file", func(t *testing.T) {
|
||||
// Create delete job for single file
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Paths: []string{"dashboard1.json"},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify file is deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "dashboard1.json")
|
||||
require.Error(t, err, "file should be deleted from repository")
|
||||
require.True(t, apierrors.IsNotFound(err), "should be not found error")
|
||||
|
||||
// Verify dashboard is removed from Grafana after sync
|
||||
dashboards, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(dashboards.Items), "should have 2 dashboards after delete")
|
||||
|
||||
// Verify other files still exist
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "dashboard2.json")
|
||||
require.NoError(t, err, "other files should still exist")
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "folder", "dashboard3.json")
|
||||
require.NoError(t, err, "nested files should still exist")
|
||||
})
|
||||
|
||||
t.Run("delete multiple files", func(t *testing.T) {
|
||||
// Create delete job for multiple files
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Paths: []string{"dashboard2.json", "folder/dashboard3.json"},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job")
|
||||
|
||||
// Wait for job to complete
|
||||
helper.AwaitJobs(t, repo)
|
||||
|
||||
// Verify files are deleted from repository
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "dashboard2.json")
|
||||
require.Error(t, err, "dashboard2.json should be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
_, err = helper.Repositories.Resource.Get(ctx, repo, metav1.GetOptions{}, "files", "folder", "dashboard3.json")
|
||||
require.Error(t, err, "folder/dashboard3.json should be deleted")
|
||||
require.True(t, apierrors.IsNotFound(err))
|
||||
|
||||
// Verify all dashboards are removed from Grafana after sync
|
||||
dashboards, err = helper.DashboardsV1.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(dashboards.Items), "should have 0 dashboards after deleting all")
|
||||
})
|
||||
|
||||
t.Run("delete non-existent file", func(t *testing.T) {
|
||||
// Create delete job for non-existent file
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Action: provisioning.JobActionDelete,
|
||||
Delete: &provisioning.DeleteJobOptions{
|
||||
Paths: []string{"non-existent.json"},
|
||||
},
|
||||
})).
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create delete job")
|
||||
|
||||
// Wait for job to complete - should fail due to strict error handling
|
||||
require.EventuallyWithT(t, func(collect *assert.CollectT) {
|
||||
list := &unstructured.UnstructuredList{}
|
||||
err := helper.AdminREST.Get().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo).
|
||||
SubResource("jobs").
|
||||
Do(ctx).Into(list)
|
||||
assert.NoError(collect, err, "should be able to list jobs")
|
||||
assert.NotEmpty(collect, list.Items, "expect at least one job")
|
||||
|
||||
// Find the delete job specifically
|
||||
var deleteJob *unstructured.Unstructured
|
||||
for _, elem := range list.Items {
|
||||
assert.Equal(collect, repo, elem.GetLabels()["provisioning.grafana.app/repository"], "should have repo label")
|
||||
|
||||
action := mustNestedString(elem.Object, "spec", "action")
|
||||
if action == "delete" {
|
||||
deleteJob = &elem
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.NotNil(collect, deleteJob, "should find a delete job")
|
||||
|
||||
state := mustNestedString(deleteJob.Object, "status", "state")
|
||||
assert.Equal(collect, "error", state, "delete job should have failed due to non-existent file")
|
||||
}, time.Second*10, time.Millisecond*100, "Expected delete job to fail with error state")
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegrationProvisioning_MoveResources(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test")
|
||||
}
|
||||
|
||||
helper := runGrafana(t)
|
||||
ctx := context.Background()
|
||||
const repo = "move-test-repo"
|
||||
localTmp := helper.RenderObject(t, "testdata/local-write.json.tmpl", map[string]any{
|
||||
"Name": repo,
|
||||
|
||||
Reference in New Issue
Block a user