Provisioning: skip export of already managed resources and parent folder export (#108893)
* Skip export on already managed resources * Add integration test * Add integration test * Handle nothing to commit error * Fix leaky abstraction issue * Handle the no commit error on commit and not on push * Fix linting * Some fixes for integration test * Improve tree to work with a root * Some fixes with hacks * Add additional checks * Fix comment * Fix path problems in test * Fix more stuff * Revert to use empty tree * Remove changes in tree * Finally fix the tests work * Remove stale comment * Fix linting * Revert changes in test * Fix error message for folder not found in resource tree Co-authored-by: roberto.jimenez <roberto.jimenez@grafana.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Roberto Jiménez Sánchez
co-authored by
Cursor Agent
Cursor Agent
parent
88a52bb6f4
commit
9ca0750134
@@ -112,11 +112,11 @@ func (h *provisioningTestHelper) AwaitJobSuccess(t *testing.T, ctx context.Conte
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
|
||||
errors := mustNestedStringSlice(result.Object, "status", "errors")
|
||||
require.Empty(t, errors, "historic job '%s' has errors: %v", job.GetName(), errors)
|
||||
state := mustNestedString(result.Object, "status", "state")
|
||||
require.Equal(t, string(provisioning.JobStateSuccess), state,
|
||||
"historic job '%s' was not successful", job.GetName())
|
||||
errors := mustNestedStringSlice(result.Object, "status", "errors")
|
||||
require.Empty(t, errors, "historic job '%s' has errors: %v", job.GetName(), errors)
|
||||
}, time.Second*10, time.Millisecond*25) {
|
||||
// We also want to add the job details to the error when it fails.
|
||||
job, err := h.Jobs.Resource.Get(ctx, job.GetName(), metav1.GetOptions{})
|
||||
@@ -163,6 +163,50 @@ func (h *provisioningTestHelper) AwaitJobs(t *testing.T, repoName string) {
|
||||
}
|
||||
}
|
||||
|
||||
// AwaitJobsWithStates waits for all jobs for a repository to complete and accepts multiple valid end states
|
||||
func (h *provisioningTestHelper) AwaitJobsWithStates(t *testing.T, repoName string, acceptedStates []string) {
|
||||
t.Helper()
|
||||
|
||||
// First, we wait for all jobs for the repository to disappear (i.e. complete/fail).
|
||||
require.EventuallyWithT(t, func(collect *assert.CollectT) {
|
||||
list, err := h.Jobs.Resource.List(context.Background(), metav1.ListOptions{})
|
||||
if assert.NoError(collect, err, "failed to list active jobs") {
|
||||
for _, elem := range list.Items {
|
||||
repo, _, err := unstructured.NestedString(elem.Object, "spec", "repository")
|
||||
require.NoError(t, err)
|
||||
if repo == repoName {
|
||||
collect.Errorf("there are still remaining jobs for %s: %+v", repoName, elem)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}, time.Second*10, time.Millisecond*25, "job queue must be empty")
|
||||
|
||||
// Then, as all jobs are now historic jobs, we make sure they are in an accepted state.
|
||||
result, err := h.Repositories.Resource.Get(context.Background(), repoName, metav1.GetOptions{}, "jobs")
|
||||
require.NoError(t, err, "failed to list historic jobs")
|
||||
|
||||
list, err := result.ToList()
|
||||
require.NoError(t, err, "results should be a list")
|
||||
require.NotEmpty(t, list.Items, "expect at least one job")
|
||||
|
||||
for _, elem := range list.Items {
|
||||
require.Equal(t, repoName, elem.GetLabels()[jobs.LabelRepository], "should have repo label")
|
||||
|
||||
state := mustNestedString(elem.Object, "status", "state")
|
||||
|
||||
// Check if state is in accepted states
|
||||
found := false
|
||||
for _, acceptedState := range acceptedStates {
|
||||
if state == acceptedState {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
require.True(t, found, "job %s completed with unexpected state %s (expected one of %v): %+v", elem.GetName(), state, acceptedStates, elem.Object)
|
||||
}
|
||||
}
|
||||
|
||||
// RenderObject reads the filePath and renders it as a template with the given values.
|
||||
// The template is expected to be a YAML or JSON file.
|
||||
//
|
||||
|
||||
@@ -2107,3 +2107,196 @@ func TestIntegrationProvisioning_MoveResources(t *testing.T) {
|
||||
}, time.Second*10, time.Millisecond*100, "Expected move job to handle non-existent resource")
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegrationProvisioning_SecondRepositoryOnlyExportsNewDashboards(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test")
|
||||
}
|
||||
|
||||
helper := runGrafana(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create some unmanaged dashboards directly in Grafana first
|
||||
dashboard1 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v1.yaml")
|
||||
dashboard1Obj, err := helper.DashboardsV1.Resource.Create(ctx, dashboard1, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create first dashboard")
|
||||
dashboard1Name := dashboard1Obj.GetName()
|
||||
|
||||
dashboard2 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v2beta1.yaml")
|
||||
dashboard2Obj, err := helper.DashboardsV2beta1.Resource.Create(ctx, dashboard2, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create second dashboard")
|
||||
dashboard2Name := dashboard2Obj.GetName()
|
||||
|
||||
// Create the first repository with sync enabled
|
||||
const repo1 = "first-repository"
|
||||
repo1Path := filepath.Join(helper.ProvisioningPath, repo1)
|
||||
err = os.MkdirAll(repo1Path, 0750)
|
||||
require.NoError(t, err, "should be able to create repository path")
|
||||
|
||||
createBody1 := helper.RenderObject(t, "testdata/local-write.json.tmpl", map[string]any{
|
||||
"Name": repo1,
|
||||
"SyncEnabled": true,
|
||||
"SyncTarget": "folder",
|
||||
"Path": repo1Path,
|
||||
})
|
||||
_, err = helper.Repositories.Resource.Create(ctx, createBody1, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create first repository")
|
||||
|
||||
// Print file tree before export
|
||||
printFileTree(t, helper.ProvisioningPath)
|
||||
|
||||
// Initial export
|
||||
result := helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo1).
|
||||
SubResource("jobs").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Push: &provisioning.ExportJobOptions{
|
||||
Folder: "", // export entire instance
|
||||
Path: "", // no prefix necessary for testing
|
||||
},
|
||||
})).
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create export job for first repo")
|
||||
helper.AwaitJobsWithStates(t, repo1, []string{"success"})
|
||||
// Wait for first repository to sync
|
||||
helper.SyncAndWait(t, repo1, nil)
|
||||
|
||||
printFileTree(t, helper.ProvisioningPath)
|
||||
// Verify that the first repository has claimed ownership of the dashboards
|
||||
managedDash1, err := helper.DashboardsV1.Resource.Get(ctx, dashboard1Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, repo1, managedDash1.GetAnnotations()[utils.AnnoKeyManagerIdentity], "dashboard1 should be managed by first repo")
|
||||
|
||||
managedDash2, err := helper.DashboardsV2beta1.Resource.Get(ctx, dashboard2Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, repo1, managedDash2.GetAnnotations()[utils.AnnoKeyManagerIdentity], "dashboard2 should be managed by first repo")
|
||||
|
||||
// Create second repository - enable sync and set different target
|
||||
|
||||
const repo2 = "second-repository"
|
||||
repo2Path := filepath.Join(helper.ProvisioningPath, repo2)
|
||||
err = os.MkdirAll(repo2Path, 0750)
|
||||
require.NoError(t, err, "should be able to create seconrd repository path")
|
||||
|
||||
printFileTree(t, helper.ProvisioningPath)
|
||||
|
||||
createBody2 := helper.RenderObject(t, "testdata/local-write.json.tmpl", map[string]any{
|
||||
"Name": repo2,
|
||||
"SyncEnabled": true,
|
||||
"SyncTarget": "folder",
|
||||
"Path": repo2Path,
|
||||
})
|
||||
|
||||
_, err = helper.Repositories.Resource.Create(ctx, createBody2, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create second repository")
|
||||
|
||||
// Wait for second repository to sync
|
||||
helper.SyncAndWait(t, repo2, nil)
|
||||
|
||||
// Validate that folders for both repositories exist
|
||||
folders, err := helper.Folders.Resource.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err, "should be able to list folders")
|
||||
|
||||
var repo1FolderFound, repo2FolderFound bool
|
||||
for _, folder := range folders.Items {
|
||||
if folder.GetName() == repo1 {
|
||||
repo1FolderFound = true
|
||||
}
|
||||
if folder.GetName() == repo2 {
|
||||
repo2FolderFound = true
|
||||
}
|
||||
}
|
||||
require.True(t, repo1FolderFound, "folder for first repository %s should exist after sync", repo1)
|
||||
require.True(t, repo2FolderFound, "folder for second repository %s should exist after sync", repo2)
|
||||
|
||||
// Create a third dashboard that won't be claimed by the first repo
|
||||
dashboard3 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v0.yaml")
|
||||
dashboard3Obj, err := helper.DashboardsV0.Resource.Create(ctx, dashboard3, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create third dashboard")
|
||||
dashboard3Name := dashboard3Obj.GetName()
|
||||
|
||||
// Verify dashboard3 is not managed by anyone initially
|
||||
unmanagedDash3, err := helper.DashboardsV0.Resource.Get(ctx, dashboard3Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
manager, found := unmanagedDash3.GetAnnotations()[utils.AnnoKeyManagerIdentity]
|
||||
require.True(t, !found || manager == "", "dashboard3 should not be managed initially")
|
||||
|
||||
printFileTree(t, helper.ProvisioningPath)
|
||||
// Count files in first repo before second export
|
||||
files1Before, err := countFilesInDir(repo1Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Export from second repository - this should only export the unmanaged dashboard3
|
||||
result = helper.AdminREST.Post().
|
||||
Namespace("default").
|
||||
Resource("repositories").
|
||||
Name(repo2).
|
||||
SubResource("jobs").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
Body(asJSON(&provisioning.JobSpec{
|
||||
Push: &provisioning.ExportJobOptions{
|
||||
Folder: "", // export entire instance
|
||||
Path: "", // no prefix necessary for testing
|
||||
},
|
||||
})).
|
||||
Do(ctx)
|
||||
require.NoError(t, result.Error(), "should be able to create export job for second repo")
|
||||
|
||||
// Wait for second repository export to complete
|
||||
helper.AwaitJobsWithStates(t, repo2, []string{"success"})
|
||||
|
||||
// Wait for second repository to sync
|
||||
helper.SyncAndWait(t, repo1, nil)
|
||||
helper.SyncAndWait(t, repo2, nil)
|
||||
|
||||
printFileTree(t, helper.ProvisioningPath)
|
||||
files1After, err := countFilesInDir(repo1Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
actualNewFiles := files1After - files1Before
|
||||
require.Equal(t, 0, actualNewFiles,
|
||||
"second repository should skip managed dashboards and had folder issues with unmanaged dashboard (expected %d new files, got %d)",
|
||||
0, actualNewFiles)
|
||||
|
||||
// Verify files in the second repository
|
||||
files2After, err := countFilesInDir(repo2Path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, files2After,
|
||||
"second repository should only export the unmanaged dashboard (expected %d new files, got %d)",
|
||||
1, files2After)
|
||||
|
||||
// Verify dashboard1 and dashboard2 are still managed by repo1 (unchanged)
|
||||
stillManagedDash1, err := helper.DashboardsV1.Resource.Get(ctx, dashboard1Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, repo1, stillManagedDash1.GetAnnotations()[utils.AnnoKeyManagerIdentity],
|
||||
"dashboard1 should still be managed by first repo")
|
||||
|
||||
stillManagedDash2, err := helper.DashboardsV2beta1.Resource.Get(ctx, dashboard2Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, repo1, stillManagedDash2.GetAnnotations()[utils.AnnoKeyManagerIdentity],
|
||||
"dashboard2 should still be managed by first repo")
|
||||
|
||||
// Verify dashboard3 is now managed by repo2
|
||||
stillManagedDash3, err := helper.DashboardsV0.Resource.Get(ctx, dashboard3Name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, repo2, stillManagedDash3.GetAnnotations()[utils.AnnoKeyManagerIdentity],
|
||||
"dashboard3 should now be managed by second repo")
|
||||
}
|
||||
|
||||
// Helper function to count files in a directory recursively
|
||||
func countFilesInDir(rootPath string) (int, error) {
|
||||
count := 0
|
||||
err := filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
count++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return count, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user