Provisioning: Cleanup folders properly with webhooks (#112031)
This commit is contained in:
Stephanie Hingtgen
+105
@@ -0,0 +1,105 @@
|
||||
{
|
||||
"ref": "refs/heads/main",
|
||||
"before": "72096e3adc646c5a5b8a91744f962b12bac06045",
|
||||
"after": "1234567890abcdef1234567890abcdef12345678",
|
||||
"repository": {
|
||||
"id": 888020043,
|
||||
"node_id": "R_kgDONO4cSw",
|
||||
"name": "git-ui-sync-demo",
|
||||
"full_name": "grafana/git-ui-sync-demo",
|
||||
"private": true,
|
||||
"owner": {
|
||||
"name": "grafana",
|
||||
"email": "hello@grafana.com",
|
||||
"login": "grafana",
|
||||
"id": 7195757,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjcxOTU3NTc=",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/7195757?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/grafana",
|
||||
"html_url": "https://github.com/grafana",
|
||||
"type": "Organization",
|
||||
"site_admin": false
|
||||
},
|
||||
"html_url": "https://github.com/grafana/git-ui-sync-demo",
|
||||
"description": "A repository containing Grafana dashboards to demo the Github Sync feature in Grafana.",
|
||||
"fork": false,
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo",
|
||||
"default_branch": "main",
|
||||
"master_branch": "main",
|
||||
"organization": "grafana"
|
||||
},
|
||||
"pusher": {
|
||||
"name": "testuser",
|
||||
"email": "test@grafana.com"
|
||||
},
|
||||
"organization": {
|
||||
"login": "grafana",
|
||||
"id": 7195757,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjcxOTU3NTc=",
|
||||
"url": "https://api.github.com/orgs/grafana",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/7195757?v=4"
|
||||
},
|
||||
"sender": {
|
||||
"login": "testuser",
|
||||
"id": 123456,
|
||||
"node_id": "MDQ6VXNlcjEyMzQ1Ng==",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/123456?v=4",
|
||||
"type": "User",
|
||||
"site_admin": false
|
||||
},
|
||||
"created": false,
|
||||
"deleted": false,
|
||||
"forced": false,
|
||||
"base_ref": null,
|
||||
"compare": "https://github.com/grafana/git-ui-sync-demo/compare/72096e3adc64...1234567890ab",
|
||||
"commits": [
|
||||
{
|
||||
"id": "1234567890abcdef1234567890abcdef12345678",
|
||||
"tree_id": "abcdef1234567890abcdef1234567890abcdef12",
|
||||
"distinct": true,
|
||||
"message": "Remove empty folder by deleting .keep file",
|
||||
"timestamp": "2024-12-09T11:00:48+03:00",
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo/commit/1234567890abcdef1234567890abcdef12345678",
|
||||
"author": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"committer": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [
|
||||
"empty-folder/.keep"
|
||||
],
|
||||
"modified": []
|
||||
}
|
||||
],
|
||||
"head_commit": {
|
||||
"id": "1234567890abcdef1234567890abcdef12345678",
|
||||
"tree_id": "abcdef1234567890abcdef1234567890abcdef12",
|
||||
"distinct": true,
|
||||
"message": "Remove empty folder by deleting .keep file",
|
||||
"timestamp": "2024-12-09T11:00:48+03:00",
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo/commit/1234567890abcdef1234567890abcdef12345678",
|
||||
"author": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"committer": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [
|
||||
"empty-folder/.keep"
|
||||
],
|
||||
"modified": []
|
||||
}
|
||||
}
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"ref": "refs/heads/main",
|
||||
"before": "72096e3adc646c5a5b8a91744f962b12bac06045",
|
||||
"after": "2345678901bcdef2345678901bcdef2345678901",
|
||||
"repository": {
|
||||
"id": 888020043,
|
||||
"node_id": "R_kgDONO4cSw",
|
||||
"name": "git-ui-sync-demo",
|
||||
"full_name": "grafana/git-ui-sync-demo",
|
||||
"private": true,
|
||||
"owner": {
|
||||
"name": "grafana",
|
||||
"email": "hello@grafana.com",
|
||||
"login": "grafana",
|
||||
"id": 7195757,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjcxOTU3NTc=",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/7195757?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/grafana",
|
||||
"html_url": "https://github.com/grafana",
|
||||
"type": "Organization",
|
||||
"site_admin": false
|
||||
},
|
||||
"html_url": "https://github.com/grafana/git-ui-sync-demo",
|
||||
"description": "A repository containing Grafana dashboards to demo the Github Sync feature in Grafana.",
|
||||
"fork": false,
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo",
|
||||
"default_branch": "main",
|
||||
"master_branch": "main",
|
||||
"organization": "grafana"
|
||||
},
|
||||
"pusher": {
|
||||
"name": "testuser",
|
||||
"email": "test@grafana.com"
|
||||
},
|
||||
"organization": {
|
||||
"login": "grafana",
|
||||
"id": 7195757,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjcxOTU3NTc=",
|
||||
"url": "https://api.github.com/orgs/grafana",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/7195757?v=4"
|
||||
},
|
||||
"sender": {
|
||||
"login": "testuser",
|
||||
"id": 123456,
|
||||
"node_id": "MDQ6VXNlcjEyMzQ1Ng==",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/123456?v=4",
|
||||
"type": "User",
|
||||
"site_admin": false
|
||||
},
|
||||
"created": false,
|
||||
"deleted": false,
|
||||
"forced": false,
|
||||
"base_ref": null,
|
||||
"compare": "https://github.com/grafana/git-ui-sync-demo/compare/72096e3adc64...2345678901bc",
|
||||
"commits": [
|
||||
{
|
||||
"id": "2345678901bcdef2345678901bcdef2345678901",
|
||||
"tree_id": "bcdef2345678901bcdef2345678901bcdef23456",
|
||||
"distinct": true,
|
||||
"message": "Remove folder with .keep and dashboard files",
|
||||
"timestamp": "2024-12-09T11:00:48+03:00",
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo/commit/2345678901bcdef2345678901bcdef2345678901",
|
||||
"author": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"committer": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [
|
||||
"dashboards/.keep",
|
||||
"dashboards/dashboard1.json",
|
||||
"dashboards/dashboard2.json"
|
||||
],
|
||||
"modified": []
|
||||
}
|
||||
],
|
||||
"head_commit": {
|
||||
"id": "2345678901bcdef2345678901bcdef2345678901",
|
||||
"tree_id": "bcdef2345678901bcdef2345678901bcdef23456",
|
||||
"distinct": true,
|
||||
"message": "Remove folder with .keep and dashboard files",
|
||||
"timestamp": "2024-12-09T11:00:48+03:00",
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo/commit/2345678901bcdef2345678901bcdef2345678901",
|
||||
"author": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"committer": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [
|
||||
"dashboards/.keep",
|
||||
"dashboards/dashboard1.json",
|
||||
"dashboards/dashboard2.json"
|
||||
],
|
||||
"modified": []
|
||||
}
|
||||
}
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"ref": "refs/heads/main",
|
||||
"before": "72096e3adc646c5a5b8a91744f962b12bac06045",
|
||||
"after": "3456789012cdef3456789012cdef3456789012cd",
|
||||
"repository": {
|
||||
"id": 888020043,
|
||||
"node_id": "R_kgDONO4cSw",
|
||||
"name": "git-ui-sync-demo",
|
||||
"full_name": "grafana/git-ui-sync-demo",
|
||||
"private": true,
|
||||
"owner": {
|
||||
"name": "grafana",
|
||||
"email": "hello@grafana.com",
|
||||
"login": "grafana",
|
||||
"id": 7195757,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjcxOTU3NTc=",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/7195757?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/grafana",
|
||||
"html_url": "https://github.com/grafana",
|
||||
"type": "Organization",
|
||||
"site_admin": false
|
||||
},
|
||||
"html_url": "https://github.com/grafana/git-ui-sync-demo",
|
||||
"description": "A repository containing Grafana dashboards to demo the Github Sync feature in Grafana.",
|
||||
"fork": false,
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo",
|
||||
"default_branch": "main",
|
||||
"master_branch": "main",
|
||||
"organization": "grafana"
|
||||
},
|
||||
"pusher": {
|
||||
"name": "testuser",
|
||||
"email": "test@grafana.com"
|
||||
},
|
||||
"organization": {
|
||||
"login": "grafana",
|
||||
"id": 7195757,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjcxOTU3NTc=",
|
||||
"url": "https://api.github.com/orgs/grafana",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/7195757?v=4"
|
||||
},
|
||||
"sender": {
|
||||
"login": "testuser",
|
||||
"id": 123456,
|
||||
"node_id": "MDQ6VXNlcjEyMzQ1Ng==",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/123456?v=4",
|
||||
"type": "User",
|
||||
"site_admin": false
|
||||
},
|
||||
"created": false,
|
||||
"deleted": false,
|
||||
"forced": false,
|
||||
"base_ref": null,
|
||||
"compare": "https://github.com/grafana/git-ui-sync-demo/compare/72096e3adc64...3456789012cd",
|
||||
"commits": [
|
||||
{
|
||||
"id": "3456789012cdef3456789012cdef3456789012cd",
|
||||
"tree_id": "cdef3456789012cdef3456789012cdef34567890",
|
||||
"distinct": true,
|
||||
"message": "Remove multiple folders, some with only .keep files",
|
||||
"timestamp": "2024-12-09T11:00:48+03:00",
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo/commit/3456789012cdef3456789012cdef3456789012cd",
|
||||
"author": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"committer": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [
|
||||
"empty-folder1/.keep",
|
||||
"dashboards-to-delete/.keep",
|
||||
"dashboards-to-delete/dashboard.json"
|
||||
],
|
||||
"modified": []
|
||||
}
|
||||
],
|
||||
"head_commit": {
|
||||
"id": "3456789012cdef3456789012cdef3456789012cd",
|
||||
"tree_id": "cdef3456789012cdef3456789012cdef34567890",
|
||||
"distinct": true,
|
||||
"message": "Remove multiple folders, some with only .keep files",
|
||||
"timestamp": "2024-12-09T11:00:48+03:00",
|
||||
"url": "https://github.com/grafana/git-ui-sync-demo/commit/3456789012cdef3456789012cdef3456789012cd",
|
||||
"author": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"committer": {
|
||||
"name": "Test User",
|
||||
"email": "test@grafana.com",
|
||||
"username": "testuser"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [
|
||||
"empty-folder1/.keep",
|
||||
"dashboards-to-delete/.keep",
|
||||
"dashboards-to-delete/dashboard.json"
|
||||
],
|
||||
"modified": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-github/v70/github"
|
||||
"github.com/google/uuid"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"github.com/grafana/grafana-app-sdk/logging"
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/safepath"
|
||||
common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
|
||||
)
|
||||
|
||||
@@ -119,13 +121,38 @@ func (r *githubWebhookRepository) parsePushEvent(event *github.PushEvent) (*prov
|
||||
return &provisioning.WebhookResponse{Code: http.StatusOK}, nil
|
||||
}
|
||||
|
||||
// whenever possible, we want to do incremental syncs to keep things performant.
|
||||
// however, if we get an event where just a .keep file is being deleted, and no other files in the folder
|
||||
// are being deleted, the folder could be gone from git, but not from grafana and we do not have a way
|
||||
// to get the grafana uid to delete the folder. so, instead, we will queue a full sync to clean things up.
|
||||
dirsWithKeepDeletes := make(map[string]struct{})
|
||||
dirsWithOtherDeletes := make(map[string]struct{})
|
||||
for _, change := range event.GetCommits() {
|
||||
for _, removedFile := range change.Removed {
|
||||
dir := safepath.Dir(removedFile)
|
||||
if strings.HasSuffix(removedFile, ".keep") {
|
||||
dirsWithKeepDeletes[dir] = struct{}{}
|
||||
} else {
|
||||
dirsWithOtherDeletes[dir] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
// if there are any keep files deleted that do not have other files deleted in the same folder, we need to queue a full sync
|
||||
incremental := true
|
||||
for dir := range dirsWithKeepDeletes {
|
||||
if _, exists := dirsWithOtherDeletes[dir]; !exists {
|
||||
incremental = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return &provisioning.WebhookResponse{
|
||||
Code: http.StatusAccepted,
|
||||
Job: &provisioning.JobSpec{
|
||||
Repository: r.config.GetName(),
|
||||
Action: provisioning.JobActionPull,
|
||||
Pull: &provisioning.SyncJobOptions{
|
||||
Incremental: true,
|
||||
Incremental: incremental,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
|
||||
@@ -69,6 +69,36 @@ func TestParseWebhooks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"push", "keep_file_only", provisioning.WebhookResponse{
|
||||
Code: http.StatusAccepted,
|
||||
Job: &provisioning.JobSpec{
|
||||
Repository: "unit-test-repo",
|
||||
Action: provisioning.JobActionPull,
|
||||
Pull: &provisioning.SyncJobOptions{
|
||||
Incremental: false,
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"push", "keep_file_with_others", provisioning.WebhookResponse{
|
||||
Code: http.StatusAccepted,
|
||||
Job: &provisioning.JobSpec{
|
||||
Repository: "unit-test-repo",
|
||||
Action: provisioning.JobActionPull,
|
||||
Pull: &provisioning.SyncJobOptions{
|
||||
Incremental: true,
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"push", "multiple_keep_files", provisioning.WebhookResponse{
|
||||
Code: http.StatusAccepted,
|
||||
Job: &provisioning.JobSpec{
|
||||
Repository: "unit-test-repo",
|
||||
Action: provisioning.JobActionPull,
|
||||
Pull: &provisioning.SyncJobOptions{
|
||||
Incremental: false,
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"issue_comment", "created", provisioning.WebhookResponse{
|
||||
Code: http.StatusNotImplemented,
|
||||
}},
|
||||
|
||||
Reference in New Issue
Block a user