From cf15826360b21d8c4e6f8bdf0e2f076376d888d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Jim=C3=A9nez=20S=C3=A1nchez?= Date: Fri, 15 Nov 2024 17:33:40 +0100 Subject: [PATCH] [Prototype] Implement repository file read endpoint (#96569) --- go.mod | 2 + go.sum | 2 + pkg/apis/provisioning/v0alpha1/types.go | 7 +- .../v0alpha1/zz_generated.openapi.go | 14 ++++ pkg/registry/apis/provisioning/read.go | 82 ++++++++++++++----- .../apis/provisioning/provisioning_test.go | 4 +- .../provisioning/testdata/github-example.yaml | 4 +- 7 files changed, 93 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index df73af80bc0..391be673dba 100644 --- a/go.mod +++ b/go.mod @@ -478,6 +478,8 @@ require github.com/grafana/loki/v3 v3.2.1 // @grafana/observability-logs require github.com/openzipkin/zipkin-go v0.4.3 // @grafana/oss-big-tent +require github.com/google/go-github/v66 v66.0.0 // @grafana/grafana-app-platform-squad + require ( cloud.google.com/go/longrunning v0.6.0 // indirect github.com/at-wat/mqtt-go v0.19.4 // indirect diff --git a/go.sum b/go.sum index e87e90b4184..0bfe39e117b 100644 --- a/go.sum +++ b/go.sum @@ -2152,6 +2152,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd/D/9+M= +github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4= github.com/google/go-pkcs11 v0.2.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= diff --git a/pkg/apis/provisioning/v0alpha1/types.go b/pkg/apis/provisioning/v0alpha1/types.go index 11a8bfc9617..58aeec5e1bb 100644 --- a/pkg/apis/provisioning/v0alpha1/types.go +++ b/pkg/apis/provisioning/v0alpha1/types.go @@ -32,10 +32,15 @@ type GitHubRepositoryConfig struct { Owner string `json:"owner,omitempty"` // The name of the repository (e.g. test in `example/test` or `https://github.com/example/test`). Repository string `json:"repository,omitempty"` + // Token for accessing the repository. + // TODO: this should be part of secrets and a simple reference. + Token string `json:"token,omitempty"` + // WebhookSecret is the secret used to validate incoming webhooks. + // TODO: this should be generated by the controller and not user-provided. + WebhookSecret string `json:"webhookSecret,omitempty"` // TODO: Do we want an SSH url instead maybe? // TODO: On-prem GitHub Enterprise support? - // TODO: How do we define access? Secrets? // Whether we should commit to change branches and use a Pull Request flow to achieve this. // By default, this is false (i.e. we will commit straight to the main branch). diff --git a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go index 35762c3f85b..65486d5e0af 100644 --- a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go +++ b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go @@ -46,6 +46,20 @@ func schema_pkg_apis_provisioning_v0alpha1_GitHubRepositoryConfig(ref common.Ref Format: "", }, }, + "token": { + SchemaProps: spec.SchemaProps{ + Description: "Token for accessing the repository.", + Type: []string{"string"}, + Format: "", + }, + }, + "webhookSecret": { + SchemaProps: spec.SchemaProps{ + Description: "WebhookSecret is the secret used to validate incoming webhooks.", + Type: []string{"string"}, + Format: "", + }, + }, "branchWorkflow": { SchemaProps: spec.SchemaProps{ Description: "Whether we should commit to change branches and use a Pull Request flow to achieve this. By default, this is false (i.e. we will commit straight to the main branch).", diff --git a/pkg/registry/apis/provisioning/read.go b/pkg/registry/apis/provisioning/read.go index d45bd80a9ee..278ac23ae13 100644 --- a/pkg/registry/apis/provisioning/read.go +++ b/pkg/registry/apis/provisioning/read.go @@ -2,17 +2,22 @@ package provisioning import ( "context" + "encoding/json" "fmt" "net/http" + "net/url" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1" + + "golang.org/x/oauth2" + + "github.com/google/go-github/v66/github" ) type readConnector struct { @@ -60,38 +65,58 @@ func (s *readConnector) Connect(ctx context.Context, name string, opts runtime.O return nil, fmt.Errorf("expected repository, but got %t", obj) } + tokenSrc := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: repo.Spec.GitHub.Token}, + ) + tokenClient := oauth2.NewClient(ctx, tokenSrc) + githubClient := github.NewClient(tokenClient) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { idx := strings.Index(r.URL.Path, "/"+name+"/read") - path := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+2:], "/") - if path == "" { - responder.Error(fmt.Errorf("missing path")) // TODO bad request + filePath := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+2:], "/") + if filePath == "" { + // TODO: return bad request status code + responder.Error(fmt.Errorf("missing path")) return } commit := r.URL.Query().Get("commit") - // TODO, actually read, and then validate - obj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "dashboards.grafana.app/v????", - "metadata": metav1.ObjectMeta{ - Name: "xxx", - }, - "spec": map[string]interface{}{ - "path": path, - "commit": commit, - "repo": repo.Name, - "type": repo.Spec.Type, - }, - }, + owner, repoName, err := extractOwnerAndRepo(repo.Spec.GitHub.Repository) + if err != nil { + responder.Error(fmt.Errorf("failed to extract owner and repo: %w", err)) + return } + content, _, _, err := githubClient.Repositories.GetContents(ctx, owner, repoName, filePath, &github.RepositoryContentGetOptions{ + Ref: commit, + }) + if err != nil { + // TODO: return bad request status code + responder.Error(fmt.Errorf("failed to get content: %w", err)) + return + } + + data, err := content.GetContent() + if err != nil { + responder.Error(err) + return + } + + var dashboardJSON map[string]interface{} + if err := json.Unmarshal([]byte(data), &dashboardJSON); err != nil { + // TODO: return bad request status code + responder.Error(fmt.Errorf("failed to unmarshal content: %w", err)) + return + } + + // TODO: validate the object is a valid dashboard // Return the wrapped response // Note we can not return it directly (using responder) because that will make sure the // top level apiVersion is provisioning, not the remote resource wrapper := &provisioning.ResourceWrapper{ Commit: commit, Resource: common.Unstructured{ - Object: obj.Object, + Object: dashboardJSON, }, } responder.Object(http.StatusOK, wrapper) @@ -105,3 +130,22 @@ var ( _ rest.SingularNameProvider = (*readConnector)(nil) _ rest.StorageMetadata = (*readConnector)(nil) ) + +// TODO: add validation in admission hook +// extractOwnerAndRepo takes a GitHub repository URL and returns the owner and repo name. +func extractOwnerAndRepo(repoURL string) (string, string, error) { + parsedURL, err := url.Parse(repoURL) + if err != nil { + return "", "", fmt.Errorf("invalid URL: %w", err) + } + + // Split the path to get owner and repo + parts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("URL does not contain owner and repo") + } + + owner := parts[0] + repo := parts[1] + return owner, repo, nil +} diff --git a/pkg/tests/apis/provisioning/provisioning_test.go b/pkg/tests/apis/provisioning/provisioning_test.go index 89c4c12d1c9..ab1668db495 100644 --- a/pkg/tests/apis/provisioning/provisioning_test.go +++ b/pkg/tests/apis/provisioning/provisioning_test.go @@ -121,7 +121,9 @@ func TestIntegrationProvisioning(t *testing.T) { "branchWorkflow": true, "generateDashboardPreviews": true, "owner": "grafana", - "repository": "git-ui-sync-demo" + "repository": "git-ui-sync-demo", + "token": "github_pat_dummy", + "webhookSecret": "dummyWebhookSecret" }, "title": "Github Example", "type": "github" diff --git a/pkg/tests/apis/provisioning/testdata/github-example.yaml b/pkg/tests/apis/provisioning/testdata/github-example.yaml index 58aaba35a2f..93d6308f0f3 100644 --- a/pkg/tests/apis/provisioning/testdata/github-example.yaml +++ b/pkg/tests/apis/provisioning/testdata/github-example.yaml @@ -10,4 +10,6 @@ spec: owner: grafana repository: git-ui-sync-demo branchWorkflow: true - generateDashboardPreviews: true \ No newline at end of file + generateDashboardPreviews: true + token: "github_pat_dummy" + webhookSecret: "dummyWebhookSecret" \ No newline at end of file