[Prototype] Implement repository file read endpoint (#96569)

go.mod

@@ -478,6 +478,8 @@ require github.com/grafana/loki/v3 v3.2.1 // @grafana/observability-logs
 
 require github.com/openzipkin/zipkin-go v0.4.3 // @grafana/oss-big-tent
 
+require github.com/google/go-github/v66 v66.0.0 // @grafana/grafana-app-platform-squad
+
 require (
 	cloud.google.com/go/longrunning v0.6.0 // indirect
 	github.com/at-wat/mqtt-go v0.19.4 // indirect

go.sum

@@ -2152,6 +2152,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd/D/9+M=
+github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4=
 github.com/google/go-pkcs11 v0.2.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
 github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=

pkg/apis/provisioning/v0alpha1/types.go

@@ -32,10 +32,15 @@ type GitHubRepositoryConfig struct {
 	Owner string `json:"owner,omitempty"`
 	// The name of the repository (e.g. test in `example/test` or `https://github.com/example/test`).
 	Repository string `json:"repository,omitempty"`
+	// Token for accessing the repository.
+	// TODO: this should be part of secrets and a simple reference.
+	Token string `json:"token,omitempty"`
+	// WebhookSecret is the secret used to validate incoming webhooks.
+	// TODO: this should be generated by the controller and not user-provided.
+	WebhookSecret string `json:"webhookSecret,omitempty"`
 
 	// TODO: Do we want an SSH url instead maybe?
 	// TODO: On-prem GitHub Enterprise support?
-	// TODO: How do we define access? Secrets?
 
 	// Whether we should commit to change branches and use a Pull Request flow to achieve this.
 	// By default, this is false (i.e. we will commit straight to the main branch).

pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go

@@ -46,6 +46,20 @@ func schema_pkg_apis_provisioning_v0alpha1_GitHubRepositoryConfig(ref common.Ref
 							Format:      "",
 						},
 					},
+					"token": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Token for accessing the repository.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"webhookSecret": {
+						SchemaProps: spec.SchemaProps{
+							Description: "WebhookSecret is the secret used to validate incoming webhooks.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
 					"branchWorkflow": {
 						SchemaProps: spec.SchemaProps{
 							Description: "Whether we should commit to change branches and use a Pull Request flow to achieve this. By default, this is false (i.e. we will commit straight to the main branch).",

pkg/registry/apis/provisioning/read.go

@@ -2,17 +2,22 @@ package provisioning
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/rest"
 
 	common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
 	provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
+
+	"golang.org/x/oauth2"
+
+	"github.com/google/go-github/v66/github"
 )
 
 type readConnector struct {
@@ -60,38 +65,58 @@ func (s *readConnector) Connect(ctx context.Context, name string, opts runtime.O
 		return nil, fmt.Errorf("expected repository, but got %t", obj)
 	}
 
+	tokenSrc := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: repo.Spec.GitHub.Token},
+	)
+	tokenClient := oauth2.NewClient(ctx, tokenSrc)
+	githubClient := github.NewClient(tokenClient)
+
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		idx := strings.Index(r.URL.Path, "/"+name+"/read")
-		path := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+2:], "/")
-		if path == "" {
-			responder.Error(fmt.Errorf("missing path")) // TODO bad request
+		filePath := strings.TrimLeft(r.URL.Path[idx+len(name+"/read")+2:], "/")
+		if filePath == "" {
+			// TODO: return bad request status code
+			responder.Error(fmt.Errorf("missing path"))
 			return
 		}
 		commit := r.URL.Query().Get("commit")
 
-		// TODO, actually read, and then validate
-		obj := &unstructured.Unstructured{
-			Object: map[string]interface{}{
-				"apiVersion": "dashboards.grafana.app/v????",
-				"metadata": metav1.ObjectMeta{
-					Name: "xxx",
-				},
-				"spec": map[string]interface{}{
-					"path":   path,
-					"commit": commit,
-					"repo":   repo.Name,
-					"type":   repo.Spec.Type,
-				},
-			},
+		owner, repoName, err := extractOwnerAndRepo(repo.Spec.GitHub.Repository)
+		if err != nil {
+			responder.Error(fmt.Errorf("failed to extract owner and repo: %w", err))
+			return
 		}
 
+		content, _, _, err := githubClient.Repositories.GetContents(ctx, owner, repoName, filePath, &github.RepositoryContentGetOptions{
+			Ref: commit,
+		})
+		if err != nil {
+			// TODO: return bad request status code
+			responder.Error(fmt.Errorf("failed to get content: %w", err))
+			return
+		}
+
+		data, err := content.GetContent()
+		if err != nil {
+			responder.Error(err)
+			return
+		}
+
+		var dashboardJSON map[string]interface{}
+		if err := json.Unmarshal([]byte(data), &dashboardJSON); err != nil {
+			// TODO: return bad request status code
+			responder.Error(fmt.Errorf("failed to unmarshal content: %w", err))
+			return
+		}
+
+		// TODO: validate the object is a valid dashboard
 		// Return the wrapped response
 		// Note we can not return it directly (using responder) because that will make sure the
 		// top level apiVersion is provisioning, not the remote resource
 		wrapper := &provisioning.ResourceWrapper{
 			Commit: commit,
 			Resource: common.Unstructured{
-				Object: obj.Object,
+				Object: dashboardJSON,
 			},
 		}
 		responder.Object(http.StatusOK, wrapper)
@@ -105,3 +130,22 @@ var (
 	_ rest.SingularNameProvider = (*readConnector)(nil)
 	_ rest.StorageMetadata      = (*readConnector)(nil)
 )
+
+// TODO: add validation in admission hook
+// extractOwnerAndRepo takes a GitHub repository URL and returns the owner and repo name.
+func extractOwnerAndRepo(repoURL string) (string, string, error) {
+	parsedURL, err := url.Parse(repoURL)
+	if err != nil {
+		return "", "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	// Split the path to get owner and repo
+	parts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/")
+	if len(parts) < 2 {
+		return "", "", fmt.Errorf("URL does not contain owner and repo")
+	}
+
+	owner := parts[0]
+	repo := parts[1]
+	return owner, repo, nil
+}

pkg/tests/apis/provisioning/provisioning_test.go

@@ -121,7 +121,9 @@ func TestIntegrationProvisioning(t *testing.T) {
 					"branchWorkflow": true,
 					"generateDashboardPreviews": true,
 					"owner": "grafana",
-					"repository": "git-ui-sync-demo"
+					"repository": "git-ui-sync-demo",
+					"token": "github_pat_dummy",
+					"webhookSecret": "dummyWebhookSecret"
 				},
 				"title": "Github Example",
 				"type": "github"

pkg/tests/apis/provisioning/testdata/github-example.yaml

@@ -10,4 +10,6 @@ spec:
     owner: grafana
     repository: git-ui-sync-demo
     branchWorkflow: true
-    generateDashboardPreviews: true
+    generateDashboardPreviews: true
+    token: "github_pat_dummy"
+    webhookSecret: "dummyWebhookSecret"