POC/Provisioning: Create an import endpoint (#97085)

* wip

* fix: don't panic on dirs

* fix: workaround dashboard Get returning 500 on 404

pkg/registry/apis/provisioning/repository/github/client.go

@@ -32,6 +32,13 @@ type Client interface {
 	// If ".." appears in the "path", this method will return an error.
 	GetContents(ctx context.Context, owner, repository, path, ref string) (fileContents RepositoryContent, dirContents []RepositoryContent, err error)
 
+	// GetTree returns the Git tree in the repository.
+	// When recursive is given, subtrees are mapped into the returned array.
+	//
+	// The truncated bool will be set to true if the tree is larger than 7 MB or 100 000 entries.
+	// When truncated is true, you may wish to read each subtree manually instead.
+	GetTree(ctx context.Context, owner, repository, ref string, recursive bool) (entries []RepositoryContent, truncated bool, err error)
+
 	// CreateFile creates a new file in the repository under the given path.
 	// The file is created on the branch given.
 	// The message given is the commit message. If none is given, an appropriate default is used.
@@ -85,6 +92,8 @@ type RepositoryContent interface {
 	// Get the SHA hash. This is usually a SHA-256, but may also be SHA-512.
 	// Directories have SHA hashes, too (TODO: how is this calculated?).
 	GetSHA() string
+	// The size of the file. Not necessarily non-zero, even if the file is supposed to be non-zero.
+	GetSize() int64
 }
 
 type CreateFileOptions struct {

pkg/registry/apis/provisioning/repository/github/client_mock.go

@@ -180,6 +180,43 @@ func (_m *MockClient) GetContents(ctx context.Context, owner string, repository
 	return r0, r1, r2
 }
 
+// GetTree provides a mock function with given fields: ctx, owner, repository, ref, recursive
+func (_m *MockClient) GetTree(ctx context.Context, owner string, repository string, ref string, recursive bool) ([]RepositoryContent, bool, error) {
+	ret := _m.Called(ctx, owner, repository, ref, recursive)
+
+	if len(ret) == 0 {
+		panic("no return value specified for GetTree")
+	}
+
+	var r0 []RepositoryContent
+	var r1 bool
+	var r2 error
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, string, bool) ([]RepositoryContent, bool, error)); ok {
+		return rf(ctx, owner, repository, ref, recursive)
+	}
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, string, bool) []RepositoryContent); ok {
+		r0 = rf(ctx, owner, repository, ref, recursive)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]RepositoryContent)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, string, bool) bool); ok {
+		r1 = rf(ctx, owner, repository, ref, recursive)
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	if rf, ok := ret.Get(2).(func(context.Context, string, string, string, bool) error); ok {
+		r2 = rf(ctx, owner, repository, ref, recursive)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
 // ListWebhooks provides a mock function with given fields: ctx, owner, repository
 func (_m *MockClient) ListWebhooks(ctx context.Context, owner string, repository string) ([]WebhookConfig, error) {
 	ret := _m.Called(ctx, owner, repository)

pkg/registry/apis/provisioning/repository/github/real.go

@@ -51,6 +51,41 @@ func (r *realImpl) GetContents(ctx context.Context, owner, repository, path, ref
 	}
 }
 
+func (r *realImpl) GetTree(ctx context.Context, owner, repository, ref string, recursive bool) ([]RepositoryContent, bool, error) {
+	tree, _, err := r.gh.Git.GetTree(ctx, owner, repository, ref, recursive)
+	if err != nil {
+		var ghErr *github.ErrorResponse
+		if !errors.As(err, &ghErr) {
+			return nil, false, err
+		}
+		if ghErr.Response.StatusCode == http.StatusServiceUnavailable {
+			return nil, false, ErrServiceUnavailable
+		}
+		if ghErr.Response.StatusCode == http.StatusNotFound {
+			return nil, false, ErrResourceNotFound
+		}
+		return nil, false, err
+	}
+
+	var entries []RepositoryContent
+	for _, te := range tree.Entries {
+		rrc := &realRepositoryContent{
+			real: &github.RepositoryContent{
+				Path: te.Path,
+				Size: te.Size,
+				SHA:  te.SHA,
+			},
+		}
+		if te.GetType() == "tree" {
+			rrc.real.Type = github.String("dir")
+		} else {
+			rrc.real.Type = te.Type
+		}
+		entries = append(entries, rrc)
+	}
+	return entries, tree.GetTruncated(), nil
+}
+
 func (r *realImpl) CreateFile(ctx context.Context, owner, repository, path, branch, message string, content []byte) error {
 	if strings.Contains(path, "..") {
 		return ErrPathTraversalDisallowed
@@ -300,3 +335,15 @@ func (c realRepositoryContent) GetPath() string {
 func (c realRepositoryContent) GetSHA() string {
 	return c.real.GetSHA()
 }
+
+func (c realRepositoryContent) GetSize() int64 {
+	if c.real.Size != nil {
+		return int64(*c.real.Size)
+	}
+	if c.real.Content != nil {
+		if c, err := c.real.GetContent(); err == nil {
+			return int64(len(c))
+		}
+	}
+	return 0
+}