POC/Provisioing: Show list in API (#97090)

pkg/apis/provisioning/v0alpha1/register.go

@@ -81,6 +81,7 @@ func AddKnownTypes(gv schema.GroupVersion, scheme *runtime.Scheme) error {
 		&HelloWorld{},
 		&WebhookResponse{},
 		&ResourceWrapper{},
+		&FileList{},
 	)
 	return nil
 }

pkg/apis/provisioning/v0alpha1/types.go

@@ -183,3 +183,22 @@ type WebhookResponse struct {
 
 	Status string `json:"status,omitempty"`
 }
+
+// Information we can get just from the file listing
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type FileList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	// should be named "items", but avoid subresource error for now:
+	// kubernetes/kubernetes#126809
+	Items []FileItem `json:"files,omitempty"`
+}
+
+type FileItem struct {
+	Path     string `json:"path"`
+	Size     int64  `json:"size,omitempty"`
+	Hash     string `json:"hash,omitempty"`
+	Modified int64  `json:"modified,omitempty"`
+	Author   string `json:"author,omitempty"`
+}

pkg/apis/provisioning/v0alpha1/zz_generated.deepcopy.go

@@ -27,6 +27,53 @@ func (in *EditingOptions) DeepCopy() *EditingOptions {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FileItem) DeepCopyInto(out *FileItem) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileItem.
+func (in *FileItem) DeepCopy() *FileItem {
+	if in == nil {
+		return nil
+	}
+	out := new(FileItem)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FileList) DeepCopyInto(out *FileList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]FileItem, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileList.
+func (in *FileList) DeepCopy() *FileList {
+	if in == nil {
+		return nil
+	}
+	out := new(FileList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *FileList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GitHubRepositoryConfig) DeepCopyInto(out *GitHubRepositoryConfig) {
 	*out = *in

pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go

@@ -15,6 +15,8 @@ import (
 func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
 	return map[string]common.OpenAPIDefinition{
 		"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.EditingOptions":         schema_pkg_apis_provisioning_v0alpha1_EditingOptions(ref),
+		"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileItem":               schema_pkg_apis_provisioning_v0alpha1_FileItem(ref),
+		"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileList":               schema_pkg_apis_provisioning_v0alpha1_FileList(ref),
 		"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.GitHubRepositoryConfig": schema_pkg_apis_provisioning_v0alpha1_GitHubRepositoryConfig(ref),
 		"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.HelloWorld":             schema_pkg_apis_provisioning_v0alpha1_HelloWorld(ref),
 		"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.LocalRepositoryConfig":  schema_pkg_apis_provisioning_v0alpha1_LocalRepositoryConfig(ref),
@@ -66,6 +68,99 @@ func schema_pkg_apis_provisioning_v0alpha1_EditingOptions(ref common.ReferenceCa
 	}
 }
 
+func schema_pkg_apis_provisioning_v0alpha1_FileItem(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Type: []string{"object"},
+				Properties: map[string]spec.Schema{
+					"path": {
+						SchemaProps: spec.SchemaProps{
+							Default: "",
+							Type:    []string{"string"},
+							Format:  "",
+						},
+					},
+					"size": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"integer"},
+							Format: "int64",
+						},
+					},
+					"hash": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"string"},
+							Format: "",
+						},
+					},
+					"modified": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"integer"},
+							Format: "int64",
+						},
+					},
+					"author": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"string"},
+							Format: "",
+						},
+					},
+				},
+				Required: []string{"path"},
+			},
+		},
+	}
+}
+
+func schema_pkg_apis_provisioning_v0alpha1_FileList(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Description: "Information we can get just from the file listing",
+				Type:        []string{"object"},
+				Properties: map[string]spec.Schema{
+					"kind": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"apiVersion": {
+						SchemaProps: spec.SchemaProps{
+							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"metadata": {
+						SchemaProps: spec.SchemaProps{
+							Default: map[string]interface{}{},
+							Ref:     ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
+						},
+					},
+					"files": {
+						SchemaProps: spec.SchemaProps{
+							Description: "should be named \"items\", but avoid subresource error for now: kubernetes/kubernetes#126809",
+							Type:        []string{"array"},
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Default: map[string]interface{}{},
+										Ref:     ref("github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileItem"),
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		Dependencies: []string{
+			"github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileItem", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"},
+	}
+}
+
 func schema_pkg_apis_provisioning_v0alpha1_GitHubRepositoryConfig(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{

pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list

@@ -1,3 +1,4 @@
 API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ResourceWrapper,Errors
+API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,FileList,Items
 API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,GitHub
 API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,PreferYAML

pkg/registry/apis/provisioning/files.go

@@ -65,7 +65,11 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
 	}
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		logger := logger.With("url", r.URL.Path)
+		query := r.URL.Query()
+		ref := query.Get("ref")
+		message := query.Get("message")
+		logger = logger.With("url", r.URL.Path, "ref", ref, "message", message)
+
 		prefix := fmt.Sprintf("/%s/files/", name)
 		idx := strings.Index(r.URL.Path, prefix)
 		if idx == -1 {
@@ -75,9 +79,30 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
 		}
 
 		filePath := r.URL.Path[idx+len(prefix):]
-		if filePath == "" {
-			logger.DebugContext(r.Context(), "got an empty file path")
-			responder.Error(apierrors.NewBadRequest("missing path"))
+		if filePath == "" || strings.HasSuffix(filePath, "/") {
+			if len(filePath) > 0 {
+				responder.Error(apierrors.NewBadRequest("folder navigation not yet supported"))
+				return
+			}
+
+			rsp, err := repo.ReadTree(r.Context(), logger, ref)
+			if err != nil {
+				responder.Error(err)
+				return
+			}
+
+			files := &provisioning.FileList{}
+			for _, v := range rsp {
+				if !v.Blob {
+					continue // folder item
+				}
+				files.Items = append(files.Items, provisioning.FileItem{
+					Path: v.Path,
+					Size: v.Size,
+					Hash: v.Hash,
+				})
+			}
+			responder.Object(http.StatusOK, files)
 			return
 		}
 
@@ -97,10 +122,6 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime.
 		}
 
 		var obj *provisioning.ResourceWrapper
-		ref := r.URL.Query().Get("ref")
-		message := r.URL.Query().Get("message")
-		logger = logger.With("ref", ref, "message", message)
-
 		code := http.StatusOK
 		switch r.Method {
 		case http.MethodGet:

pkg/registry/apis/provisioning/register.go

@@ -153,17 +153,6 @@ func (b *ProvisioningAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserv
 		logger:        b.logger.With("connector", "hello_world"),
 	}
 
-	exportConnector := &exportConnector{
-		repoGetter: b,
-		client:     b.client,
-		logger:     b.logger.With("connector", "export"),
-	}
-	importConnector := &importConnector{
-		repoGetter: b,
-		client:     b.client,
-		logger:     b.logger.With("connector", "import"),
-	}
-
 	storage := map[string]rest.Storage{}
 	storage[provisioning.RepositoryResourceInfo.StoragePath()] = repositoryStorage
 	// Can be used by kubectl: kubectl --kubeconfig grafana.kubeconfig patch Repository local-devenv --type=merge --subresource=status --patch='status: {"currentGitCommit": "hello"}'
@@ -179,8 +168,16 @@ func (b *ProvisioningAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserv
 		client: b.client,
 		logger: b.logger.With("connector", "files"),
 	}
-	storage[provisioning.RepositoryResourceInfo.StoragePath("export")] = exportConnector
-	storage[provisioning.RepositoryResourceInfo.StoragePath("import")] = importConnector
+	storage[provisioning.RepositoryResourceInfo.StoragePath("import")] = &importConnector{
+		repoGetter: b,
+		client:     b.client,
+		logger:     b.logger.With("connector", "import"),
+	}
+	storage[provisioning.RepositoryResourceInfo.StoragePath("export")] = &exportConnector{
+		repoGetter: b,
+		client:     b.client,
+		logger:     b.logger.With("connector", "export"),
+	}
 	apiGroupInfo.VersionedResourcesStorageMap[provisioning.VERSION] = storage
 	return nil
 }
@@ -464,6 +461,8 @@ func (b *ProvisioningAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.
 	root := "/apis/" + b.GetGroupVersion().String() + "/"
 	repoprefix := root + "namespaces/{namespace}/repositories/{name}"
 
+	defs := b.GetOpenAPIDefinitions()(func(path string) spec.Ref { return spec.Ref{} })
+
 	// TODO: we might want to register some extras for subresources here.
 	sub := oas.Paths.Paths[repoprefix+"/hello"]
 	if sub != nil && sub.Get != nil {
@@ -521,8 +520,23 @@ func (b *ProvisioningAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.
 		sub.Post.Parameters = []*spec3.Parameter{ref}
 	}
 
-	// hide the version with no path
-	delete(oas.Paths.Paths, repoprefix+"/files")
+	// Show a special list command
+	sub = oas.Paths.Paths[repoprefix+"/files"]
+	if sub != nil {
+		delete(oas.Paths.Paths, repoprefix+"/files")
+		oas.Paths.Paths[repoprefix+"/files/"] = sub // add the trailing final slash
+		sub.Get.Description = "Get the files and content hash"
+		sub.Get.Summary = "File listing"
+		sub.Get.Parameters = []*spec3.Parameter{ref}
+		sub.Post = nil
+		sub.Put = nil
+		sub.Delete = nil
+
+		// Replace the content type for this response
+		mt := sub.Get.Responses.StatusCodeResponses[200].Content
+		s := defs["github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileList"].Schema
+		mt["*/*"].Schema = &s
+	}
 
 	// update the version with a path
 	sub = oas.Paths.Paths[repoprefix+"/files/{path}"]

pkg/registry/apis/provisioning/repository/local.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"log/slog"
 	"net/http"
 	"os"
@@ -119,19 +120,27 @@ func (r *localRepository) Test(ctx context.Context, logger *slog.Logger) error {
 	}
 }
 
-// ReadResource implements provisioning.Repository.
-func (r *localRepository) Read(ctx context.Context, logger *slog.Logger, path string, ref string) (*FileInfo, error) {
+// Test implements provisioning.Repository.
+func (r *localRepository) validateRequest(ref string) error {
 	if ref != "" {
-		return nil, apierrors.NewBadRequest("local repository does not support ref")
+		return apierrors.NewBadRequest("local repository does not support ref")
 	}
 	if r.path == "" {
-		return nil, &apierrors.StatusError{
+		return &apierrors.StatusError{
 			ErrStatus: metav1.Status{
 				Message: "the service is missing a root path",
 				Code:    http.StatusFailedDependency,
 			},
 		}
 	}
+	return nil
+}
+
+// ReadResource implements provisioning.Repository.
+func (r *localRepository) Read(ctx context.Context, logger *slog.Logger, path string, ref string) (*FileInfo, error) {
+	if err := r.validateRequest(ref); err != nil {
+		return nil, err
+	}
 
 	fullpath := filepath.Join(r.path, path)
 	info, err := os.Stat(fullpath)
@@ -164,59 +173,32 @@ func (r *localRepository) Read(ctx context.Context, logger *slog.Logger, path st
 
 // ReadResource implements provisioning.Repository.
 func (r *localRepository) ReadTree(ctx context.Context, logger *slog.Logger, ref string) ([]FileTreeEntry, error) {
-	if ref != "" {
-		return nil, apierrors.NewBadRequest("local repository does not support ref")
-	}
-	if r.path == "" {
-		return nil, &apierrors.StatusError{
-			ErrStatus: metav1.Status{
-				Message: "the service is missing a root path",
-				Code:    http.StatusFailedDependency,
-			},
-		}
-	}
-
-	return r.recursivelyReadTree(ctx, r.path)
-}
-
-func (r *localRepository) recursivelyReadTree(ctx context.Context, path string) ([]FileTreeEntry, error) {
-	osEntries, err := os.ReadDir(path)
-	if err != nil {
+	if err := r.validateRequest(ref); err != nil {
 		return nil, err
 	}
-	entries := make([]FileTreeEntry, 0, len(osEntries))
-	for _, oe := range osEntries {
-		if err := ctx.Err(); err != nil {
-			return nil, err
-		}
 
-		oePath := filepath.Join(path, oe.Name())
-		if oe.IsDir() {
-			entries = append(entries, FileTreeEntry{
-				Path: oePath,
-				Hash: "",
-				Size: 0,
-				Blob: false,
-			})
-			subEntries, err := r.recursivelyReadTree(ctx, oePath)
-			if err != nil {
-				return nil, err
-			}
-			entries = append(entries, subEntries...)
-		} else {
-			hash, size, err := r.calculateFileHash(ctx, oePath)
-			if err != nil {
-				return nil, fmt.Errorf("failed to read and calculate hash of path %s: %w", oePath, err)
-			}
-			entries = append(entries, FileTreeEntry{
-				Path: oePath,
-				Blob: true,
-				Hash: hash,
-				Size: size,
-			})
+	rootlen := len(r.path)
+	entries := make([]FileTreeEntry, 0, 100)
+	err := filepath.Walk(r.path, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
 		}
-	}
-	return entries, nil
+		entry := FileTreeEntry{
+			Path: strings.TrimLeft(path[rootlen:], "/"),
+			Size: info.Size(),
+		}
+		if !info.IsDir() {
+			entry.Blob = true
+			entry.Hash, _, err = r.calculateFileHash(ctx, path)
+			if err != nil {
+				return fmt.Errorf("failed to read and calculate hash of path %s: %w", path, err)
+			}
+		}
+		entries = append(entries, entry)
+		return err
+	})
+
+	return entries, err
 }
 
 func (r *localRepository) calculateFileHash(ctx context.Context, path string) (string, int64, error) {
@@ -237,17 +219,8 @@ func (r *localRepository) calculateFileHash(ctx context.Context, path string) (s
 }
 
 func (r *localRepository) Create(ctx context.Context, logger *slog.Logger, path string, ref string, data []byte, comment string) error {
-	if ref != "" {
-		return apierrors.NewBadRequest("local repository does not support ref")
-	}
-
-	if r.path == "" {
-		return &apierrors.StatusError{
-			ErrStatus: metav1.Status{
-				Message: "the service is missing a root path",
-				Code:    http.StatusFailedDependency,
-			},
-		}
+	if err := r.validateRequest(ref); err != nil {
+		return err
 	}
 
 	path = filepath.Join(r.path, path)
@@ -261,17 +234,8 @@ func (r *localRepository) Create(ctx context.Context, logger *slog.Logger, path
 }
 
 func (r *localRepository) Update(ctx context.Context, logger *slog.Logger, path string, ref string, data []byte, comment string) error {
-	if ref != "" {
-		return apierrors.NewBadRequest("local repository does not support ref")
-	}
-
-	if r.path == "" {
-		return &apierrors.StatusError{
-			ErrStatus: metav1.Status{
-				Message: "the service is missing a root path",
-				Code:    http.StatusFailedDependency,
-			},
-		}
+	if err := r.validateRequest(ref); err != nil {
+		return err
 	}
 
 	path = filepath.Join(r.path, path)
@@ -282,17 +246,8 @@ func (r *localRepository) Update(ctx context.Context, logger *slog.Logger, path
 }
 
 func (r *localRepository) Delete(ctx context.Context, logger *slog.Logger, path string, ref string, comment string) error {
-	if ref != "" {
-		return apierrors.NewBadRequest("local repository does not support ref")
-	}
-
-	if r.path == "" {
-		return &apierrors.StatusError{
-			ErrStatus: metav1.Status{
-				Message: "the service is missing a root path",
-				Code:    http.StatusFailedDependency,
-			},
-		}
+	if err := r.validateRequest(ref); err != nil {
+		return err
 	}
 
 	return os.Remove(filepath.Join(r.path, path))