diff --git a/pkg/apis/provisioning/v0alpha1/register.go b/pkg/apis/provisioning/v0alpha1/register.go index b556f45dfc9..ad99a42bdce 100644 --- a/pkg/apis/provisioning/v0alpha1/register.go +++ b/pkg/apis/provisioning/v0alpha1/register.go @@ -81,6 +81,7 @@ func AddKnownTypes(gv schema.GroupVersion, scheme *runtime.Scheme) error { &HelloWorld{}, &WebhookResponse{}, &ResourceWrapper{}, + &FileList{}, ) return nil } diff --git a/pkg/apis/provisioning/v0alpha1/types.go b/pkg/apis/provisioning/v0alpha1/types.go index 2893b78cecb..1d317263562 100644 --- a/pkg/apis/provisioning/v0alpha1/types.go +++ b/pkg/apis/provisioning/v0alpha1/types.go @@ -183,3 +183,22 @@ type WebhookResponse struct { Status string `json:"status,omitempty"` } + +// Information we can get just from the file listing +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type FileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + // should be named "items", but avoid subresource error for now: + // kubernetes/kubernetes#126809 + Items []FileItem `json:"files,omitempty"` +} + +type FileItem struct { + Path string `json:"path"` + Size int64 `json:"size,omitempty"` + Hash string `json:"hash,omitempty"` + Modified int64 `json:"modified,omitempty"` + Author string `json:"author,omitempty"` +} diff --git a/pkg/apis/provisioning/v0alpha1/zz_generated.deepcopy.go b/pkg/apis/provisioning/v0alpha1/zz_generated.deepcopy.go index 86d5ccab6f6..f6ddb992ed2 100644 --- a/pkg/apis/provisioning/v0alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/provisioning/v0alpha1/zz_generated.deepcopy.go @@ -27,6 +27,53 @@ func (in *EditingOptions) DeepCopy() *EditingOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileItem) DeepCopyInto(out *FileItem) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileItem. +func (in *FileItem) DeepCopy() *FileItem { + if in == nil { + return nil + } + out := new(FileItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileList) DeepCopyInto(out *FileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FileItem, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileList. +func (in *FileList) DeepCopy() *FileList { + if in == nil { + return nil + } + out := new(FileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitHubRepositoryConfig) DeepCopyInto(out *GitHubRepositoryConfig) { *out = *in diff --git a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go index afdfb17c722..be8be00ca70 100644 --- a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go +++ b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go @@ -15,6 +15,8 @@ import ( func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.EditingOptions": schema_pkg_apis_provisioning_v0alpha1_EditingOptions(ref), + "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileItem": schema_pkg_apis_provisioning_v0alpha1_FileItem(ref), + "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileList": schema_pkg_apis_provisioning_v0alpha1_FileList(ref), "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.GitHubRepositoryConfig": schema_pkg_apis_provisioning_v0alpha1_GitHubRepositoryConfig(ref), "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.HelloWorld": schema_pkg_apis_provisioning_v0alpha1_HelloWorld(ref), "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.LocalRepositoryConfig": schema_pkg_apis_provisioning_v0alpha1_LocalRepositoryConfig(ref), @@ -66,6 +68,99 @@ func schema_pkg_apis_provisioning_v0alpha1_EditingOptions(ref common.ReferenceCa } } +func schema_pkg_apis_provisioning_v0alpha1_FileItem(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "path": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "size": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int64", + }, + }, + "hash": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "modified": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int64", + }, + }, + "author": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"path"}, + }, + }, + } +} + +func schema_pkg_apis_provisioning_v0alpha1_FileList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Information we can get just from the file listing", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "files": { + SchemaProps: spec.SchemaProps{ + Description: "should be named \"items\", but avoid subresource error for now: kubernetes/kubernetes#126809", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileItem"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileItem", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"}, + } +} + func schema_pkg_apis_provisioning_v0alpha1_GitHubRepositoryConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list index 456e92d9f12..ba25c7c5c79 100644 --- a/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list +++ b/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list @@ -1,3 +1,4 @@ API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,ResourceWrapper,Errors +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,FileList,Items API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,GitHub API rule violation: names_match,github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1,RepositorySpec,PreferYAML diff --git a/pkg/registry/apis/provisioning/files.go b/pkg/registry/apis/provisioning/files.go index 0f6525a4f76..8e1cbb30801 100644 --- a/pkg/registry/apis/provisioning/files.go +++ b/pkg/registry/apis/provisioning/files.go @@ -65,7 +65,11 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - logger := logger.With("url", r.URL.Path) + query := r.URL.Query() + ref := query.Get("ref") + message := query.Get("message") + logger = logger.With("url", r.URL.Path, "ref", ref, "message", message) + prefix := fmt.Sprintf("/%s/files/", name) idx := strings.Index(r.URL.Path, prefix) if idx == -1 { @@ -75,9 +79,30 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. } filePath := r.URL.Path[idx+len(prefix):] - if filePath == "" { - logger.DebugContext(r.Context(), "got an empty file path") - responder.Error(apierrors.NewBadRequest("missing path")) + if filePath == "" || strings.HasSuffix(filePath, "/") { + if len(filePath) > 0 { + responder.Error(apierrors.NewBadRequest("folder navigation not yet supported")) + return + } + + rsp, err := repo.ReadTree(r.Context(), logger, ref) + if err != nil { + responder.Error(err) + return + } + + files := &provisioning.FileList{} + for _, v := range rsp { + if !v.Blob { + continue // folder item + } + files.Items = append(files.Items, provisioning.FileItem{ + Path: v.Path, + Size: v.Size, + Hash: v.Hash, + }) + } + responder.Object(http.StatusOK, files) return } @@ -97,10 +122,6 @@ func (s *filesConnector) Connect(ctx context.Context, name string, opts runtime. } var obj *provisioning.ResourceWrapper - ref := r.URL.Query().Get("ref") - message := r.URL.Query().Get("message") - logger = logger.With("ref", ref, "message", message) - code := http.StatusOK switch r.Method { case http.MethodGet: diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index b19a167be6f..78bcd65ecd0 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -153,17 +153,6 @@ func (b *ProvisioningAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserv logger: b.logger.With("connector", "hello_world"), } - exportConnector := &exportConnector{ - repoGetter: b, - client: b.client, - logger: b.logger.With("connector", "export"), - } - importConnector := &importConnector{ - repoGetter: b, - client: b.client, - logger: b.logger.With("connector", "import"), - } - storage := map[string]rest.Storage{} storage[provisioning.RepositoryResourceInfo.StoragePath()] = repositoryStorage // Can be used by kubectl: kubectl --kubeconfig grafana.kubeconfig patch Repository local-devenv --type=merge --subresource=status --patch='status: {"currentGitCommit": "hello"}' @@ -179,8 +168,16 @@ func (b *ProvisioningAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserv client: b.client, logger: b.logger.With("connector", "files"), } - storage[provisioning.RepositoryResourceInfo.StoragePath("export")] = exportConnector - storage[provisioning.RepositoryResourceInfo.StoragePath("import")] = importConnector + storage[provisioning.RepositoryResourceInfo.StoragePath("import")] = &importConnector{ + repoGetter: b, + client: b.client, + logger: b.logger.With("connector", "import"), + } + storage[provisioning.RepositoryResourceInfo.StoragePath("export")] = &exportConnector{ + repoGetter: b, + client: b.client, + logger: b.logger.With("connector", "export"), + } apiGroupInfo.VersionedResourcesStorageMap[provisioning.VERSION] = storage return nil } @@ -464,6 +461,8 @@ func (b *ProvisioningAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3. root := "/apis/" + b.GetGroupVersion().String() + "/" repoprefix := root + "namespaces/{namespace}/repositories/{name}" + defs := b.GetOpenAPIDefinitions()(func(path string) spec.Ref { return spec.Ref{} }) + // TODO: we might want to register some extras for subresources here. sub := oas.Paths.Paths[repoprefix+"/hello"] if sub != nil && sub.Get != nil { @@ -521,8 +520,23 @@ func (b *ProvisioningAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3. sub.Post.Parameters = []*spec3.Parameter{ref} } - // hide the version with no path - delete(oas.Paths.Paths, repoprefix+"/files") + // Show a special list command + sub = oas.Paths.Paths[repoprefix+"/files"] + if sub != nil { + delete(oas.Paths.Paths, repoprefix+"/files") + oas.Paths.Paths[repoprefix+"/files/"] = sub // add the trailing final slash + sub.Get.Description = "Get the files and content hash" + sub.Get.Summary = "File listing" + sub.Get.Parameters = []*spec3.Parameter{ref} + sub.Post = nil + sub.Put = nil + sub.Delete = nil + + // Replace the content type for this response + mt := sub.Get.Responses.StatusCodeResponses[200].Content + s := defs["github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1.FileList"].Schema + mt["*/*"].Schema = &s + } // update the version with a path sub = oas.Paths.Paths[repoprefix+"/files/{path}"] diff --git a/pkg/registry/apis/provisioning/repository/local.go b/pkg/registry/apis/provisioning/repository/local.go index 49992345c60..1b9262225bb 100644 --- a/pkg/registry/apis/provisioning/repository/local.go +++ b/pkg/registry/apis/provisioning/repository/local.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "io/fs" "log/slog" "net/http" "os" @@ -119,19 +120,27 @@ func (r *localRepository) Test(ctx context.Context, logger *slog.Logger) error { } } -// ReadResource implements provisioning.Repository. -func (r *localRepository) Read(ctx context.Context, logger *slog.Logger, path string, ref string) (*FileInfo, error) { +// Test implements provisioning.Repository. +func (r *localRepository) validateRequest(ref string) error { if ref != "" { - return nil, apierrors.NewBadRequest("local repository does not support ref") + return apierrors.NewBadRequest("local repository does not support ref") } if r.path == "" { - return nil, &apierrors.StatusError{ + return &apierrors.StatusError{ ErrStatus: metav1.Status{ Message: "the service is missing a root path", Code: http.StatusFailedDependency, }, } } + return nil +} + +// ReadResource implements provisioning.Repository. +func (r *localRepository) Read(ctx context.Context, logger *slog.Logger, path string, ref string) (*FileInfo, error) { + if err := r.validateRequest(ref); err != nil { + return nil, err + } fullpath := filepath.Join(r.path, path) info, err := os.Stat(fullpath) @@ -164,59 +173,32 @@ func (r *localRepository) Read(ctx context.Context, logger *slog.Logger, path st // ReadResource implements provisioning.Repository. func (r *localRepository) ReadTree(ctx context.Context, logger *slog.Logger, ref string) ([]FileTreeEntry, error) { - if ref != "" { - return nil, apierrors.NewBadRequest("local repository does not support ref") - } - if r.path == "" { - return nil, &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Message: "the service is missing a root path", - Code: http.StatusFailedDependency, - }, - } - } - - return r.recursivelyReadTree(ctx, r.path) -} - -func (r *localRepository) recursivelyReadTree(ctx context.Context, path string) ([]FileTreeEntry, error) { - osEntries, err := os.ReadDir(path) - if err != nil { + if err := r.validateRequest(ref); err != nil { return nil, err } - entries := make([]FileTreeEntry, 0, len(osEntries)) - for _, oe := range osEntries { - if err := ctx.Err(); err != nil { - return nil, err - } - oePath := filepath.Join(path, oe.Name()) - if oe.IsDir() { - entries = append(entries, FileTreeEntry{ - Path: oePath, - Hash: "", - Size: 0, - Blob: false, - }) - subEntries, err := r.recursivelyReadTree(ctx, oePath) - if err != nil { - return nil, err - } - entries = append(entries, subEntries...) - } else { - hash, size, err := r.calculateFileHash(ctx, oePath) - if err != nil { - return nil, fmt.Errorf("failed to read and calculate hash of path %s: %w", oePath, err) - } - entries = append(entries, FileTreeEntry{ - Path: oePath, - Blob: true, - Hash: hash, - Size: size, - }) + rootlen := len(r.path) + entries := make([]FileTreeEntry, 0, 100) + err := filepath.Walk(r.path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err } - } - return entries, nil + entry := FileTreeEntry{ + Path: strings.TrimLeft(path[rootlen:], "/"), + Size: info.Size(), + } + if !info.IsDir() { + entry.Blob = true + entry.Hash, _, err = r.calculateFileHash(ctx, path) + if err != nil { + return fmt.Errorf("failed to read and calculate hash of path %s: %w", path, err) + } + } + entries = append(entries, entry) + return err + }) + + return entries, err } func (r *localRepository) calculateFileHash(ctx context.Context, path string) (string, int64, error) { @@ -237,17 +219,8 @@ func (r *localRepository) calculateFileHash(ctx context.Context, path string) (s } func (r *localRepository) Create(ctx context.Context, logger *slog.Logger, path string, ref string, data []byte, comment string) error { - if ref != "" { - return apierrors.NewBadRequest("local repository does not support ref") - } - - if r.path == "" { - return &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Message: "the service is missing a root path", - Code: http.StatusFailedDependency, - }, - } + if err := r.validateRequest(ref); err != nil { + return err } path = filepath.Join(r.path, path) @@ -261,17 +234,8 @@ func (r *localRepository) Create(ctx context.Context, logger *slog.Logger, path } func (r *localRepository) Update(ctx context.Context, logger *slog.Logger, path string, ref string, data []byte, comment string) error { - if ref != "" { - return apierrors.NewBadRequest("local repository does not support ref") - } - - if r.path == "" { - return &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Message: "the service is missing a root path", - Code: http.StatusFailedDependency, - }, - } + if err := r.validateRequest(ref); err != nil { + return err } path = filepath.Join(r.path, path) @@ -282,17 +246,8 @@ func (r *localRepository) Update(ctx context.Context, logger *slog.Logger, path } func (r *localRepository) Delete(ctx context.Context, logger *slog.Logger, path string, ref string, comment string) error { - if ref != "" { - return apierrors.NewBadRequest("local repository does not support ref") - } - - if r.path == "" { - return &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Message: "the service is missing a root path", - Code: http.StatusFailedDependency, - }, - } + if err := r.validateRequest(ref); err != nil { + return err } return os.Remove(filepath.Join(r.path, path))