diff --git a/pkg/infra/filestorage/api.go b/pkg/infra/filestorage/api.go index 9d849cf4a85..3a4a5c285ba 100644 --- a/pkg/infra/filestorage/api.go +++ b/pkg/infra/filestorage/api.go @@ -15,6 +15,7 @@ var ( ErrPathInvalid = errors.New("path is invalid") ErrPathEndsWithDelimiter = errors.New("path can not end with delimiter") Delimiter = "/" + DirectoryMimeType = "directory" multipleDelimiters = regexp.MustCompile(`/+`) ) @@ -30,6 +31,10 @@ type File struct { FileMetadata } +func (f *File) IsFolder() bool { + return f.MimeType == DirectoryMimeType +} + type FileMetadata struct { Name string FullPath string @@ -40,12 +45,6 @@ type FileMetadata struct { Properties map[string]string } -type ListFilesResponse struct { - Files []FileMetadata - HasMore bool - LastPath string -} - type Paging struct { After string First int @@ -133,8 +132,17 @@ func (f *PathFilters) IsAllowed(path string) bool { return false } +type ListResponse struct { + Files []*File + HasMore bool + LastPath string +} + type ListOptions struct { - Recursive bool + Recursive bool + WithFiles bool + WithFolders bool + WithContents bool *PathFilters } @@ -143,8 +151,8 @@ type FileStorage interface { Delete(ctx context.Context, path string) error Upsert(ctx context.Context, command *UpsertFileCommand) error - ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) - ListFolders(ctx context.Context, folderPath string, options *ListOptions) ([]FileMetadata, error) + // List lists only files without content by default + List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) CreateFolder(ctx context.Context, path string) error DeleteFolder(ctx context.Context, path string) error diff --git a/pkg/infra/filestorage/cdk_blob_filestorage.go b/pkg/infra/filestorage/cdk_blob_filestorage.go index 1ece2574bee..a49ab1ebeec 100644 --- a/pkg/infra/filestorage/cdk_blob_filestorage.go +++ b/pkg/infra/filestorage/cdk_blob_filestorage.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "sort" "strings" "github.com/grafana/grafana/pkg/infra/log" @@ -134,145 +133,6 @@ func (c cdkBlobStorage) Upsert(ctx context.Context, command *UpsertFileCommand) }) } -func (c cdkBlobStorage) listFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) { - iterator := c.bucket.List(&blob.ListOptions{ - Prefix: strings.ToLower(folderPath), - Delimiter: Delimiter, - }) - - recursive := options.Recursive - - pageSize := paging.First - - foundCursor := true - if paging.After != "" { - foundCursor = false - } - - hasMore := true - files := make([]FileMetadata, 0) - for { - obj, err := iterator.Next(ctx) - if obj != nil && strings.HasSuffix(obj.Key, directoryMarker) { - continue - } - - if errors.Is(err, io.EOF) { - hasMore = false - break - } else { - hasMore = true - } - - if err != nil { - c.log.Error("Failed while iterating over files", "err", err) - return nil, err - } - - if len(files) >= pageSize { - break - } - - path := obj.Key - - allowed := options.IsAllowed(obj.Key) - if obj.IsDir && recursive { - newPaging := &Paging{ - First: pageSize - len(files), - } - if paging != nil { - newPaging.After = paging.After - } - - resp, err := c.listFiles(ctx, path, newPaging, options) - - if err != nil { - return nil, err - } - - if len(files) > 0 { - foundCursor = true - } - - files = append(files, resp.Files...) - if len(files) >= pageSize { - //nolint: staticcheck - hasMore = resp.HasMore - } - } else if !obj.IsDir && allowed { - if !foundCursor { - res := strings.Compare(obj.Key, paging.After) - if res < 0 { - continue - } else if res == 0 { - foundCursor = true - continue - } else { - foundCursor = true - } - } - - attributes, err := c.bucket.Attributes(ctx, strings.ToLower(path)) - if err != nil { - if gcerrors.Code(err) == gcerrors.NotFound { - attributes, err = c.bucket.Attributes(ctx, path) - if err != nil { - c.log.Error("Failed while retrieving attributes", "path", path, "err", err) - return nil, err - } - } else { - c.log.Error("Failed while retrieving attributes", "path", path, "err", err) - return nil, err - } - } - - if attributes.ContentType == "application/x-directory; charset=UTF-8" { - // S3 directory representation - continue - } - - if attributes.ContentType == "text/plain" && obj.Key == folderPath && attributes.Size == 0 { - // GCS directory representation - continue - } - - var originalPath string - var props map[string]string - if attributes.Metadata != nil { - props = attributes.Metadata - if path, ok := attributes.Metadata[originalPathAttributeKey]; ok { - originalPath = path - delete(props, originalPathAttributeKey) - } - } else { - props = make(map[string]string) - originalPath = strings.TrimSuffix(path, Delimiter) - } - - files = append(files, FileMetadata{ - Name: getName(originalPath), - FullPath: originalPath, - Created: attributes.CreateTime, - Properties: props, - Modified: attributes.ModTime, - Size: attributes.Size, - MimeType: detectContentType(originalPath, attributes.ContentType), - }) - } - } - - lastPath := "" - if len(files) > 0 { - lastPath = files[len(files)-1].FullPath - } - - return &ListFilesResponse{ - Files: files, - HasMore: hasMore, - LastPath: lastPath, - }, nil -} - func (c cdkBlobStorage) convertFolderPathToPrefix(path string) string { if path != "" && !strings.HasSuffix(path, Delimiter) { return path + Delimiter @@ -280,124 +140,6 @@ func (c cdkBlobStorage) convertFolderPathToPrefix(path string) string { return path } -func (c cdkBlobStorage) ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) { - prefix := c.convertFolderPathToPrefix(folderPath) - files, err := c.listFiles(ctx, prefix, paging, options) - return files, err -} - -func (c cdkBlobStorage) listFolderPaths(ctx context.Context, parentFolderPath string, options *ListOptions) ([]string, error) { - iterator := c.bucket.List(&blob.ListOptions{ - Prefix: strings.ToLower(parentFolderPath), - Delimiter: Delimiter, - }) - - recursive := options.Recursive - - dirPath := "" - dirMarkerPath := "" - foundPaths := make([]string, 0) - for { - obj, err := iterator.Next(ctx) - if errors.Is(err, io.EOF) { - break - } - - if err != nil { - c.log.Error("Failed while iterating over files", "err", err) - return nil, err - } - - if options.IsAllowed(obj.Key) { - if obj.IsDir && !recursive && options.IsAllowed(obj.Key) { - var nestedDirPath string - dirMPath := obj.Key + directoryMarker - attributes, err := c.bucket.Attributes(ctx, dirMPath) - if err != nil { - c.log.Error("Failed while retrieving attributes", "path", obj.Key, "err", err) - } - - if attributes != nil && attributes.Metadata != nil { - if path, ok := attributes.Metadata[originalPathAttributeKey]; ok { - nestedDirPath = getParentFolderPath(path) - } - } - - if nestedDirPath != "" { - foundPaths = append(foundPaths, nestedDirPath) - } else { - foundPaths = append(foundPaths, strings.TrimSuffix(obj.Key, Delimiter)) - } - } - - if dirPath == "" && !obj.IsDir { - dirPath = getParentFolderPath(obj.Key) - } - - if dirMarkerPath == "" && !obj.IsDir { - attributes, err := c.bucket.Attributes(ctx, obj.Key) - if err != nil { - c.log.Error("Failed while retrieving attributes", "path", obj.Key, "err", err) - return nil, err - } - - if attributes.Metadata != nil { - if path, ok := attributes.Metadata[originalPathAttributeKey]; ok { - dirMarkerPath = getParentFolderPath(path) - } - } - } - } - - if obj.IsDir && recursive { - resp, err := c.listFolderPaths(ctx, obj.Key, options) - - if err != nil { - return nil, err - } - - if len(resp) > 0 { - foundPaths = append(foundPaths, resp...) - } - continue - } - } - - var foundPath string - if dirMarkerPath != "" { - foundPath = dirMarkerPath - } else if dirPath != "" { - foundPath = dirPath - } - - if foundPath != "" && options.IsAllowed(foundPath+Delimiter) { - foundPaths = append(foundPaths, foundPath) - } - return foundPaths, nil -} - -func (c cdkBlobStorage) ListFolders(ctx context.Context, prefix string, options *ListOptions) ([]FileMetadata, error) { - fixedPrefix := c.convertFolderPathToPrefix(prefix) - foundPaths, err := c.listFolderPaths(ctx, fixedPrefix, options) - if err != nil { - return nil, err - } - - sort.Strings(foundPaths) - folders := make([]FileMetadata, 0) - - for _, path := range foundPaths { - if strings.Compare(path, fixedPrefix) > 0 { - folders = append(folders, FileMetadata{ - Name: getName(path), - FullPath: path, - }) - } - } - - return folders, err -} - func precedingFolders(path string) []string { parts := strings.Split(path, Delimiter) if len(parts) == 0 { @@ -489,6 +231,186 @@ func (c cdkBlobStorage) DeleteFolder(ctx context.Context, folderPath string) err return err } +//nolint: gocyclo +func (c cdkBlobStorage) list(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) { + lowerRootPath := strings.ToLower(folderPath) + iterators := []*blob.ListIterator{c.bucket.List(&blob.ListOptions{ + Prefix: lowerRootPath, + Delimiter: Delimiter, + })} + + recursive := options.Recursive + pageSize := paging.First + + foundCursor := true + if paging.After != "" { + foundCursor = false + } + + files := make([]*File, 0) + + visitedFolders := map[string]bool{} + visitedFolders[lowerRootPath] = true + + for len(iterators) > 0 && len(files) <= pageSize { + obj, err := iterators[0].Next(ctx) + if errors.Is(err, io.EOF) { + iterators = iterators[1:] + continue + } + + if err != nil { + c.log.Error("Failed while iterating over files", "err", err) + return nil, err + } + + path := obj.Key + lowerPath := strings.ToLower(path) + allowed := options.IsAllowed(lowerPath) + + if obj.IsDir && recursive && !visitedFolders[lowerPath] { + iterators = append([]*blob.ListIterator{c.bucket.List(&blob.ListOptions{ + Prefix: lowerPath, + Delimiter: Delimiter, + })}, iterators...) + visitedFolders[lowerPath] = true + } + + if !foundCursor { + res := strings.Compare(strings.TrimSuffix(lowerPath, Delimiter), paging.After) + if res < 0 { + continue + } else if res == 0 { + foundCursor = true + continue + } else { + foundCursor = true + } + } + + if obj.IsDir { + if options.WithFolders && allowed { + originalCasingPath := "" + dirMarkerPath := obj.Key + directoryMarker + attributes, err := c.bucket.Attributes(ctx, dirMarkerPath) + if err == nil && attributes != nil && attributes.Metadata != nil { + if path, ok := attributes.Metadata[originalPathAttributeKey]; ok { + originalCasingPath = getParentFolderPath(path) + } + } + + var p string + if originalCasingPath != "" { + p = originalCasingPath + } else { + p = strings.TrimSuffix(obj.Key, Delimiter) + } + + files = append(files, &File{ + Contents: nil, + FileMetadata: FileMetadata{ + MimeType: DirectoryMimeType, + Name: getName(p), + Properties: map[string]string{}, + FullPath: p, + }, + }) + } + continue + } + + if strings.HasSuffix(obj.Key, directoryMarker) { + continue + } + + if options.WithFiles && allowed { + attributes, err := c.bucket.Attributes(ctx, strings.ToLower(path)) + if err != nil { + if gcerrors.Code(err) == gcerrors.NotFound { + attributes, err = c.bucket.Attributes(ctx, path) + if err != nil { + c.log.Error("Failed while retrieving attributes", "path", path, "err", err) + return nil, err + } + } else { + c.log.Error("Failed while retrieving attributes", "path", path, "err", err) + return nil, err + } + } + + if attributes.ContentType == "application/x-directory; charset=UTF-8" { + // S3 directory representation + continue + } + + if attributes.ContentType == "text/plain" && obj.Key == folderPath && attributes.Size == 0 { + // GCS directory representation + continue + } + + var originalPath string + var props map[string]string + if attributes.Metadata != nil { + props = attributes.Metadata + if path, ok := attributes.Metadata[originalPathAttributeKey]; ok { + originalPath = path + delete(props, originalPathAttributeKey) + } + } else { + props = make(map[string]string) + originalPath = strings.TrimSuffix(path, Delimiter) + } + + var contents []byte + if options.WithContents { + c, err := c.bucket.ReadAll(ctx, lowerPath) + if err != nil && gcerrors.Code(err) != gcerrors.NotFound { + return nil, err + } + + if c != nil { + contents = c + } + } + + files = append(files, &File{ + Contents: contents, + FileMetadata: FileMetadata{ + Name: getName(originalPath), + FullPath: originalPath, + Created: attributes.CreateTime, + Properties: props, + Modified: attributes.ModTime, + Size: attributes.Size, + MimeType: detectContentType(originalPath, attributes.ContentType), + }, + }) + } + } + + hasMore := false + if len(files) > pageSize { + hasMore = true + files = files[:len(files)-pageSize] + } + + lastPath := "" + if len(files) > 0 { + lastPath = files[len(files)-1].FullPath + } + + return &ListResponse{ + Files: files, + HasMore: hasMore, + LastPath: lastPath, + }, nil +} + +func (c cdkBlobStorage) List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) { + prefix := c.convertFolderPathToPrefix(folderPath) + return c.list(ctx, prefix, paging, options) +} + func (c cdkBlobStorage) close() error { return c.bucket.Close() } diff --git a/pkg/infra/filestorage/db_filestorage.go b/pkg/infra/filestorage/db_filestorage.go index 0b7dde660f2..06cfba9c1f7 100644 --- a/pkg/infra/filestorage/db_filestorage.go +++ b/pkg/infra/filestorage/db_filestorage.go @@ -2,7 +2,6 @@ package filestorage import ( "context" - "fmt" "strings" "time" @@ -160,7 +159,7 @@ func (s dbFileStorage) Upsert(ctx context.Context, cmd *UpsertFileCommand) error file := &file{ Path: cmd.Path, - ParentFolderPath: getParentFolderPath(cmd.Path), + ParentFolderPath: strings.ToLower(getParentFolderPath(cmd.Path)), Contents: contentsToInsert, MimeType: cmd.MimeType, Size: int64(len(contentsToInsert)), @@ -223,33 +222,85 @@ func upsertProperty(sess *sqlstore.DBSession, now time.Time, path string, key st return err } -func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) { - var resp *ListFilesResponse +//nolint: gocyclo +func (s dbFileStorage) List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) { + var resp *ListResponse err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - var foundFiles = make([]*file, 0) - - sess.Table("file") - lowerFolderPath := strings.ToLower(folderPath) - if options.Recursive { - var nestedFolders string - if folderPath == Delimiter { - nestedFolders = "%" + cursor := "" + if paging != nil && paging.After != "" { + exists, err := sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(paging.After)+Delimiter).Cols("mime_type").Exist() + if err != nil { + return err + } + if exists { + cursor = paging.After + Delimiter } else { - nestedFolders = fmt.Sprintf("%s%s%s", lowerFolderPath, Delimiter, "%") + cursor = paging.After } - sess.Where("(LOWER(parent_folder_path) = ?) OR (LOWER(parent_folder_path) LIKE ?)", lowerFolderPath, nestedFolders) - } else { - sess.Where("LOWER(parent_folder_path) = ?", lowerFolderPath) } - sess.Where("LOWER(path) NOT LIKE ?", fmt.Sprintf("%s%s%s", "%", Delimiter, directoryMarker)) - if options.PathFilters.isDenyAll() { - sess.Where("1 == 2") + var foundFiles = make([]*file, 0) + sess.Table("file") + lowerFolderPrefix := "" + lowerFolderPath := strings.ToLower(folderPath) + if lowerFolderPath == "" || lowerFolderPath == Delimiter { + lowerFolderPrefix = Delimiter + lowerFolderPath = Delimiter } else { + lowerFolderPath = strings.TrimSuffix(lowerFolderPath, Delimiter) + lowerFolderPrefix = lowerFolderPath + Delimiter + } + + sess.Where("LOWER(path) != ?", lowerFolderPrefix) + + if !options.Recursive { + sess.Where("parent_folder_path = ?", lowerFolderPath) + } else { + sess.Where("(parent_folder_path = ?) OR (parent_folder_path LIKE ?)", lowerFolderPath, lowerFolderPrefix+"%") + } + + if !options.WithFolders && options.WithFiles { + sess.Where("path NOT LIKE ?", "%/") + } + + if options.WithFolders && !options.WithFiles { + sess.Where("path LIKE ?", "%/") + } + + if len(options.PathFilters.allowedPrefixes)+len(options.PathFilters.allowedPaths) > 0 { + queries := make([]string, 0) + args := make([]interface{}, 0) for _, prefix := range options.PathFilters.allowedPrefixes { - sess.Where("LOWER(path) LIKE ?", fmt.Sprintf("%s%s", strings.ToLower(prefix), "%")) + queries = append(queries, "LOWER(path) LIKE ?") + args = append(args, prefix+"%") } + + for _, path := range options.PathFilters.allowedPaths { + queries = append(queries, "LOWER(path) = ?") + args = append(args, path) + } + sess.Where(strings.Join(queries, " OR "), args...) + } + + if options.PathFilters.disallowedPrefixes != nil && len(options.PathFilters.disallowedPrefixes) > 0 { + queries := make([]string, 0) + args := make([]interface{}, 0) + for _, prefix := range options.PathFilters.disallowedPrefixes { + queries = append(queries, "LOWER(path) NOT LIKE ?") + args = append(args, prefix+"%") + } + sess.Where(strings.Join(queries, " AND "), args...) + } + + if options.PathFilters.disallowedPaths != nil && len(options.PathFilters.disallowedPaths) > 0 { + queries := make([]string, 0) + args := make([]interface{}, 0) + for _, path := range options.PathFilters.disallowedPaths { + queries = append(queries, "LOWER(path) != ?") + args = append(args, path) + } + sess.Where(strings.Join(queries, " AND "), args...) } sess.OrderBy("path") @@ -257,8 +308,8 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging pageSize := paging.First sess.Limit(pageSize + 1) - if paging != nil && paging.After != "" { - sess.Where("path > ?", paging.After) + if cursor != "" { + sess.Where("path > ?", cursor) } if err := sess.Find(&foundFiles); err != nil { @@ -272,24 +323,33 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging lowerCasePaths := make([]string, 0) for i := 0; i < foundLength; i++ { - lowerCasePaths = append(lowerCasePaths, strings.ToLower(foundFiles[i].Path)) + isFolder := strings.HasSuffix(foundFiles[i].Path, Delimiter) + if !isFolder { + lowerCasePaths = append(lowerCasePaths, strings.ToLower(foundFiles[i].Path)) + } } propertiesByLowerPath, err := s.getProperties(sess, lowerCasePaths) if err != nil { return err } - files := make([]FileMetadata, 0) + files := make([]*File, 0) for i := 0; i < foundLength; i++ { var props map[string]string - path := foundFiles[i].Path + path := strings.TrimSuffix(foundFiles[i].Path, Delimiter) if foundProps, ok := propertiesByLowerPath[strings.ToLower(path)]; ok { props = foundProps } else { props = make(map[string]string) } - files = append(files, FileMetadata{ + var contents []byte + if options.WithContents { + contents = foundFiles[i].Contents + } else { + contents = []byte{} + } + files = append(files, &File{Contents: contents, FileMetadata: FileMetadata{ Name: getName(path), FullPath: path, Created: foundFiles[i].Created, @@ -297,7 +357,7 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging Modified: foundFiles[i].Updated, Size: foundFiles[i].Size, MimeType: foundFiles[i].MimeType, - }) + }}) } lastPath := "" @@ -305,7 +365,7 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging lastPath = files[len(files)-1].FullPath } - resp = &ListFilesResponse{ + resp = &ListResponse{ Files: files, LastPath: lastPath, HasMore: len(foundFiles) == pageSize+1, @@ -316,67 +376,6 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging return resp, err } -func (s dbFileStorage) ListFolders(ctx context.Context, parentFolderPath string, options *ListOptions) ([]FileMetadata, error) { - folders := make([]FileMetadata, 0) - - parentFolderPath = strings.TrimSuffix(parentFolderPath, Delimiter) - err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - var foundPaths []string - - sess.Table("file") - sess.Distinct("parent_folder_path") - - if options.Recursive { - sess.Where("LOWER(parent_folder_path) > ?", strings.ToLower(parentFolderPath)) - } else { - sess.Where("LOWER(parent_folder_path) LIKE ? AND LOWER(parent_folder_path) NOT LIKE ?", strings.ToLower(parentFolderPath)+Delimiter+"%", strings.ToLower(parentFolderPath)+Delimiter+"%"+Delimiter+"%") - } - - if options.PathFilters.isDenyAll() { - sess.Where("1 == 2") - } else { - for _, prefix := range options.PathFilters.allowedPrefixes { - sess.Where("LOWER(parent_folder_path) LIKE ?", fmt.Sprintf("%s%s", strings.ToLower(prefix), "%")) - } - } - - sess.OrderBy("parent_folder_path") - sess.Cols("parent_folder_path") - - if err := sess.Find(&foundPaths); err != nil { - return err - } - - mem := make(map[string]bool) - for i := 0; i < len(foundPaths); i++ { - path := foundPaths[i] - parts := strings.Split(path, Delimiter) - acc := parts[0] - j := 1 - for { - acc = fmt.Sprintf("%s%s%s", acc, Delimiter, parts[j]) - comparison := strings.Compare(acc, parentFolderPath) - if !mem[acc] && comparison > 0 { - folders = append(folders, FileMetadata{ - Name: getName(acc), - FullPath: acc, - }) - } - mem[acc] = true - - j += 1 - if j >= len(parts) { - break - } - } - } - - return nil - }) - - return folders, err -} - func (s dbFileStorage) CreateFolder(ctx context.Context, path string) error { now := time.Now() precedingFolders := precedingFolders(path) @@ -384,29 +383,32 @@ func (s dbFileStorage) CreateFolder(ctx context.Context, path string) error { err := s.db.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error { var insertErr error sess.MustLogSQL(true) - previousFolder := "" + previousFolder := Delimiter for i := 0; i < len(precedingFolders); i++ { existing := &file{} - directoryMarkerParentPath := previousFolder + Delimiter + getName(precedingFolders[i]) - previousFolder = directoryMarkerParentPath - directoryMarkerPath := fmt.Sprintf("%s%s%s", directoryMarkerParentPath, Delimiter, directoryMarker) - lower := strings.ToLower(directoryMarkerPath) - exists, err := sess.Table("file").Where("LOWER(path) = ?", lower).Get(existing) + currentFolderParentPath := previousFolder + previousFolder = Join(previousFolder, getName(precedingFolders[i])) + currentFolderPath := previousFolder + if !strings.HasSuffix(currentFolderPath, Delimiter) { + currentFolderPath = currentFolderPath + Delimiter + } + exists, err := sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(currentFolderPath)).Get(existing) if err != nil { insertErr = err break } if exists { - previousFolder = existing.ParentFolderPath + previousFolder = strings.TrimSuffix(existing.Path, Delimiter) continue } file := &file{ - Path: strings.ToLower(directoryMarkerPath), - ParentFolderPath: directoryMarkerParentPath, + Path: currentFolderPath, + ParentFolderPath: strings.ToLower(currentFolderParentPath), Contents: make([]byte, 0), Updated: now, + MimeType: DirectoryMimeType, Created: now, } _, err = sess.Insert(file) @@ -433,8 +435,8 @@ func (s dbFileStorage) CreateFolder(ctx context.Context, path string) error { func (s dbFileStorage) DeleteFolder(ctx context.Context, folderPath string) error { err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { existing := &file{} - directoryMarkerPath := fmt.Sprintf("%s%s%s", folderPath, Delimiter, directoryMarker) - exists, err := sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(directoryMarkerPath)).Get(existing) + internalFolderPath := strings.ToLower(folderPath) + Delimiter + exists, err := sess.Table("file").Where("LOWER(path) = ?", internalFolderPath).Get(existing) if err != nil { return err } @@ -443,7 +445,7 @@ func (s dbFileStorage) DeleteFolder(ctx context.Context, folderPath string) erro return nil } - _, err = sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(directoryMarkerPath)).Delete(existing) + _, err = sess.Table("file").Where("LOWER(path) = ?", internalFolderPath).Delete(existing) return err }) diff --git a/pkg/infra/filestorage/fs_integration_test.go b/pkg/infra/filestorage/fs_integration_test.go index 8e9957ffe71..a9b8b21581c 100644 --- a/pkg/infra/filestorage/fs_integration_test.go +++ b/pkg/infra/filestorage/fs_integration_test.go @@ -70,6 +70,12 @@ func runTests(createCases func() []fsTestCase, t *testing.T) { // sqlStore = sqlstore.InitTestDB(t) // filestorage = NewDbStorage(testLogger, sqlStore, nil, "/") //} + // + //setupSqlFSNestedPath := func() { + // commonSetup() + // sqlStore = sqlstore.InitTestDB(t) + // filestorage = NewDbStorage(testLogger, sqlStore, nil, "/dashboards/") + //} setupLocalFs := func() { commonSetup() @@ -127,6 +133,10 @@ func runTests(createCases func() []fsTestCase, t *testing.T) { // setup: setupSqlFS, // name: "SQL FS", //}, + //{ + // setup: setupSqlFSNestedPath, + // name: "SQL FS with nested path", + //}, } for _, backend := range backends { @@ -180,11 +190,28 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}}, + list: checks(listSize(4), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), + files: [][]interface{}{ + checks(fPath("/folder1/file-inner.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), + checks(fPath("/folder1/file-inner2.jpg"), fProperties(map[string]string{})), + checks(fPath("/folder1/folder2"), fProperties(map[string]string{}), fMimeType(DirectoryMimeType)), + checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: false}}, list: checks(listSize(0), listHasMore(false), listLastPath("")), files: [][]interface{}{}, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder1")), + files: [][]interface{}{ + checks(fPath("/folder1"), fProperties(map[string]string{}), fMimeType(DirectoryMimeType)), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: false}}, list: checks(listSize(2), listHasMore(false), listLastPath("/folder1/file-inner2.jpg")), @@ -193,6 +220,15 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/file-inner2.jpg"), fProperties(map[string]string{})), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}}, + list: checks(listSize(3), listHasMore(false), listLastPath("/folder1/folder2")), + files: [][]interface{}{ + checks(fPath("/folder1/file-inner.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), + checks(fPath("/folder1/file-inner2.jpg"), fProperties(map[string]string{})), + checks(fPath("/folder1/folder2"), fProperties(map[string]string{}), fMimeType(DirectoryMimeType)), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false}}, list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), @@ -200,11 +236,23 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), + files: [][]interface{}{ + checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false}, paging: &Paging{After: "/folder1/folder2/file.jpg"}}, list: checks(listSize(0), listHasMore(false), listLastPath("")), files: [][]interface{}{}, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}, paging: &Paging{After: "/folder1/folder2/file.jpg"}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + files: [][]interface{}{}, + }, }, }, { @@ -242,6 +290,15 @@ func TestFsStorage(t *testing.T) { checks(fPath("/ab/a/a.jpg")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/ab", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}}, + list: checks(listSize(3), listHasMore(false), listLastPath("/ab/a/a.jpg")), + files: [][]interface{}{ + checks(fPath("/ab/a.jpg")), + checks(fPath("/ab/a"), fMimeType(DirectoryMimeType)), + checks(fPath("/ab/a/a.jpg")), + }, + }, }, }, { @@ -265,10 +322,18 @@ func TestFsStorage(t *testing.T) { input: queryListFilesInput{path: "/folder1/file-inner.jp", options: &ListOptions{Recursive: true}}, list: checks(listSize(0), listHasMore(false), listLastPath("")), }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/file-inner.jp", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + }, queryListFiles{ input: queryListFilesInput{path: "/folder1/file-inner", options: &ListOptions{Recursive: true}}, list: checks(listSize(0), listHasMore(false), listLastPath("")), }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/file-inner", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + }, queryListFiles{ input: queryListFilesInput{path: "/folder1/folder2/file.jpg", options: &ListOptions{Recursive: true}}, list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), @@ -276,6 +341,13 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/folder2/file.jpg"), fName("file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/folder2/file.jpg", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), + files: [][]interface{}{ + checks(fPath("/folder1/folder2/file.jpg"), fName("file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/folder1/file-inner.jpg", options: &ListOptions{Recursive: true}}, list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/file-inner.jpg")), @@ -283,6 +355,13 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/file-inner.jpg"), fName("file-inner.jpg"), fProperties(map[string]string{"prop1": "val1"})), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/file-inner.jpg", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/file-inner.jpg")), + files: [][]interface{}{ + checks(fPath("/folder1/file-inner.jpg"), fName("file-inner.jpg"), fProperties(map[string]string{"prop1": "val1"})), + }, + }, }, }, { @@ -304,6 +383,11 @@ func TestFsStorage(t *testing.T) { input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder2"}}}}, list: checks(listSize(0), listHasMore(false), listLastPath("")), }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder2"}}}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + files: [][]interface{}{}, + }, queryListFiles{ input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}}, list: checks(listSize(1), listHasMore(false)), @@ -311,6 +395,14 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/folder2/file.jpg")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}}, + list: checks(listSize(2), listHasMore(false)), + files: [][]interface{}{ + checks(fPath("/folder1/folder2"), fMimeType("directory")), + checks(fPath("/folder1/folder2/file.jpg")), + }, + }, }, }, { @@ -341,6 +433,20 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/a")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: ""}}, + list: checks(listSize(1), listHasMore(true), listLastPath("/folder1")), + files: [][]interface{}{ + checks(fPath("/folder1"), fMimeType(DirectoryMimeType)), + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1"}}, + list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")), + files: [][]interface{}{ + checks(fPath("/folder1/a")), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/a"}}, list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")), @@ -348,6 +454,13 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/b")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1/a"}}, + list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")), + files: [][]interface{}{ + checks(fPath("/folder1/b")), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/b"}}, list: checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")), @@ -355,6 +468,20 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder2/c")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1/b"}}, + list: checks(listSize(1), listHasMore(true), listLastPath("/folder2")), + files: [][]interface{}{ + checks(fPath("/folder2"), fMimeType(DirectoryMimeType)), + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder2"}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")), + files: [][]interface{}{ + checks(fPath("/folder2/c")), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: ""}}, list: checks(listSize(3), listHasMore(false), listLastPath("/folder2/c")), @@ -364,14 +491,33 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder2/c")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: ""}}, + list: checks(listSize(5), listHasMore(false), listLastPath("/folder2/c")), + files: [][]interface{}{ + checks(fPath("/folder1"), fMimeType(DirectoryMimeType)), + checks(fPath("/folder1/a")), + checks(fPath("/folder1/b")), + checks(fPath("/folder2"), fMimeType(DirectoryMimeType)), + checks(fPath("/folder2/c")), + }, + }, queryListFiles{ input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2"}}, list: checks(listSize(1), listHasMore(false)), }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: "/folder2"}}, + list: checks(listSize(1), listHasMore(false)), + }, queryListFiles{ input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2/c"}}, list: checks(listSize(0), listHasMore(false)), }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: "/folder2/c"}}, + list: checks(listSize(0), listHasMore(false)), + }, }, }, } @@ -417,6 +563,18 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folderX/folderZ")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: false, WithFolders: true}}, + list: checks(listSize(6), listHasMore(false), listLastPath("/folderX/folderZ")), + files: [][]interface{}{ + checks(fPath("/folder1"), fMimeType(DirectoryMimeType)), + checks(fPath("/folder1/folder2"), fMimeType(DirectoryMimeType)), + checks(fPath("/folderA"), fMimeType(DirectoryMimeType)), + checks(fPath("/folderA/folderB"), fMimeType(DirectoryMimeType)), + checks(fPath("/folderX"), fMimeType(DirectoryMimeType)), + checks(fPath("/folderX/folderZ"), fMimeType(DirectoryMimeType)), + }, + }, }, }, { @@ -452,10 +610,27 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folder1/folder2")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2")), + files: [][]interface{}{ + checks(fPath("/folder1/folder2"), fMimeType(DirectoryMimeType)), + }, + }, queryListFolders{ input: queryListFoldersInput{path: "/folderZ", options: &ListOptions{Recursive: false}}, checks: [][]interface{}{}, }, + queryListFiles{ + input: queryListFilesInput{path: "/folderZ", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + files: [][]interface{}{}, + }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + files: [][]interface{}{}, + }, queryListFolders{ input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: false}}, checks: [][]interface{}{ @@ -464,6 +639,15 @@ func TestFsStorage(t *testing.T) { checks(fPath("/folderX")), }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}}, + list: checks(listSize(3), listHasMore(false), listLastPath("/folderX")), + files: [][]interface{}{ + checks(fPath("/folder1"), fMimeType(DirectoryMimeType)), + checks(fPath("/folderA"), fMimeType(DirectoryMimeType)), + checks(fPath("/folderX"), fMimeType(DirectoryMimeType)), + }, + }, }, }, } @@ -1047,6 +1231,31 @@ func TestFsStorage(t *testing.T) { // /s3/folder/nested/dashboard.json is denied with '/s3/folder/nested/' prefix }, }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{ + Recursive: true, + PathFilters: pathFilters, + WithFiles: true, + WithFolders: true, + }}, + list: checks(listSize(10), listHasMore(false), listLastPath("/s3/folder/dashboard.json")), + files: [][]interface{}{ + // /gitA/dashboard.json is not explicitly allowed + checks(fPath("/gitA/dashboard2.json")), // explicitly allowed by allowedPath + checks(fPath("/gitB")), // allowed by '/gitB/' prefix + checks(fPath("/gitB/nested")), // allowed by '/gitB/' prefix + checks(fPath("/gitB/nested/dashboard.json")), // allowed by '/gitB/' prefix + checks(fPath("/gitB/nested2")), // allowed by '/gitB/' prefix + checks(fPath("/gitB/nested2/dashboard2.json")), // allowed by '/gitB/' prefix + checks(fPath("/gitC")), // allowed by '/gitC/' prefix + // /gitC/nestedC is explicitly denied + checks(fPath("/gitC/nestedC/dashboardC.json")), // allowed by '/gitC/' prefix + // /s3 is not explicitly allowed + checks(fPath("/s3/folder")), + checks(fPath("/s3/folder/dashboard.json")), // allowed by '/s3/folder/' prefix + // /s3/folder/nested/dashboard.json is denied with '/s3/folder/nested/' prefix + }, + }, queryListFolders{ input: queryListFoldersInput{path: "/", options: &ListOptions{ Recursive: true, diff --git a/pkg/infra/filestorage/test_utils.go b/pkg/infra/filestorage/test_utils.go index 2ecbb1054aa..695a774a68e 100644 --- a/pkg/infra/filestorage/test_utils.go +++ b/pkg/infra/filestorage/test_utils.go @@ -137,6 +137,7 @@ type queryListFiles struct { type queryListFoldersInput struct { path string + paging *Paging options *ListOptions } @@ -221,7 +222,7 @@ func runChecks(t *testing.T, stepName string, path string, output interface{}, c } switch o := output.(type) { - case File: + case *File: for _, check := range checks { checkName := interfaceName(check) if fileContentsCheck, ok := check.(fileContentsCheck); ok { @@ -234,7 +235,7 @@ func runChecks(t *testing.T, stepName string, path string, output interface{}, c for _, check := range checks { runFileMetadataCheck(o, check, interfaceName(check)) } - case ListFilesResponse: + case ListResponse: for _, check := range checks { c := check checkName := interfaceName(c) @@ -249,13 +250,14 @@ func runChecks(t *testing.T, stepName string, path string, output interface{}, c t.Fatalf("unrecognized list check %s", checkName) } } + default: t.Fatalf("unrecognized output %s", interfaceName(output)) } } -func formatPathStructure(files []FileMetadata) string { +func formatPathStructure(files []*File) string { if len(files) == 0 { return "<>" } @@ -278,13 +280,13 @@ func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName if q.checks != nil && len(q.checks) > 0 { require.NotNil(t, file, "%s %s", queryName, inputPath) require.Equal(t, strings.ToLower(inputPath), strings.ToLower(file.FullPath), "%s %s", queryName, inputPath) - runChecks(t, queryName, inputPath, *file, q.checks) + runChecks(t, queryName, inputPath, file, q.checks) } else { require.Nil(t, file, "%s %s", queryName, inputPath) } case queryListFiles: inputPath := q.input.path - resp, err := fs.ListFiles(ctx, inputPath, q.input.paging, q.input.options) + resp, err := fs.List(ctx, inputPath, q.input.paging, q.input.options) require.NoError(t, err, "%s: should be able to list files in %s", queryName, inputPath) require.NotNil(t, resp) if q.list != nil && len(q.list) > 0 { @@ -304,17 +306,33 @@ func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName } case queryListFolders: inputPath := q.input.path - resp, err := fs.ListFolders(ctx, inputPath, q.input.options) + opts := q.input.options + if opts == nil { + opts = &ListOptions{ + Recursive: true, + WithFiles: false, + WithFolders: true, + WithContents: false, + PathFilters: nil, + } + } else { + opts.WithFolders = true + opts.WithFiles = false + } + resp, err := fs.List(ctx, inputPath, &Paging{ + After: "", + First: 100000, + }, opts) require.NotNil(t, resp) require.NoError(t, err, "%s: should be able to list folders in %s", queryName, inputPath) if q.checks != nil { - require.Equal(t, len(resp), len(q.checks), "%s: expected a check for each actual folder at path: \"%s\". actual: %s", queryName, inputPath, formatPathStructure(resp)) - for i, file := range resp { + require.Equal(t, len(resp.Files), len(q.checks), "%s: expected a check for each actual folder at path: \"%s\". actual: %s", queryName, inputPath, formatPathStructure(resp.Files)) + for i, file := range resp.Files { runChecks(t, queryName, inputPath, file, q.checks[i]) } } else { - require.Equal(t, 0, len(resp), "%s %s", queryName, inputPath) + require.Equal(t, 0, len(resp.Files), "%s %s", queryName, inputPath) } default: t.Fatalf("unrecognized query %s", queryName) diff --git a/pkg/infra/filestorage/wrapper.go b/pkg/infra/filestorage/wrapper.go index 07b4c62641e..b9718f6a0ed 100644 --- a/pkg/infra/filestorage/wrapper.go +++ b/pkg/infra/filestorage/wrapper.go @@ -202,6 +202,10 @@ func (b wrapper) Get(ctx context.Context, path string) (*File, error) { return nil, nil } + if b.rootFolder == rootedPath { + return nil, nil + } + file, err := b.wrapped.Get(ctx, rootedPath) if file != nil { file.FullPath = b.removeRoot(file.FullPath) @@ -276,90 +280,41 @@ func (b wrapper) pagingOptionsWithDefaults(paging *Paging) *Paging { return paging } -func (b wrapper) listOptionsWithDefaults(options *ListOptions, folderQuery bool) *ListOptions { +func (b wrapper) listOptionsWithDefaults(options *ListOptions) *ListOptions { if options == nil { - options = &ListOptions{} - options.Recursive = folderQuery - options.PathFilters = b.pathFilters - return &ListOptions{ - Recursive: folderQuery, - PathFilters: b.pathFilters, + Recursive: false, + PathFilters: b.pathFilters, + WithFiles: true, + WithFolders: false, + WithContents: false, } } + withFiles := options.WithFiles + if !options.WithFiles && !options.WithFolders { + withFiles = true + } if options.PathFilters == nil { return &ListOptions{ - Recursive: options.Recursive, - PathFilters: b.pathFilters, + Recursive: options.Recursive, + PathFilters: b.pathFilters, + WithFiles: withFiles, + WithFolders: options.WithFolders, + WithContents: options.WithContents, } } rootedFilters := addRootFolderToFilters(copyPathFilters(options.PathFilters), b.rootFolder) return &ListOptions{ - Recursive: options.Recursive, - PathFilters: addPathFilters(rootedFilters, b.pathFilters), + Recursive: options.Recursive, + PathFilters: addPathFilters(rootedFilters, b.pathFilters), + WithFiles: withFiles, + WithFolders: options.WithFolders, + WithContents: options.WithContents, } } -func (b wrapper) ListFiles(ctx context.Context, path string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) { - if err := b.validatePath(path); err != nil { - return nil, err - } - - pathWithRoot := b.addRoot(path) - resp, err := b.wrapped.ListFiles(ctx, pathWithRoot, b.pagingOptionsWithDefaults(paging), b.listOptionsWithDefaults(options, false)) - - if resp != nil && resp.Files != nil { - if resp.LastPath != "" { - resp.LastPath = b.removeRoot(resp.LastPath) - } - - for i := 0; i < len(resp.Files); i++ { - resp.Files[i].FullPath = b.removeRoot(resp.Files[i].FullPath) - } - } - - if err != nil { - return resp, err - } - - if len(resp.Files) != 0 { - return resp, err - } - - // TODO: optimize, don't fetch the contents in this case - file, err := b.Get(ctx, path) - if err != nil { - return resp, err - } - - if file != nil { - file.FileMetadata.FullPath = b.removeRoot(file.FileMetadata.FullPath) - return &ListFilesResponse{ - Files: []FileMetadata{file.FileMetadata}, - HasMore: false, - LastPath: file.FileMetadata.FullPath, - }, nil - } - - return resp, err -} - -func (b wrapper) ListFolders(ctx context.Context, path string, options *ListOptions) ([]FileMetadata, error) { - if err := b.validatePath(path); err != nil { - return nil, err - } - - folders, err := b.wrapped.ListFolders(ctx, b.addRoot(path), b.listOptionsWithDefaults(options, true)) - if folders != nil { - for i := 0; i < len(folders); i++ { - folders[i].FullPath = b.removeRoot(folders[i].FullPath) - } - } - return folders, err -} - func (b wrapper) CreateFolder(ctx context.Context, path string) error { if err := b.validatePath(path); err != nil { return err @@ -395,24 +350,77 @@ func (b wrapper) DeleteFolder(ctx context.Context, path string) error { return b.wrapped.DeleteFolder(ctx, rootedPath) } +func (b wrapper) List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) { + if err := b.validatePath(folderPath); err != nil { + return nil, err + } + + options = b.listOptionsWithDefaults(options) + if (!options.WithFiles && !options.WithFolders) || options.isDenyAll() { + return &ListResponse{ + Files: []*File{}, + HasMore: false, + LastPath: "", + }, nil + } + + var fileChan = make(chan *File) + fileRetrievalCtx, cancelFileGet := context.WithCancel(ctx) + defer cancelFileGet() + + go func() { + if options.WithFiles { + if f, err := b.Get(fileRetrievalCtx, folderPath); err == nil { + fileChan <- f + return + } + } + fileChan <- nil + }() + + pathWithRoot := b.addRoot(folderPath) + resp, err := b.wrapped.List(ctx, pathWithRoot, b.pagingOptionsWithDefaults(paging), options) + if err != nil { + return nil, err + } + + if resp != nil && resp.Files != nil && len(resp.Files) > 0 { + if resp.LastPath != "" { + resp.LastPath = b.removeRoot(resp.LastPath) + } + + for i := 0; i < len(resp.Files); i++ { + resp.Files[i].FullPath = b.removeRoot(resp.Files[i].FullPath) + } + return resp, err + } + + file := <-fileChan + if file != nil { + file.FileMetadata.FullPath = b.removeRoot(file.FileMetadata.FullPath) + var contents []byte + if options.WithContents { + contents = file.Contents + } else { + contents = []byte{} + } + return &ListResponse{ + Files: []*File{{Contents: contents, FileMetadata: file.FileMetadata}}, + HasMore: false, + LastPath: file.FileMetadata.FullPath, + }, nil + } + + return resp, err +} + func (b wrapper) isFolderEmpty(ctx context.Context, path string) (bool, error) { - filesInFolder, err := b.ListFiles(ctx, path, &Paging{First: 1}, &ListOptions{Recursive: true}) + resp, err := b.List(ctx, path, &Paging{First: 1}, &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}) if err != nil { return false, err } - if len(filesInFolder.Files) > 0 { - return false, nil - } - - folders, err := b.ListFolders(ctx, path, &ListOptions{ - Recursive: true, - }) - if err != nil { - return false, err - } - - if len(folders) > 0 { + if len(resp.Files) > 0 { return false, nil }