From 6970c9ebfd140c0b96c4ffa02788de34773eaa1a Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Mon, 24 May 2021 15:17:40 +0300 Subject: [PATCH] Use `laher/mergefs` - remove custom code (#34571) --- go.mod | 1 + go.sum | 4 + pkg/cmd/grafana-cli/commands/mergefs.go | 92 ------------------- pkg/cmd/grafana-cli/commands/mergefs_test.go | 68 -------------- .../scuemata_validation_command_test.go | 7 +- 5 files changed, 9 insertions(+), 163 deletions(-) delete mode 100644 pkg/cmd/grafana-cli/commands/mergefs.go delete mode 100644 pkg/cmd/grafana-cli/commands/mergefs_test.go diff --git a/go.mod b/go.mod index 7761246b9ca..4f382baddf2 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.11 github.com/jung-kurt/gofpdf v1.16.2 + github.com/laher/mergefs v0.1.1 github.com/lib/pq v1.10.0 github.com/linkedin/goavro/v2 v2.10.0 github.com/magefile/mage v1.11.0 diff --git a/go.sum b/go.sum index b200f536f05..7c57193f70b 100644 --- a/go.sum +++ b/go.sum @@ -1203,6 +1203,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/laher/mergefs v0.1.1 h1:nV2bTS57vrmbMxeR6uvJpI8LyGl3QHj4bLBZO3aUV58= +github.com/laher/mergefs v0.1.1/go.mod h1:FSY1hYy94on4Tz60waRMGdO1awwS23BacqJlqf9lJ9Q= github.com/lann/builder v0.0.0-20150808151131-f22ce00fd939/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= @@ -1243,6 +1245,8 @@ github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattermost/xml-roundtrip-validator v0.0.0-20201213122252-bcd7e1b9601e h1:qqXczln0qwkVGcpQ+sQuPOVntt2FytYarXXxYSNJkgw= github.com/mattermost/xml-roundtrip-validator v0.0.0-20201213122252-bcd7e1b9601e/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= diff --git a/pkg/cmd/grafana-cli/commands/mergefs.go b/pkg/cmd/grafana-cli/commands/mergefs.go deleted file mode 100644 index c31449f5e94..00000000000 --- a/pkg/cmd/grafana-cli/commands/mergefs.go +++ /dev/null @@ -1,92 +0,0 @@ -package commands - -import ( - "errors" - "io/fs" - "os" - "sort" - - "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" -) - -// MergeFS contains a slice of different filesystems that can be merged together -type MergeFS struct { - filesystems []fs.FS -} - -// Merge filesystems -func Merge(filesystems ...fs.FS) fs.FS { - return MergeFS{filesystems: filesystems} -} - -// Open opens the named file. -func (mfs MergeFS) Open(name string) (fs.File, error) { - for _, filesystem := range mfs.filesystems { - file, err := filesystem.Open(name) - if err == nil { - return file, nil - } - } - return nil, os.ErrNotExist -} - -// ReadDir reads from the directory, and produces a DirEntry array of different -// directories. -// -// It iterates through all different filesystems that exist in the mfs MergeFS -// filesystem slice and it identifies overlapping directories that exist in different -// filesystems -func (mfs MergeFS) ReadDir(name string) ([]fs.DirEntry, error) { - dirsMap := make(map[string]fs.DirEntry) - for _, filesystem := range mfs.filesystems { - if fsys, ok := filesystem.(fs.ReadDirFS); ok { - dir, err := fsys.ReadDir(name) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - logger.Debugf("directory in filepath %s was not found in filesystem", name) - continue - } - return nil, err - } - for _, v := range dir { - if _, ok := dirsMap[v.Name()]; !ok { - dirsMap[v.Name()] = v - } - } - continue - } - - file, err := filesystem.Open(name) - if err != nil { - logger.Debugf("filepath %s was not found in filesystem", name) - continue - } - - dir, ok := file.(fs.ReadDirFile) - if !ok { - return nil, &fs.PathError{Op: "readdir", Path: name, Err: errors.New("not implemented")} - } - - fsDirs, err := dir.ReadDir(-1) - if err != nil { - return nil, err - } - sort.Slice(fsDirs, func(i, j int) bool { return fsDirs[i].Name() < fsDirs[j].Name() }) - for _, v := range fsDirs { - if _, ok := dirsMap[v.Name()]; !ok { - dirsMap[v.Name()] = v - } - } - if err := file.Close(); err != nil { - logger.Error("failed to close file", "err", err) - } - } - dirs := make([]fs.DirEntry, 0, len(dirsMap)) - - for _, value := range dirsMap { - dirs = append(dirs, value) - } - - sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() }) - return dirs, nil -} diff --git a/pkg/cmd/grafana-cli/commands/mergefs_test.go b/pkg/cmd/grafana-cli/commands/mergefs_test.go deleted file mode 100644 index e79503db4be..00000000000 --- a/pkg/cmd/grafana-cli/commands/mergefs_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package commands - -import ( - "io/fs" - "os" - "path/filepath" - "testing" - "testing/fstest" - - "github.com/stretchr/testify/require" -) - -func TestMergeFS(t *testing.T) { - var filePaths = []struct { - path string - dirArrayLength int - child string - }{ - // MapFS takes in account the current directory in addition to all included directories and produces a "" dir - {"a", 1, "z"}, - {"a/z", 1, "bar.cue"}, - {"b", 1, "z"}, - {"b/z", 1, "foo.cue"}, - } - - tempDir := os.DirFS(filepath.Join("testdata", "mergefs")) - a := fstest.MapFS{ - "a": &fstest.MapFile{Mode: fs.ModeDir}, - "a/z": &fstest.MapFile{Mode: fs.ModeDir}, - "a/z/bar.cue": &fstest.MapFile{Data: []byte("bar")}, - } - - filesystem := Merge(tempDir, a) - - t.Run("testing mergefs.ReadDir", func(t *testing.T) { - for _, fp := range filePaths { - t.Run("testing path: "+fp.path, func(t *testing.T) { - dirs, err := fs.ReadDir(filesystem, fp.path) - require.NoError(t, err) - require.Len(t, dirs, fp.dirArrayLength) - - for i := 0; i < len(dirs); i++ { - require.Equal(t, dirs[i].Name(), fp.child) - } - }) - } - }) - - t.Run("testing mergefs.Open", func(t *testing.T) { - data := make([]byte, 3) - file, err := filesystem.Open("a/z/bar.cue") - require.NoError(t, err) - - _, err = file.Read(data) - require.NoError(t, err) - require.Equal(t, "bar", string(data)) - - file, err = filesystem.Open("b/z/foo.cue") - require.NoError(t, err) - - _, err = file.Read(data) - require.NoError(t, err) - require.Equal(t, "foo", string(data)) - - err = file.Close() - require.NoError(t, err) - }) -} diff --git a/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go b/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go index 2f9294daf99..cca7ed473d7 100644 --- a/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go +++ b/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go @@ -8,6 +8,7 @@ import ( "testing/fstest" "github.com/grafana/grafana/pkg/schema/load" + "github.com/laher/mergefs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -35,7 +36,7 @@ func TestValidateScuemataBasics(t *testing.T) { filesystem := fstest.MapFS{ "cue/data/gen.cue": &fstest.MapFile{Data: genCue}, } - mergedFS := Merge(filesystem, defaultBaseLoadPaths.BaseCueFS) + mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS) var baseLoadPaths = load.BaseLoadPaths{ BaseCueFS: mergedFS, @@ -53,7 +54,7 @@ func TestValidateScuemataBasics(t *testing.T) { filesystem := fstest.MapFS{ "cue/data/gen.cue": &fstest.MapFile{Data: genCue}, } - mergedFS := Merge(filesystem, defaultBaseLoadPaths.BaseCueFS) + mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS) var baseLoadPaths = load.BaseLoadPaths{ BaseCueFS: mergedFS, @@ -78,7 +79,7 @@ func TestValidateScuemataBasics(t *testing.T) { "valid.json": &fstest.MapFile{Data: validPanel}, "invalid.json": &fstest.MapFile{Data: invalidPanel}, } - mergedFS := Merge(filesystem, defaultBaseLoadPaths.BaseCueFS) + mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS) var baseLoadPaths = load.BaseLoadPaths{ BaseCueFS: mergedFS,