From 1d1d988aa7caf60d6769bb474d4ec2f872eaaad4 Mon Sep 17 00:00:00 2001 From: Zahoor Mohamed Date: Wed, 12 Apr 2017 05:36:02 +0530 Subject: swarm/api: FUSE read-write support (#13872) - Moved fuse related code in a new package, swarm/fuse - Added write support - Create new files - Delete existing files - Append to files (with limitations) - More test coverage --- swarm/fuse/fuse_dir.go | 155 +++++++ swarm/fuse/fuse_file.go | 144 +++++++ swarm/fuse/fuse_root.go | 35 ++ swarm/fuse/swarmfs.go | 64 +++ swarm/fuse/swarmfs_fallback.go | 51 +++ swarm/fuse/swarmfs_test.go | 897 +++++++++++++++++++++++++++++++++++++++++ swarm/fuse/swarmfs_unix.go | 240 +++++++++++ swarm/fuse/swarmfs_util.go | 144 +++++++ 8 files changed, 1730 insertions(+) create mode 100644 swarm/fuse/fuse_dir.go create mode 100644 swarm/fuse/fuse_file.go create mode 100644 swarm/fuse/fuse_root.go create mode 100644 swarm/fuse/swarmfs.go create mode 100644 swarm/fuse/swarmfs_fallback.go create mode 100644 swarm/fuse/swarmfs_test.go create mode 100644 swarm/fuse/swarmfs_unix.go create mode 100644 swarm/fuse/swarmfs_util.go (limited to 'swarm/fuse') diff --git a/swarm/fuse/fuse_dir.go b/swarm/fuse/fuse_dir.go new file mode 100644 index 000000000..91b236ae8 --- /dev/null +++ b/swarm/fuse/fuse_dir.go @@ -0,0 +1,155 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse" + "bazil.org/fuse/fs" + "golang.org/x/net/context" + "os" + "path/filepath" + "sync" +) + +var ( + _ fs.Node = (*SwarmDir)(nil) + _ fs.NodeRequestLookuper = (*SwarmDir)(nil) + _ fs.HandleReadDirAller = (*SwarmDir)(nil) + _ fs.NodeCreater = (*SwarmDir)(nil) + _ fs.NodeRemover = (*SwarmDir)(nil) + _ fs.NodeMkdirer = (*SwarmDir)(nil) +) + +type SwarmDir struct { + inode uint64 + name string + path string + directories []*SwarmDir + files []*SwarmFile + + mountInfo *MountInfo + lock *sync.RWMutex +} + +func NewSwarmDir(fullpath string, minfo *MountInfo) *SwarmDir { + newdir := &SwarmDir{ + inode: NewInode(), + name: filepath.Base(fullpath), + path: fullpath, + directories: []*SwarmDir{}, + files: []*SwarmFile{}, + mountInfo: minfo, + lock: &sync.RWMutex{}, + } + return newdir +} + +func (sd *SwarmDir) Attr(ctx context.Context, a *fuse.Attr) error { + a.Inode = sd.inode + a.Mode = os.ModeDir | 0700 + a.Uid = uint32(os.Getuid()) + a.Gid = uint32(os.Getegid()) + return nil +} + +func (sd *SwarmDir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { + + for _, n := range sd.files { + if n.name == req.Name { + return n, nil + } + } + for _, n := range sd.directories { + if n.name == req.Name { + return n, nil + } + } + return nil, fuse.ENOENT +} + +func (sd *SwarmDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + var children []fuse.Dirent + for _, file := range sd.files { + children = append(children, fuse.Dirent{Inode: file.inode, Type: fuse.DT_File, Name: file.name}) + } + for _, dir := range sd.directories { + children = append(children, fuse.Dirent{Inode: dir.inode, Type: fuse.DT_Dir, Name: dir.name}) + } + return children, nil +} + +func (sd *SwarmDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + + newFile := NewSwarmFile(sd.path, req.Name, sd.mountInfo) + newFile.fileSize = 0 // 0 means, file is not in swarm yet and it is just created + + sd.lock.Lock() + defer sd.lock.Unlock() + sd.files = append(sd.files, newFile) + + return newFile, newFile, nil +} + +func (sd *SwarmDir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + + if req.Dir && sd.directories != nil { + newDirs := []*SwarmDir{} + for _, dir := range sd.directories { + if dir.name == req.Name { + removeDirectoryFromSwarm(dir) + } else { + newDirs = append(newDirs, dir) + } + } + if len(sd.directories) > len(newDirs) { + sd.lock.Lock() + defer sd.lock.Unlock() + sd.directories = newDirs + } + return nil + } else if !req.Dir && sd.files != nil { + newFiles := []*SwarmFile{} + for _, f := range sd.files { + if f.name == req.Name { + removeFileFromSwarm(f) + } else { + newFiles = append(newFiles, f) + } + } + if len(sd.files) > len(newFiles) { + sd.lock.Lock() + defer sd.lock.Unlock() + sd.files = newFiles + } + return nil + } + return fuse.ENOENT +} + +func (sd *SwarmDir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { + + newDir := NewSwarmDir(req.Name, sd.mountInfo) + + sd.lock.Lock() + defer sd.lock.Unlock() + sd.directories = append(sd.directories, newDir) + + return newDir, nil + +} diff --git a/swarm/fuse/fuse_file.go b/swarm/fuse/fuse_file.go new file mode 100644 index 000000000..0cb59dfb3 --- /dev/null +++ b/swarm/fuse/fuse_file.go @@ -0,0 +1,144 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse" + "bazil.org/fuse/fs" + "errors" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage" + "golang.org/x/net/context" + "io" + "os" + "sync" +) + +const ( + MaxAppendFileSize = 10485760 // 10Mb +) + +var ( + errInvalidOffset = errors.New("Invalid offset during write") + errFileSizeMaxLimixReached = errors.New("File size exceeded max limit") +) + +var ( + _ fs.Node = (*SwarmFile)(nil) + _ fs.HandleReader = (*SwarmFile)(nil) + _ fs.HandleWriter = (*SwarmFile)(nil) +) + +type SwarmFile struct { + inode uint64 + name string + path string + key storage.Key + fileSize int64 + reader storage.LazySectionReader + + mountInfo *MountInfo + lock *sync.RWMutex +} + +func NewSwarmFile(path, fname string, minfo *MountInfo) *SwarmFile { + newFile := &SwarmFile{ + inode: NewInode(), + name: fname, + path: path, + key: nil, + fileSize: -1, // -1 means , file already exists in swarm and you need to just get the size from swarm + reader: nil, + + mountInfo: minfo, + lock: &sync.RWMutex{}, + } + return newFile +} + +func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { + + a.Inode = file.inode + //TODO: need to get permission as argument + a.Mode = 0700 + a.Uid = uint32(os.Getuid()) + a.Gid = uint32(os.Getegid()) + + if file.fileSize == -1 { + reader := file.mountInfo.swarmApi.Retrieve(file.key) + quitC := make(chan bool) + size, err := reader.Size(quitC) + if err != nil { + log.Warn("Couldnt get size of file %s : %v", file.path, err) + } + file.fileSize = int64(size) + } + a.Size = uint64(file.fileSize) + return nil +} + +func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { + + sf.lock.RLock() + defer sf.lock.RUnlock() + if sf.reader == nil { + sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key) + } + buf := make([]byte, req.Size) + n, err := sf.reader.ReadAt(buf, req.Offset) + if err == io.ErrUnexpectedEOF || err == io.EOF { + err = nil + } + resp.Data = buf[:n] + sf.reader = nil + return err + +} + +func (sf *SwarmFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + + if sf.fileSize == 0 && req.Offset == 0 { + + // A new file is created + err := addFileToSwarm(sf, req.Data, len(req.Data)) + if err != nil { + return err + } + resp.Size = len(req.Data) + + } else if req.Offset <= sf.fileSize { + + totalSize := sf.fileSize + int64(len(req.Data)) + if totalSize > MaxAppendFileSize { + log.Warn("Append file size reached (%v) : (%v)", sf.fileSize, len(req.Data)) + return errFileSizeMaxLimixReached + } + + err := appendToExistingFileInSwarm(sf, req.Data, req.Offset, int64(len(req.Data))) + if err != nil { + return err + } + resp.Size = int(sf.fileSize) + } else { + log.Warn("Invalid write request size(%v) : off(%v)", sf.fileSize, req.Offset) + return errInvalidOffset + } + + return nil +} diff --git a/swarm/fuse/fuse_root.go b/swarm/fuse/fuse_root.go new file mode 100644 index 000000000..b2262d1c5 --- /dev/null +++ b/swarm/fuse/fuse_root.go @@ -0,0 +1,35 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse/fs" +) + +var ( + _ fs.Node = (*SwarmDir)(nil) +) + +type SwarmRoot struct { + root *SwarmDir +} + +func (filesystem *SwarmRoot) Root() (fs.Node, error) { + return filesystem.root, nil +} diff --git a/swarm/fuse/swarmfs.go b/swarm/fuse/swarmfs.go new file mode 100644 index 000000000..2493bdab1 --- /dev/null +++ b/swarm/fuse/swarmfs.go @@ -0,0 +1,64 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fuse + +import ( + "github.com/ethereum/go-ethereum/swarm/api" + "sync" + "time" +) + +const ( + Swarmfs_Version = "0.1" + mountTimeout = time.Second * 5 + unmountTimeout = time.Second * 10 + maxFuseMounts = 5 +) + +var ( + swarmfs *SwarmFS // Swarm file system singleton + swarmfsLock sync.Once + + inode uint64 = 1 // global inode + inodeLock sync.RWMutex +) + +type SwarmFS struct { + swarmApi *api.Api + activeMounts map[string]*MountInfo + swarmFsLock *sync.RWMutex +} + +func NewSwarmFS(api *api.Api) *SwarmFS { + swarmfsLock.Do(func() { + swarmfs = &SwarmFS{ + swarmApi: api, + swarmFsLock: &sync.RWMutex{}, + activeMounts: map[string]*MountInfo{}, + } + }) + return swarmfs + +} + +// Inode numbers need to be unique, they are used for caching inside fuse +func NewInode() uint64 { + inodeLock.Lock() + defer inodeLock.Unlock() + inode += 1 + return inode +} diff --git a/swarm/fuse/swarmfs_fallback.go b/swarm/fuse/swarmfs_fallback.go new file mode 100644 index 000000000..4864c8689 --- /dev/null +++ b/swarm/fuse/swarmfs_fallback.go @@ -0,0 +1,51 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !linux,!darwin,!freebsd + +package fuse + +import ( + "errors" +) + +var errNoFUSE = errors.New("FUSE is not supported on this platform") + +func isFUSEUnsupportedError(err error) bool { + return err == errNoFUSE +} + +type MountInfo struct { + MountPoint string + StartManifest string + LatestManifest string +} + +func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { + return nil, errNoFUSE +} + +func (self *SwarmFS) Unmount(mountpoint string) (bool, error) { + return false, errNoFUSE +} + +func (self *SwarmFS) Listmounts() ([]*MountInfo, error) { + return nil, errNoFUSE +} + +func (self *SwarmFS) Stop() error { + return nil +} diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go new file mode 100644 index 000000000..f307b38ea --- /dev/null +++ b/swarm/fuse/swarmfs_test.go @@ -0,0 +1,897 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build linux darwin freebsd + +package fuse + +import ( + "bytes" + "crypto/rand" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/storage" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +type fileInfo struct { + perm uint64 + uid int + gid int + contents []byte +} + +func testFuseFileSystem(t *testing.T, f func(*api.Api)) { + + datadir, err := ioutil.TempDir("", "fuse") + if err != nil { + t.Fatalf("unable to create temp dir: %v", err) + } + os.RemoveAll(datadir) + + dpa, err := storage.NewLocalDPA(datadir) + if err != nil { + return + } + api := api.NewApi(dpa, nil) + dpa.Start() + f(api) + dpa.Stop() +} + +func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string) string { + + os.RemoveAll(uploadDir) + + for fname, finfo := range files { + actualPath := filepath.Join(uploadDir, fname) + filePath := filepath.Dir(actualPath) + + err := os.MkdirAll(filePath, 0777) + if err != nil { + t.Fatalf("Error creating directory '%v' : %v", filePath, err) + } + + fd, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(finfo.perm)) + if err1 != nil { + t.Fatalf("Error creating file %v: %v", actualPath, err1) + } + + fd.Write(finfo.contents) + fd.Chown(finfo.uid, finfo.gid) + fd.Chmod(os.FileMode(finfo.perm)) + fd.Sync() + fd.Close() + } + + bzzhash, err := api.Upload(uploadDir, "") + if err != nil { + t.Fatalf("Error uploading directory %v: %v", uploadDir, err) + } + + return bzzhash +} + +func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash string, mountDir string) *SwarmFS { + + // Test Mount + os.RemoveAll(mountDir) + os.MkdirAll(mountDir, 0777) + swarmfs := NewSwarmFS(api) + _, err := swarmfs.Mount(bzzHash, mountDir) + if isFUSEUnsupportedError(err) { + t.Skip("FUSE not supported:", err) + } else if err != nil { + t.Fatalf("Error mounting hash %v: %v", bzzHash, err) + } + + found := false + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == mountDir { + if minfo.StartManifest != bzzHash || + minfo.LatestManifest != bzzHash || + minfo.fuseConnection == nil { + t.Fatalf("Error mounting: exp(%s): act(%s)", bzzHash, minfo.StartManifest) + } + found = true + } + } + + // Test listMounts + if found == false { + t.Fatalf("Error getting mounts information for %v: %v", mountDir, err) + } + + // Check if file and their attributes are as expected + compareGeneratedFileWithFileInMount(t, files, mountDir) + + return swarmfs + +} + +func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo, mountDir string) { + + err := filepath.Walk(mountDir, func(path string, f os.FileInfo, err error) error { + if f.IsDir() { + return nil + } + fname := path[len(mountDir)+1:] + if _, ok := files[fname]; !ok { + t.Fatalf(" file %v present in mount dir and is not expected", fname) + } + return nil + }) + if err != nil { + t.Fatalf("Error walking dir %v", mountDir) + } + + for fname, finfo := range files { + + destinationFile := filepath.Join(mountDir, fname) + + dfinfo, err := os.Stat(destinationFile) + if err != nil { + t.Fatalf("Destination file %v missing in mount: %v", fname, err) + } + + if int64(len(finfo.contents)) != dfinfo.Size() { + t.Fatalf("file %v Size mismatch source (%v) vs destination(%v)", fname, int64(len(finfo.contents)), dfinfo.Size()) + } + + if dfinfo.Mode().Perm().String() != "-rwx------" { + t.Fatalf("file %v Permission mismatch source (-rwx------) vs destination(%v)", fname, dfinfo.Mode().Perm()) + } + + fileContents, err := ioutil.ReadFile(filepath.Join(mountDir, fname)) + if err != nil { + t.Fatalf("Could not readfile %v : %v", fname, err) + } + + if bytes.Compare(fileContents, finfo.contents) != 0 { + t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents) + + } + + // TODO: check uid and gid + } +} + +func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { + + destinationFile := filepath.Join(testMountDir, fname) + dfinfo, err1 := os.Stat(destinationFile) + if err1 != nil { + t.Fatalf("Could not stat file %v", destinationFile) + } + if dfinfo.Size() != int64(len(contents)) { + t.Fatalf("Mismatch in size actual(%v) vs expected(%v)", dfinfo.Size(), int64(len(contents))) + } + + fd, err2 := os.OpenFile(destinationFile, os.O_RDONLY, os.FileMode(0665)) + if err2 != nil { + t.Fatalf("Could not open file %v", destinationFile) + } + newcontent := make([]byte, len(contents)) + fd.Read(newcontent) + fd.Close() + + if !bytes.Equal(contents, newcontent) { + t.Fatalf("File content mismatch expected (%v): received (%v) ", contents, newcontent) + } +} + +func getRandomBtes(size int) []byte { + contents := make([]byte, size) + rand.Read(contents) + return contents + +} + +func IsDirEmpty(name string) bool { + f, err := os.Open(name) + if err != nil { + return false + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true + } + return false +} + +func testMountListAndUnmount(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "fuse-source") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "fuse-dest") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["2.txt"] = fileInfo{0711, 333, 444, getRandomBtes(10)} + files["3.txt"] = fileInfo{0622, 333, 444, getRandomBtes(100)} + files["4.txt"] = fileInfo{0533, 333, 444, getRandomBtes(1024)} + files["5.txt"] = fileInfo{0544, 333, 444, getRandomBtes(10)} + files["6.txt"] = fileInfo{0555, 333, 444, getRandomBtes(10)} + files["7.txt"] = fileInfo{0666, 333, 444, getRandomBtes(10)} + files["8.txt"] = fileInfo{0777, 333, 333, getRandomBtes(10)} + files["11.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["111.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBtes(10)} + files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBtes(200)} + files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10240)} + files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + // Check unmount + _, err := swarmfs.Unmount(testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", bzzHash) + } + if !IsDirEmpty(testMountDir) { + t.Fatalf("unmount didnt work for %v", testMountDir) + } + +} + +func testMaxMounts(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir1, _ := ioutil.TempDir(os.TempDir(), "max-upload1") + bzzHash1 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir1) + mount1, _ := ioutil.TempDir(os.TempDir(), "max-mount1") + swarmfs1 := mountDir(t, api, files, bzzHash1, mount1) + defer swarmfs1.Stop() + + files["2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir2, _ := ioutil.TempDir(os.TempDir(), "max-upload2") + bzzHash2 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir2) + mount2, _ := ioutil.TempDir(os.TempDir(), "max-mount2") + swarmfs2 := mountDir(t, api, files, bzzHash2, mount2) + defer swarmfs2.Stop() + + files["3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir3, _ := ioutil.TempDir(os.TempDir(), "max-upload3") + bzzHash3 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir3) + mount3, _ := ioutil.TempDir(os.TempDir(), "max-mount3") + swarmfs3 := mountDir(t, api, files, bzzHash3, mount3) + defer swarmfs3.Stop() + + files["4.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir4, _ := ioutil.TempDir(os.TempDir(), "max-upload4") + bzzHash4 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir4) + mount4, _ := ioutil.TempDir(os.TempDir(), "max-mount4") + swarmfs4 := mountDir(t, api, files, bzzHash4, mount4) + defer swarmfs4.Stop() + + files["5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir5, _ := ioutil.TempDir(os.TempDir(), "max-upload5") + bzzHash5 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir5) + mount5, _ := ioutil.TempDir(os.TempDir(), "max-mount5") + swarmfs5 := mountDir(t, api, files, bzzHash5, mount5) + defer swarmfs5.Stop() + + files["6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir6, _ := ioutil.TempDir(os.TempDir(), "max-upload6") + bzzHash6 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir6) + mount6, _ := ioutil.TempDir(os.TempDir(), "max-mount6") + + os.RemoveAll(mount6) + os.MkdirAll(mount6, 0777) + _, err := swarmfs.Mount(bzzHash6, mount6) + if err == nil { + t.Fatalf("Error: Going beyond max mounts %v", bzzHash6) + } + +} + +func testReMounts(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir1, _ := ioutil.TempDir(os.TempDir(), "re-upload1") + bzzHash1 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir1) + testMountDir1, _ := ioutil.TempDir(os.TempDir(), "re-mount1") + swarmfs := mountDir(t, api, files, bzzHash1, testMountDir1) + defer swarmfs.Stop() + + uploadDir2, _ := ioutil.TempDir(os.TempDir(), "re-upload2") + bzzHash2 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir2) + testMountDir2, _ := ioutil.TempDir(os.TempDir(), "re-mount2") + + // try mounting the same hash second time + os.RemoveAll(testMountDir2) + os.MkdirAll(testMountDir2, 0777) + _, err := swarmfs.Mount(bzzHash1, testMountDir2) + if err != nil { + t.Fatalf("Error mounting hash %v", bzzHash1) + } + + // mount a different hash in already mounted point + _, err = swarmfs.Mount(bzzHash2, testMountDir1) + if err == nil { + t.Fatalf("Error mounting hash %v", bzzHash2) + } + + // mount nonexistent hash + _, err = swarmfs.Mount("0xfea11223344", testMountDir1) + if err == nil { + t.Fatalf("Error mounting hash %v", bzzHash2) + } + +} + +func testUnmount(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + uploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, uploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + swarmfs.Unmount(testMountDir) + + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + } + } + +} + +func testUnmountWhenResourceBusy(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + actualPath := filepath.Join(testMountDir, "2.txt") + d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700)) + d.Write(getRandomBtes(10)) + + _, err = swarmfs.Unmount(testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", bzzHash) + } + d.Close() + + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + } + } + +} +func testSeekInMultiChunkFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "seek-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "seek-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10240)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + // Create a new file seek the second chunk + actualPath := filepath.Join(testMountDir, "1.txt") + d, _ := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700)) + + d.Seek(5000, 0) + + contents := make([]byte, 1024) + d.Read(contents) + finfo := files["1.txt"] + + if bytes.Compare(finfo.contents[:6024][5000:], contents) != 0 { + t.Fatalf("File seek contents mismatch") + } + d.Close() + +} + +func testCreateNewFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "create-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "create-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file in the root dir and check + actualPath := filepath.Join(testMountDir, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "2.txt", contents) + +} + +func testCreateNewFileInsideDirectory(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-mount") + + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file inside a existing dir and check + dirToCreate := filepath.Join(testMountDir, "one") + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "one/2.txt", contents) + +} + +func testCreateNewFileInsideNewDirectory(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file inside a existing dir and check + dirToCreate := filepath.Join(testMountDir, "one") + os.MkdirAll(dirToCreate, 0777) + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "one/2.txt", contents) + +} + +func testRemoveExistingFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Remove a file in the root dir and check + actualPath := filepath.Join(testMountDir, "five.txt") + os.Remove(actualPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + delete(files, "five.txt") + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testRemoveExistingFileInsideADir(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Remove a file in the root dir and check + actualPath := filepath.Join(testMountDir, "one/five.txt") + os.Remove(actualPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + delete(files, "one/five.txt") + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testRemoveNewlyAddedFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "removenew-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "removenew-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Adda a new file and remove it + dirToCreate := filepath.Join(testMountDir, "one") + os.MkdirAll(dirToCreate, os.FileMode(0665)) + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + checkFile(t, testMountDir, "one/2.txt", contents) + + os.Remove(actualPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + if bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + } + +} + +func testAddNewFileAndModifyContents(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file in the root dir and check + actualPath := filepath.Join(testMountDir, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + line1 := []byte("Line 1") + rand.Read(line1) + d.Write(line1) + d.Close() + + mi1, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["2.txt"] = fileInfo{0700, 333, 444, line1} + swarmfs2 := mountDir(t, api, files, mi1.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "2.txt", line1) + + mi2, err3 := swarmfs2.Unmount(testMountDir) + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + + // mount again and modify + swarmfs3 := mountDir(t, api, files, mi2.LatestManifest, testMountDir) + defer swarmfs3.Stop() + + fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + if err4 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err4) + } + line2 := []byte("Line 2") + rand.Read(line2) + fd.Seek(int64(len(line1)), 0) + fd.Write(line2) + fd.Close() + + mi3, err5 := swarmfs3.Unmount(testMountDir) + if err5 != nil { + t.Fatalf("Could not unmount %v", err5) + } + + // mount again and see if things are okay + b := [][]byte{line1, line2} + line1and2 := bytes.Join(b, []byte("")) + files["2.txt"] = fileInfo{0700, 333, 444, line1and2} + swarmfs4 := mountDir(t, api, files, mi3.LatestManifest, testMountDir) + defer swarmfs4.Stop() + + checkFile(t, testMountDir, "2.txt", line1and2) + +} + +func testRemoveEmptyDir(api *api.Api, t *testing.T) { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + os.MkdirAll(filepath.Join(testMountDir, "newdir"), 0777) + + mi, err3 := swarmfs1.Unmount(testMountDir) + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + + if bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + } + +} + +func testRemoveDirWhichHasFiles(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") + + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + dirPath := filepath.Join(testMountDir, "two") + os.RemoveAll(dirPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v ", err2) + } + + // mount again and see if things are okay + delete(files, "two/five.txt") + delete(files, "two/six.txt") + + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testRemoveDirWhichHasSubDirs(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-mount") + + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + dirPath := filepath.Join(testMountDir, "two") + os.RemoveAll(dirPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v ", err2) + } + + // mount again and see if things are okay + delete(files, "two/three/2.txt") + delete(files, "two/three/3.txt") + delete(files, "two/four/5.txt") + delete(files, "two/four/6.txt") + delete(files, "two/four/six/7.txt") + + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testAppendFileContentsToEnd(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-mount") + + line1 := make([]byte, 10) + rand.Read(line1) + files["1.txt"] = fileInfo{0700, 333, 444, line1} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + actualPath := filepath.Join(testMountDir, "1.txt") + fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + if err4 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err4) + } + line2 := make([]byte, 5) + rand.Read(line2) + fd.Seek(int64(len(line1)), 0) + fd.Write(line2) + fd.Close() + + mi1, err5 := swarmfs1.Unmount(testMountDir) + if err5 != nil { + t.Fatalf("Could not unmount %v ", err5) + } + + // mount again and see if things are okay + b := [][]byte{line1, line2} + line1and2 := bytes.Join(b, []byte("")) + files["1.txt"] = fileInfo{0700, 333, 444, line1and2} + swarmfs2 := mountDir(t, api, files, mi1.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "1.txt", line1and2) + +} + +func TestSwarmFileSystem(t *testing.T) { + testFuseFileSystem(t, func(api *api.Api) { + + testMountListAndUnmount(api, t) + + testMaxMounts(api, t) + + testReMounts(api, t) + + testUnmount(api, t) + + testUnmountWhenResourceBusy(api, t) + + testSeekInMultiChunkFile(api, t) + + testCreateNewFile(api, t) + + testCreateNewFileInsideDirectory(api, t) + + testCreateNewFileInsideNewDirectory(api, t) + + testRemoveExistingFile(api, t) + + testRemoveExistingFileInsideADir(api, t) + + testRemoveNewlyAddedFile(api, t) + + testAddNewFileAndModifyContents(api, t) + + testRemoveEmptyDir(api, t) + + testRemoveDirWhichHasFiles(api, t) + + testRemoveDirWhichHasSubDirs(api, t) + + testAppendFileContentsToEnd(api, t) + + }) +} diff --git a/swarm/fuse/swarmfs_unix.go b/swarm/fuse/swarmfs_unix.go new file mode 100644 index 000000000..f4eecef24 --- /dev/null +++ b/swarm/fuse/swarmfs_unix.go @@ -0,0 +1,240 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse" + "bazil.org/fuse/fs" + "errors" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + errEmptyMountPoint = errors.New("need non-empty mount point") + errMaxMountCount = errors.New("max FUSE mount count reached") + errMountTimeout = errors.New("mount timeout") + errAlreadyMounted = errors.New("mount point is already serving") +) + +func isFUSEUnsupportedError(err error) bool { + if perr, ok := err.(*os.PathError); ok { + return perr.Op == "open" && perr.Path == "/dev/fuse" + } + return err == fuse.ErrOSXFUSENotFound +} + +// information about every active mount +type MountInfo struct { + MountPoint string + StartManifest string + LatestManifest string + rootDir *SwarmDir + fuseConnection *fuse.Conn + swarmApi *api.Api + lock *sync.RWMutex +} + +// Inode numbers need to be unique, they are used for caching inside fuse +func newInode() uint64 { + inodeLock.Lock() + defer inodeLock.Unlock() + inode += 1 + return inode +} + +func NewMountInfo(mhash, mpoint string, sapi *api.Api) *MountInfo { + newMountInfo := &MountInfo{ + MountPoint: mpoint, + StartManifest: mhash, + LatestManifest: mhash, + rootDir: nil, + fuseConnection: nil, + swarmApi: sapi, + lock: &sync.RWMutex{}, + } + return newMountInfo +} + +func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { + + if mountpoint == "" { + return nil, errEmptyMountPoint + } + cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint)) + if err != nil { + return nil, err + } + + self.swarmFsLock.Lock() + defer self.swarmFsLock.Unlock() + + noOfActiveMounts := len(self.activeMounts) + if noOfActiveMounts >= maxFuseMounts { + return nil, errMaxMountCount + } + + if _, ok := self.activeMounts[cleanedMountPoint]; ok { + return nil, errAlreadyMounted + } + + log.Info(fmt.Sprintf("Attempting to mount %s ", cleanedMountPoint)) + key, manifestEntryMap, err := self.swarmApi.BuildDirectoryTree(mhash, true) + if err != nil { + return nil, err + } + + mi := NewMountInfo(mhash, cleanedMountPoint, self.swarmApi) + + dirTree := map[string]*SwarmDir{} + rootDir := NewSwarmDir("/", mi) + dirTree["/"] = rootDir + mi.rootDir = rootDir + + for suffix, entry := range manifestEntryMap { + + key = common.Hex2Bytes(entry.Hash) + fullpath := "/" + suffix + basepath := filepath.Dir(fullpath) + + parentDir := rootDir + dirUntilNow := "" + paths := strings.Split(basepath, "/") + for i := range paths { + if paths[i] != "" { + thisDir := paths[i] + dirUntilNow = dirUntilNow + "/" + thisDir + + if _, ok := dirTree[dirUntilNow]; !ok { + dirTree[dirUntilNow] = NewSwarmDir(dirUntilNow, mi) + parentDir.directories = append(parentDir.directories, dirTree[dirUntilNow]) + parentDir = dirTree[dirUntilNow] + + } else { + parentDir = dirTree[dirUntilNow] + } + + } + } + thisFile := NewSwarmFile(basepath, filepath.Base(fullpath), mi) + thisFile.key = key + + parentDir.files = append(parentDir.files, thisFile) + } + + fconn, err := fuse.Mount(cleanedMountPoint, fuse.FSName("swarmfs"), fuse.VolumeName(mhash)) + if isFUSEUnsupportedError(err) { + log.Warn("Fuse not installed", "mountpoint", cleanedMountPoint, "err", err) + return nil, err + } else if err != nil { + fuse.Unmount(cleanedMountPoint) + log.Warn("Error mounting swarm manifest", "mountpoint", cleanedMountPoint, "err", err) + return nil, err + } + mi.fuseConnection = fconn + + serverr := make(chan error, 1) + go func() { + log.Info(fmt.Sprintf("Serving %s at %s", mhash, cleanedMountPoint)) + filesys := &SwarmRoot{root: rootDir} + if err := fs.Serve(fconn, filesys); err != nil { + log.Warn(fmt.Sprintf("Could not Serve SwarmFileSystem error: %v", err)) + serverr <- err + } + + }() + + // Check if the mount process has an error to report. + select { + case <-time.After(mountTimeout): + fuse.Unmount(cleanedMountPoint) + return nil, errMountTimeout + + case err := <-serverr: + fuse.Unmount(cleanedMountPoint) + log.Warn("Error serving swarm FUSE FS", "mountpoint", cleanedMountPoint, "err", err) + return nil, err + + case <-fconn.Ready: + log.Info("Now serving swarm FUSE FS", "manifest", mhash, "mountpoint", cleanedMountPoint) + } + + self.activeMounts[cleanedMountPoint] = mi + return mi, nil +} + +func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) { + + self.swarmFsLock.Lock() + defer self.swarmFsLock.Unlock() + + cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint)) + if err != nil { + return nil, err + } + + mountInfo := self.activeMounts[cleanedMountPoint] + + if mountInfo == nil || mountInfo.MountPoint != cleanedMountPoint { + return nil, fmt.Errorf("%s is not mounted", cleanedMountPoint) + } + err = fuse.Unmount(cleanedMountPoint) + if err != nil { + err1 := externalUnMount(cleanedMountPoint) + if err1 != nil { + errStr := fmt.Sprintf("UnMount error: %v", err) + log.Warn(errStr) + return nil, err1 + } + } + + mountInfo.fuseConnection.Close() + delete(self.activeMounts, cleanedMountPoint) + + succString := fmt.Sprintf("UnMounting %v succeeded", cleanedMountPoint) + log.Info(succString) + + return mountInfo, nil +} + +func (self *SwarmFS) Listmounts() []*MountInfo { + self.swarmFsLock.RLock() + defer self.swarmFsLock.RUnlock() + + rows := make([]*MountInfo, 0, len(self.activeMounts)) + for _, mi := range self.activeMounts { + rows = append(rows, mi) + } + return rows +} + +func (self *SwarmFS) Stop() bool { + for mp := range self.activeMounts { + mountInfo := self.activeMounts[mp] + self.Unmount(mountInfo.MountPoint) + } + return true +} diff --git a/swarm/fuse/swarmfs_util.go b/swarm/fuse/swarmfs_util.go new file mode 100644 index 000000000..d20ab258e --- /dev/null +++ b/swarm/fuse/swarmfs_util.go @@ -0,0 +1,144 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build linux darwin freebsd + +package fuse + +import ( + "fmt" + "github.com/ethereum/go-ethereum/log" + "os/exec" + "runtime" + "time" +) + +func externalUnMount(mountPoint string) error { + + var cmd *exec.Cmd + + switch runtime.GOOS { + + case "darwin": + cmd = exec.Command("/usr/bin/diskutil", "umount", "force", mountPoint) + + case "linux": + cmd = exec.Command("fusermount", "-u", mountPoint) + + default: + return fmt.Errorf("unmount: unimplemented") + } + + errc := make(chan error, 1) + go func() { + defer close(errc) + + if err := exec.Command("umount", mountPoint).Run(); err == nil { + return + } + errc <- cmd.Run() + }() + + select { + + case <-time.After(unmountTimeout): + return fmt.Errorf("umount timeout") + + case err := <-errc: + return err + } +} + +func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { + + fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(sf.mountInfo.LatestManifest, sf.path, sf.name, content, true) + if err != nil { + return err + } + + sf.lock.Lock() + defer sf.lock.Unlock() + sf.key = fkey + sf.fileSize = int64(size) + + sf.mountInfo.lock.Lock() + defer sf.mountInfo.lock.Unlock() + sf.mountInfo.LatestManifest = mhash + + log.Info("Added new file:", "fname", sf.name, "New Manifest hash", mhash) + return nil + +} + +func removeFileFromSwarm(sf *SwarmFile) error { + + mkey, err := sf.mountInfo.swarmApi.RemoveFile(sf.mountInfo.LatestManifest, sf.path, sf.name, true) + if err != nil { + return err + } + + sf.mountInfo.lock.Lock() + defer sf.mountInfo.lock.Unlock() + sf.mountInfo.LatestManifest = mkey + + log.Info("Removed file:", "fname", sf.name, "New Manifest hash", mkey) + return nil +} + +func removeDirectoryFromSwarm(sd *SwarmDir) error { + + if len(sd.directories) == 0 && len(sd.files) == 0 { + return nil + } + + for _, d := range sd.directories { + err := removeDirectoryFromSwarm(d) + if err != nil { + return err + } + } + + for _, f := range sd.files { + err := removeFileFromSwarm(f) + if err != nil { + return err + } + } + + return nil + +} + +func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error { + + fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.key, offset, length, true) + if err != nil { + return err + } + + sf.lock.Lock() + defer sf.lock.Unlock() + sf.key = fkey + sf.fileSize = sf.fileSize + int64(len(content)) + + sf.mountInfo.lock.Lock() + defer sf.mountInfo.lock.Unlock() + sf.mountInfo.LatestManifest = mhash + + log.Info("Appended file:", "fname", sf.name, "New Manifest hash", mhash) + return nil + +} -- cgit v1.2.3