diff options
author | ethersphere <thesw@rm.eth> | 2018-06-20 20:06:27 +0800 |
---|---|---|
committer | ethersphere <thesw@rm.eth> | 2018-06-22 03:10:31 +0800 |
commit | e187711c6545487d4cac3701f0f506bb536234e2 (patch) | |
tree | d2f6150f70b84b36e49a449082aeda267b4b9046 /swarm/fuse | |
parent | 574378edb50c907b532946a1d4654dbd6701b20a (diff) | |
download | go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.tar go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.tar.gz go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.tar.bz2 go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.tar.lz go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.tar.xz go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.tar.zst go-tangerine-e187711c6545487d4cac3701f0f506bb536234e2.zip |
swarm: network rewrite merge
Diffstat (limited to 'swarm/fuse')
-rw-r--r-- | swarm/fuse/fuse_dir.go | 15 | ||||
-rw-r--r-- | swarm/fuse/fuse_file.go | 43 | ||||
-rw-r--r-- | swarm/fuse/swarmfs.go | 4 | ||||
-rw-r--r-- | swarm/fuse/swarmfs_test.go | 1695 | ||||
-rw-r--r-- | swarm/fuse/swarmfs_unix.go | 156 | ||||
-rw-r--r-- | swarm/fuse/swarmfs_util.go | 18 |
6 files changed, 1422 insertions, 509 deletions
diff --git a/swarm/fuse/fuse_dir.go b/swarm/fuse/fuse_dir.go index a7701985e..7f66451f1 100644 --- a/swarm/fuse/fuse_dir.go +++ b/swarm/fuse/fuse_dir.go @@ -25,6 +25,7 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" + "github.com/ethereum/go-ethereum/swarm/log" "golang.org/x/net/context" ) @@ -49,6 +50,7 @@ type SwarmDir struct { } func NewSwarmDir(fullpath string, minfo *MountInfo) *SwarmDir { + log.Debug("swarmfs", "NewSwarmDir", fullpath) newdir := &SwarmDir{ inode: NewInode(), name: filepath.Base(fullpath), @@ -62,6 +64,8 @@ func NewSwarmDir(fullpath string, minfo *MountInfo) *SwarmDir { } func (sd *SwarmDir) Attr(ctx context.Context, a *fuse.Attr) error { + sd.lock.RLock() + defer sd.lock.RUnlock() a.Inode = sd.inode a.Mode = os.ModeDir | 0700 a.Uid = uint32(os.Getuid()) @@ -70,7 +74,7 @@ func (sd *SwarmDir) Attr(ctx context.Context, a *fuse.Attr) error { } func (sd *SwarmDir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { - + log.Debug("swarmfs", "Lookup", req.Name) for _, n := range sd.files { if n.name == req.Name { return n, nil @@ -85,6 +89,7 @@ func (sd *SwarmDir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *f } func (sd *SwarmDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + log.Debug("swarmfs ReadDirAll") var children []fuse.Dirent for _, file := range sd.files { children = append(children, fuse.Dirent{Inode: file.inode, Type: fuse.DT_File, Name: file.name}) @@ -96,6 +101,7 @@ func (sd *SwarmDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { } func (sd *SwarmDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + log.Debug("swarmfs Create", "path", sd.path, "req.Name", req.Name) newFile := NewSwarmFile(sd.path, req.Name, sd.mountInfo) newFile.fileSize = 0 // 0 means, file is not in swarm yet and it is just created @@ -108,6 +114,7 @@ func (sd *SwarmDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *f } func (sd *SwarmDir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + log.Debug("swarmfs Remove", "path", sd.path, "req.Name", req.Name) if req.Dir && sd.directories != nil { newDirs := []*SwarmDir{} @@ -144,13 +151,11 @@ func (sd *SwarmDir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { } func (sd *SwarmDir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { - - newDir := NewSwarmDir(req.Name, sd.mountInfo) - + log.Debug("swarmfs Mkdir", "path", sd.path, "req.Name", req.Name) + newDir := NewSwarmDir(filepath.Join(sd.path, req.Name), sd.mountInfo) sd.lock.Lock() defer sd.lock.Unlock() sd.directories = append(sd.directories, newDir) return newDir, nil - } diff --git a/swarm/fuse/fuse_file.go b/swarm/fuse/fuse_file.go index c94a0773f..80c26fe05 100644 --- a/swarm/fuse/fuse_file.go +++ b/swarm/fuse/fuse_file.go @@ -26,7 +26,7 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" "golang.org/x/net/context" ) @@ -50,7 +50,7 @@ type SwarmFile struct { inode uint64 name string path string - key storage.Key + addr storage.Address fileSize int64 reader storage.LazySectionReader @@ -63,7 +63,7 @@ func NewSwarmFile(path, fname string, minfo *MountInfo) *SwarmFile { inode: NewInode(), name: fname, path: path, - key: nil, + addr: nil, fileSize: -1, // -1 means , file already exists in swarm and you need to just get the size from swarm reader: nil, @@ -73,33 +73,38 @@ func NewSwarmFile(path, fname string, minfo *MountInfo) *SwarmFile { return newFile } -func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { - - a.Inode = file.inode +func (sf *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { + log.Debug("swarmfs Attr", "path", sf.path) + sf.lock.Lock() + defer sf.lock.Unlock() + a.Inode = sf.inode //TODO: need to get permission as argument a.Mode = 0700 a.Uid = uint32(os.Getuid()) a.Gid = uint32(os.Getegid()) - if file.fileSize == -1 { - reader := file.mountInfo.swarmApi.Retrieve(file.key) + if sf.fileSize == -1 { + reader, _ := sf.mountInfo.swarmApi.Retrieve(sf.addr) quitC := make(chan bool) size, err := reader.Size(quitC) if err != nil { - log.Warn("Couldnt get size of file %s : %v", file.path, err) + log.Error("Couldnt get size of file %s : %v", sf.path, err) + return err } - file.fileSize = size + sf.fileSize = size + log.Trace("swarmfs Attr", "size", size) + close(quitC) } - a.Size = uint64(file.fileSize) + a.Size = uint64(sf.fileSize) return nil } func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - + log.Debug("swarmfs Read", "path", sf.path, "req.String", req.String()) sf.lock.RLock() defer sf.lock.RUnlock() if sf.reader == nil { - sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key) + sf.reader, _ = sf.mountInfo.swarmApi.Retrieve(sf.addr) } buf := make([]byte, req.Size) n, err := sf.reader.ReadAt(buf, req.Offset) @@ -108,26 +113,23 @@ func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse } resp.Data = buf[:n] sf.reader = nil - return err + return err } func (sf *SwarmFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { - + log.Debug("swarmfs Write", "path", sf.path, "req.String", req.String()) if sf.fileSize == 0 && req.Offset == 0 { - // A new file is created err := addFileToSwarm(sf, req.Data, len(req.Data)) if err != nil { return err } resp.Size = len(req.Data) - } else if req.Offset <= sf.fileSize { - totalSize := sf.fileSize + int64(len(req.Data)) if totalSize > MaxAppendFileSize { - log.Warn("Append file size reached (%v) : (%v)", sf.fileSize, len(req.Data)) + log.Warn("swarmfs Append file size reached (%v) : (%v)", sf.fileSize, len(req.Data)) return errFileSizeMaxLimixReached } @@ -137,9 +139,8 @@ func (sf *SwarmFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *fu } resp.Size = len(req.Data) } else { - log.Warn("Invalid write request size(%v) : off(%v)", sf.fileSize, req.Offset) + log.Warn("swarmfs Invalid write request size(%v) : off(%v)", sf.fileSize, req.Offset) return errInvalidOffset } - return nil } diff --git a/swarm/fuse/swarmfs.go b/swarm/fuse/swarmfs.go index e56d0ad4e..c7aa983b7 100644 --- a/swarm/fuse/swarmfs.go +++ b/swarm/fuse/swarmfs.go @@ -39,12 +39,12 @@ var ( ) type SwarmFS struct { - swarmApi *api.Api + swarmApi *api.API activeMounts map[string]*MountInfo swarmFsLock *sync.RWMutex } -func NewSwarmFS(api *api.Api) *SwarmFS { +func NewSwarmFS(api *api.API) *SwarmFS { swarmfsLock.Do(func() { swarmfs = &SwarmFS{ swarmApi: api, diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 42af36345..accf9b02b 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -21,6 +21,8 @@ package fuse import ( "bytes" "crypto/rand" + "flag" + "fmt" "io" "io/ioutil" "os" @@ -29,8 +31,24 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" + + "github.com/ethereum/go-ethereum/log" + + colorable "github.com/mattn/go-colorable" +) + +var ( + loglevel = flag.Int("loglevel", 4, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "turn off terminal formatting in logs") + longrunning = flag.Bool("longrunning", false, "do run long-running tests") ) +func init() { + flag.Parse() + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog)))) +} + type fileInfo struct { perm uint64 uid int @@ -38,41 +56,70 @@ type fileInfo struct { contents []byte } -func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string) string { - os.RemoveAll(uploadDir) +//create files from the map of name and content provided and upload them to swarm via api +func createTestFilesAndUploadToSwarm(t *testing.T, api *api.API, files map[string]fileInfo, uploadDir string, toEncrypt bool) string { + //iterate the map for fname, finfo := range files { actualPath := filepath.Join(uploadDir, fname) filePath := filepath.Dir(actualPath) + //create directory err := os.MkdirAll(filePath, 0777) if err != nil { t.Fatalf("Error creating directory '%v' : %v", filePath, err) } + //create file fd, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(finfo.perm)) if err1 != nil { t.Fatalf("Error creating file %v: %v", actualPath, err1) } - fd.Write(finfo.contents) - fd.Chown(finfo.uid, finfo.gid) - fd.Chmod(os.FileMode(finfo.perm)) - fd.Sync() - fd.Close() + //write content to file + _, err = fd.Write(finfo.contents) + if err != nil { + t.Fatalf("Error writing to file '%v' : %v", filePath, err) + } + /* + Note @holisticode: It's not clear why the Chown command was added to the test suite. + Some files are initialized with different permissions in the individual test, + resulting in errors on Chown which were not checked. + After adding the checks tests would fail. + + What's then the reason to have this check in the first place? + Disabling for now + + err = fd.Chown(finfo.uid, finfo.gid) + if err != nil { + t.Fatalf("Error chown file '%v' : %v", filePath, err) + } + */ + err = fd.Chmod(os.FileMode(finfo.perm)) + if err != nil { + t.Fatalf("Error chmod file '%v' : %v", filePath, err) + } + err = fd.Sync() + if err != nil { + t.Fatalf("Error sync file '%v' : %v", filePath, err) + } + err = fd.Close() + if err != nil { + t.Fatalf("Error closing file '%v' : %v", filePath, err) + } } - bzzhash, err := api.Upload(uploadDir, "") + //upload directory to swarm and return hash + bzzhash, err := api.Upload(uploadDir, "", toEncrypt) if err != nil { - t.Fatalf("Error uploading directory %v: %v", uploadDir, err) + t.Fatalf("Error uploading directory %v: %vm encryption: %v", uploadDir, err, toEncrypt) } return bzzhash } -func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash string, mountDir string) *SwarmFS { - os.RemoveAll(mountDir) - os.MkdirAll(mountDir, 0777) +//mount a swarm hash as a directory on files system via FUSE +func mountDir(t *testing.T, api *api.API, files map[string]fileInfo, bzzHash string, mountDir string) *SwarmFS { swarmfs := NewSwarmFS(api) _, err := swarmfs.Mount(bzzHash, mountDir) if isFUSEUnsupportedError(err) { @@ -81,17 +128,21 @@ func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash str t.Fatalf("Error mounting hash %v: %v", bzzHash, err) } + //check directory is mounted found := false mi := swarmfs.Listmounts() for _, minfo := range mi { + minfo.lock.RLock() if minfo.MountPoint == mountDir { if minfo.StartManifest != bzzHash || minfo.LatestManifest != bzzHash || minfo.fuseConnection == nil { + minfo.lock.RUnlock() t.Fatalf("Error mounting: exp(%s): act(%s)", bzzHash, minfo.StartManifest) } found = true } + minfo.lock.RUnlock() } // Test listMounts @@ -105,6 +156,7 @@ func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash str return swarmfs } +// Check if file and their attributes are as expected func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo, mountDir string) { err := filepath.Walk(mountDir, func(path string, f os.FileInfo, err error) error { if f.IsDir() { @@ -142,12 +194,12 @@ func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo } if !bytes.Equal(fileContents, finfo.contents) { t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents) - } // TODO: check uid and gid } } +//check mounted file with provided content func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { destinationFile := filepath.Join(testMountDir, fname) dfinfo, err1 := os.Stat(destinationFile) @@ -163,15 +215,21 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { t.Fatalf("Could not open file %v", destinationFile) } newcontent := make([]byte, len(contents)) - fd.Read(newcontent) - fd.Close() + _, err := fd.Read(newcontent) + if err != nil { + t.Fatalf("Could not read from file %v", err) + } + err = fd.Close() + if err != nil { + t.Fatalf("Could not close file %v", err) + } if !bytes.Equal(contents, newcontent) { t.Fatalf("File content mismatch expected (%v): received (%v) ", contents, newcontent) } } -func getRandomBtes(size int) []byte { +func getRandomBytes(size int) []byte { contents := make([]byte, size) rand.Read(contents) return contents @@ -190,647 +248,1448 @@ func isDirEmpty(name string) bool { } type testAPI struct { - api *api.Api -} - -func (ta *testAPI) mountListAndUnmount(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "fuse-source") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "fuse-dest") - - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["2.txt"] = fileInfo{0711, 333, 444, getRandomBtes(10)} - files["3.txt"] = fileInfo{0622, 333, 444, getRandomBtes(100)} - files["4.txt"] = fileInfo{0533, 333, 444, getRandomBtes(1024)} - files["5.txt"] = fileInfo{0544, 333, 444, getRandomBtes(10)} - files["6.txt"] = fileInfo{0555, 333, 444, getRandomBtes(10)} - files["7.txt"] = fileInfo{0666, 333, 444, getRandomBtes(10)} - files["8.txt"] = fileInfo{0777, 333, 333, getRandomBtes(10)} - files["11.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["111.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBtes(10)} - files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBtes(200)} - files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10240)} - files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) - - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + api *api.API +} + +type testData struct { + testDir string + testUploadDir string + testMountDir string + bzzHash string + files map[string]fileInfo + toEncrypt bool + swarmfs *SwarmFS +} + +//create the root dir of a test +func (ta *testAPI) initSubtest(name string) (*testData, error) { + var err error + d := &testData{} + d.testDir, err = ioutil.TempDir(os.TempDir(), name) + if err != nil { + return nil, fmt.Errorf("Couldn't create test dir: %v", err) + } + return d, nil +} + +//upload data and mount directory +func (ta *testAPI) uploadAndMount(dat *testData, t *testing.T) (*testData, error) { + //create upload dir + err := os.MkdirAll(dat.testUploadDir, 0777) + if err != nil { + return nil, fmt.Errorf("Couldn't create upload dir: %v", err) + } + //create mount dir + err = os.MkdirAll(dat.testMountDir, 0777) + if err != nil { + return nil, fmt.Errorf("Couldn't create mount dir: %v", err) + } + //upload the file + dat.bzzHash = createTestFilesAndUploadToSwarm(t, ta.api, dat.files, dat.testUploadDir, dat.toEncrypt) + log.Debug("Created test files and uploaded to Swarm") + //mount the directory + dat.swarmfs = mountDir(t, ta.api, dat.files, dat.bzzHash, dat.testMountDir) + log.Debug("Mounted swarm fs") + return dat, nil +} + +//add a directory to the test directory tree +func addDir(root string, name string) (string, error) { + d := filepath.Join(root, name) + err := os.MkdirAll(d, 0777) + if err != nil { + return "", fmt.Errorf("Couldn't create dir inside test dir: %v", err) + } + return d, nil +} +func (ta *testAPI) mountListAndUnmountEncrypted(t *testing.T) { + log.Debug("Starting mountListAndUnmountEncrypted test") + ta.mountListAndUnmount(t, true) + log.Debug("Test mountListAndUnmountEncrypted terminated") +} + +func (ta *testAPI) mountListAndUnmountNonEncrypted(t *testing.T) { + log.Debug("Starting mountListAndUnmountNonEncrypted test") + ta.mountListAndUnmount(t, false) + log.Debug("Test mountListAndUnmountNonEncrypted terminated") +} + +//mount a directory unmount and check the directory is empty afterwards +func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("mountListAndUnmount") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) + + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "testUploadDir") + dat.testMountDir = filepath.Join(dat.testDir, "testMountDir") + dat.files = make(map[string]fileInfo) + + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)} + dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)} + dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)} + dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)} + dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)} + dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)} + dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)} + dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)} + dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)} + dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)} + dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() // Check unmount - _, err := swarmfs.Unmount(testMountDir) - if err != nil { - t.Fatalf("could not unmount %v", bzzHash) - } - if !isDirEmpty(testMountDir) { - t.Fatalf("unmount didnt work for %v", testMountDir) - } - -} - -func (ta *testAPI) maxMounts(t *testing.T) { - files := make(map[string]fileInfo) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir1, _ := ioutil.TempDir(os.TempDir(), "max-upload1") - bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1) - mount1, _ := ioutil.TempDir(os.TempDir(), "max-mount1") - swarmfs1 := mountDir(t, ta.api, files, bzzHash1, mount1) - defer swarmfs1.Stop() - - files["2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir2, _ := ioutil.TempDir(os.TempDir(), "max-upload2") - bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2) - mount2, _ := ioutil.TempDir(os.TempDir(), "max-mount2") - swarmfs2 := mountDir(t, ta.api, files, bzzHash2, mount2) - defer swarmfs2.Stop() - - files["3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir3, _ := ioutil.TempDir(os.TempDir(), "max-upload3") - bzzHash3 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir3) - mount3, _ := ioutil.TempDir(os.TempDir(), "max-mount3") - swarmfs3 := mountDir(t, ta.api, files, bzzHash3, mount3) - defer swarmfs3.Stop() - - files["4.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir4, _ := ioutil.TempDir(os.TempDir(), "max-upload4") - bzzHash4 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir4) - mount4, _ := ioutil.TempDir(os.TempDir(), "max-mount4") - swarmfs4 := mountDir(t, ta.api, files, bzzHash4, mount4) - defer swarmfs4.Stop() - - files["5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir5, _ := ioutil.TempDir(os.TempDir(), "max-upload5") - bzzHash5 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir5) - mount5, _ := ioutil.TempDir(os.TempDir(), "max-mount5") - swarmfs5 := mountDir(t, ta.api, files, bzzHash5, mount5) - defer swarmfs5.Stop() - - files["6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir6, _ := ioutil.TempDir(os.TempDir(), "max-upload6") - bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir6) - mount6, _ := ioutil.TempDir(os.TempDir(), "max-mount6") - - os.RemoveAll(mount6) - os.MkdirAll(mount6, 0777) - _, err := swarmfs.Mount(bzzHash6, mount6) + _, err = dat.swarmfs.Unmount(dat.testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", dat.bzzHash) + } + log.Debug("Unmount successful") + if !isDirEmpty(dat.testMountDir) { + t.Fatalf("unmount didnt work for %v", dat.testMountDir) + } + log.Debug("subtest terminated") +} + +func (ta *testAPI) maxMountsEncrypted(t *testing.T) { + log.Debug("Starting maxMountsEncrypted test") + ta.runMaxMounts(t, true) + log.Debug("Test maxMountsEncrypted terminated") +} + +func (ta *testAPI) maxMountsNonEncrypted(t *testing.T) { + log.Debug("Starting maxMountsNonEncrypted test") + ta.runMaxMounts(t, false) + log.Debug("Test maxMountsNonEncrypted terminated") +} + +//mount several different directories until the maximum has been reached +func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("runMaxMounts") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) + + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1") + dat.testMountDir = filepath.Join(dat.testDir, "max-mount1") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() + + dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2") + dat.testMountDir = filepath.Join(dat.testDir, "max-mount2") + dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + + dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3") + dat.testMountDir = filepath.Join(dat.testDir, "max-mount3") + dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + + dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4") + dat.testMountDir = filepath.Join(dat.testDir, "max-mount4") + dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + + dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5") + dat.testMountDir = filepath.Join(dat.testDir, "max-mount5") + dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + + //now try an additional mount, should fail due to max mounts reached + testUploadDir6 := filepath.Join(dat.testDir, "max-upload6") + err = os.MkdirAll(testUploadDir6, 0777) + if err != nil { + t.Fatalf("Couldn't create upload dir 6: %v", err) + } + dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + testMountDir6 := filepath.Join(dat.testDir, "max-mount6") + err = os.MkdirAll(testMountDir6, 0777) + if err != nil { + t.Fatalf("Couldn't create mount dir 5: %v", err) + } + bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, dat.files, testUploadDir6, toEncrypt) + log.Debug("Created test files and uploaded to swarm with uploadDir6") + _, err = dat.swarmfs.Mount(bzzHash6, testMountDir6) if err == nil { - t.Fatalf("Error: Going beyond max mounts %v", bzzHash6) + t.Fatalf("Expected this mount to fail due to exceeding max number of allowed mounts, but succeeded. %v", bzzHash6) } + log.Debug("Maximum mount reached, additional mount failed. Correct.") +} +func (ta *testAPI) remountEncrypted(t *testing.T) { + log.Debug("Starting remountEncrypted test") + ta.remount(t, true) + log.Debug("Test remountEncrypted terminated") +} +func (ta *testAPI) remountNonEncrypted(t *testing.T) { + log.Debug("Starting remountNonEncrypted test") + ta.remount(t, false) + log.Debug("Test remountNonEncrypted terminated") } -func (ta *testAPI) remount(t *testing.T) { - files := make(map[string]fileInfo) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir1, _ := ioutil.TempDir(os.TempDir(), "re-upload1") - bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1) - testMountDir1, _ := ioutil.TempDir(os.TempDir(), "re-mount1") - swarmfs := mountDir(t, ta.api, files, bzzHash1, testMountDir1) - defer swarmfs.Stop() +//test remounting same hash second time and different hash in already mounted point +func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("remount") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - uploadDir2, _ := ioutil.TempDir(os.TempDir(), "re-upload2") - bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2) - testMountDir2, _ := ioutil.TempDir(os.TempDir(), "re-mount2") + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "remount-upload1") + dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1") + dat.files = make(map[string]fileInfo) - // try mounting the same hash second time - os.RemoveAll(testMountDir2) - os.MkdirAll(testMountDir2, 0777) - _, err := swarmfs.Mount(bzzHash1, testMountDir2) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) if err != nil { - t.Fatalf("Error mounting hash %v", bzzHash1) + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() + + // try mounting the same hash second time + testMountDir2, err2 := addDir(dat.testDir, "remount-mount2") + if err2 != nil { + t.Fatalf("Error creating second mount dir: %v", err2) + } + _, err2 = dat.swarmfs.Mount(dat.bzzHash, testMountDir2) + if err2 != nil { + t.Fatalf("Error mounting hash second time on different dir %v", dat.bzzHash) } // mount a different hash in already mounted point - _, err = swarmfs.Mount(bzzHash2, testMountDir1) + dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2") + if err3 != nil { + t.Fatalf("Error creating second upload dir: %v", err3) + } + bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, dat.files, testUploadDir2, toEncrypt) + _, err = swarmfs.Mount(bzzHash2, dat.testMountDir) if err == nil { t.Fatalf("Error mounting hash %v", bzzHash2) } + log.Debug("Mount on existing mount point failed. Correct.") // mount nonexistent hash - _, err = swarmfs.Mount("0xfea11223344", testMountDir1) + failDir, err3 := addDir(dat.testDir, "remount-fail") + if err3 != nil { + t.Fatalf("Error creating remount dir: %v", bzzHash2) + } + failHash := "0xfea11223344" + _, err = swarmfs.Mount(failHash, failDir) if err == nil { - t.Fatalf("Error mounting hash %v", bzzHash2) + t.Fatalf("Expected this mount to fail due to non existing hash. But succeeded %v", failHash) } + log.Debug("Nonexistent hash hasn't been mounted. Correct.") +} + +func (ta *testAPI) unmountEncrypted(t *testing.T) { + log.Debug("Starting unmountEncrypted test") + ta.unmount(t, true) + log.Debug("Test unmountEncrypted terminated") +} + +func (ta *testAPI) unmountNonEncrypted(t *testing.T) { + log.Debug("Starting unmountNonEncrypted test") + ta.unmount(t, false) + log.Debug("Test unmountNonEncrypted terminated") } -func (ta *testAPI) unmount(t *testing.T) { - files := make(map[string]fileInfo) - uploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") +//mount then unmount and check that it has been unmounted +func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("unmount") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir) + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") + dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() - swarmfs.Unmount(testMountDir) + _, err = dat.swarmfs.Unmount(dat.testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", dat.bzzHash) + } + log.Debug("Unmounted Dir") mi := swarmfs.Listmounts() + log.Debug("Going to list mounts") for _, minfo := range mi { - if minfo.MountPoint == testMountDir { - t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + log.Debug("Mount point in list: ", "point", minfo.MountPoint) + if minfo.MountPoint == dat.testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", dat.testMountDir) } } + log.Debug("subtest terminated") } -func (ta *testAPI) unmountWhenResourceBusy(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") +func (ta *testAPI) unmountWhenResourceBusyEncrypted(t *testing.T) { + log.Debug("Starting unmountWhenResourceBusyEncrypted test") + ta.unmountWhenResourceBusy(t, true) + log.Debug("Test unmountWhenResourceBusyEncrypted terminated") +} +func (ta *testAPI) unmountWhenResourceBusyNonEncrypted(t *testing.T) { + log.Debug("Starting unmountWhenResourceBusyNonEncrypted test") + ta.unmountWhenResourceBusy(t, false) + log.Debug("Test unmountWhenResourceBusyNonEncrypted terminated") +} - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +//unmount while a resource is busy; should fail +func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("unmountWhenResourceBusy") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") + dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - actualPath := filepath.Join(testMountDir, "2.txt") - d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700)) - d.Write(getRandomBtes(10)) + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() - _, err = swarmfs.Unmount(testMountDir) + //create a file in the mounted directory, then try to unmount - should fail + actualPath := filepath.Join(dat.testMountDir, "2.txt") + //d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700)) + d, err := os.Create(actualPath) if err != nil { - t.Fatalf("could not unmount %v", bzzHash) + t.Fatalf("Couldn't create new file: %v", err) } - d.Close() + //we need to manually close the file before mount for this test + //but let's defer too in case of errors + defer d.Close() + _, err = d.Write(getRandomBytes(10)) + if err != nil { + t.Fatalf("Couldn't write to file: %v", err) + } + log.Debug("Bytes written") - mi := swarmfs.Listmounts() + _, err = dat.swarmfs.Unmount(dat.testMountDir) + if err == nil { + t.Fatalf("Expected mount to fail due to resource busy, but it succeeded...") + } + //free resources + err = d.Close() + if err != nil { + t.Fatalf("Couldn't close file! %v", dat.bzzHash) + } + log.Debug("File closed") + + //now unmount after explicitly closed file + _, err = dat.swarmfs.Unmount(dat.testMountDir) + if err != nil { + t.Fatalf("Expected mount to succeed after freeing resource, but it failed: %v", err) + } + //check if the dir is still mounted + mi := dat.swarmfs.Listmounts() + log.Debug("Going to list mounts") for _, minfo := range mi { - if minfo.MountPoint == testMountDir { - t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + log.Debug("Mount point in list: ", "point", minfo.MountPoint) + if minfo.MountPoint == dat.testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", dat.testMountDir) } } + log.Debug("subtest terminated") } -func (ta *testAPI) seekInMultiChunkFile(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "seek-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "seek-mount") +func (ta *testAPI) seekInMultiChunkFileEncrypted(t *testing.T) { + log.Debug("Starting seekInMultiChunkFileEncrypted test") + ta.seekInMultiChunkFile(t, true) + log.Debug("Test seekInMultiChunkFileEncrypted terminated") +} - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10240)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +func (ta *testAPI) seekInMultiChunkFileNonEncrypted(t *testing.T) { + log.Debug("Starting seekInMultiChunkFileNonEncrypted test") + ta.seekInMultiChunkFile(t, false) + log.Debug("Test seekInMultiChunkFileNonEncrypted terminated") +} - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() +//open a file in a mounted dir and go to a certain position +func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("seekInMultiChunkFile") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - // Create a new file seek the second chunk - actualPath := filepath.Join(testMountDir, "1.txt") - d, _ := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700)) + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1") + dat.testMountDir = filepath.Join(dat.testDir, "seek-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)} - d.Seek(5000, 0) + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() + + // Open the file in the mounted dir and seek the second chunk + actualPath := filepath.Join(dat.testMountDir, "1.txt") + d, err := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700)) + if err != nil { + t.Fatalf("Couldn't open file: %v", err) + } + log.Debug("Opened file") + defer func() { + err := d.Close() + if err != nil { + t.Fatalf("Error closing file! %v", err) + } + }() + + _, err = d.Seek(5000, 0) + if err != nil { + t.Fatalf("Error seeking in file: %v", err) + } contents := make([]byte, 1024) - d.Read(contents) - finfo := files["1.txt"] + _, err = d.Read(contents) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + log.Debug("Read contents") + finfo := dat.files["1.txt"] if !bytes.Equal(finfo.contents[:6024][5000:], contents) { t.Fatalf("File seek contents mismatch") } - d.Close() + log.Debug("subtest terminated") +} + +func (ta *testAPI) createNewFileEncrypted(t *testing.T) { + log.Debug("Starting createNewFileEncrypted test") + ta.createNewFile(t, true) + log.Debug("Test createNewFileEncrypted terminated") +} + +func (ta *testAPI) createNewFileNonEncrypted(t *testing.T) { + log.Debug("Starting createNewFileNonEncrypted test") + ta.createNewFile(t, false) + log.Debug("Test createNewFileNonEncrypted terminated") } -func (ta *testAPI) createNewFile(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "create-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "create-mount") +//create a new file in a mounted swarm directory, +//unmount the fuse dir and then remount to see if new file is still there +func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("createNewFile") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1") + dat.testMountDir = filepath.Join(dat.testDir, "create-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() // Create a new file in the root dir and check - actualPath := filepath.Join(testMountDir, "2.txt") + actualPath := filepath.Join(dat.testMountDir, "2.txt") d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) if err1 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err1) + t.Fatalf("Could not open file %s : %v", actualPath, err1) } + defer d.Close() + log.Debug("Opened file") contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() + _, err = rand.Read(contents) + if err != nil { + t.Fatalf("Could not rand read contents %v", err) + } + log.Debug("content read") + _, err = d.Write(contents) + if err != nil { + t.Fatalf("Couldn't write contents: %v", err) + } + log.Debug("content written") + err = d.Close() + if err != nil { + t.Fatalf("Couldn't close file: %v", err) + } + log.Debug("file closed") - mi, err2 := swarmfs1.Unmount(testMountDir) + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") + testMountDir2, err3 := addDir(dat.testDir, "create-mount2") + if err3 != nil { + t.Fatalf("Error creating mount dir2: %v", err3) + } // mount again and see if things are okay - files["2.txt"] = fileInfo{0700, 333, 444, contents} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + dat.files["2.txt"] = fileInfo{0700, 333, 444, contents} + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, testMountDir2) + log.Debug("Directory mounted again") + + checkFile(t, testMountDir2, "2.txt", contents) + _, err2 = dat.swarmfs.Unmount(testMountDir2) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + log.Debug("subtest terminated") +} - checkFile(t, testMountDir, "2.txt", contents) +func (ta *testAPI) createNewFileInsideDirectoryEncrypted(t *testing.T) { + log.Debug("Starting createNewFileInsideDirectoryEncrypted test") + ta.createNewFileInsideDirectory(t, true) + log.Debug("Test createNewFileInsideDirectoryEncrypted terminated") } -func (ta *testAPI) createNewFileInsideDirectory(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-mount") +func (ta *testAPI) createNewFileInsideDirectoryNonEncrypted(t *testing.T) { + log.Debug("Starting createNewFileInsideDirectoryNonEncrypted test") + ta.createNewFileInsideDirectory(t, false) + log.Debug("Test createNewFileInsideDirectoryNonEncrypted terminated") +} - files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +//create a new file inside a directory inside the mount +func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("createNewFileInsideDirectory") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload") + dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount") + dat.files = make(map[string]fileInfo) + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() // Create a new file inside a existing dir and check - dirToCreate := filepath.Join(testMountDir, "one") + dirToCreate := filepath.Join(dat.testMountDir, "one") actualPath := filepath.Join(dirToCreate, "2.txt") d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) if err1 != nil { t.Fatalf("Could not create file %s : %v", actualPath, err1) } + defer d.Close() + log.Debug("File opened") contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() + _, err = rand.Read(contents) + if err != nil { + t.Fatalf("Error filling random bytes into byte array %v", err) + } + log.Debug("Content read") + _, err = d.Write(contents) + if err != nil { + t.Fatalf("Error writing random bytes into file %v", err) + } + log.Debug("Content written") + err = d.Close() + if err != nil { + t.Fatalf("Error closing file %v", err) + } + log.Debug("File closed") - mi, err2 := swarmfs1.Unmount(testMountDir) + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") + testMountDir2, err3 := addDir(dat.testDir, "createinsidedir-mount2") + if err3 != nil { + t.Fatalf("Error creating mount dir2: %v", err3) + } // mount again and see if things are okay - files["one/2.txt"] = fileInfo{0700, 333, 444, contents} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + dat.files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, testMountDir2) + log.Debug("Directory mounted again") - checkFile(t, testMountDir, "one/2.txt", contents) + checkFile(t, testMountDir2, "one/2.txt", contents) + _, err = dat.swarmfs.Unmount(testMountDir2) + if err != nil { + t.Fatalf("could not unmount %v", dat.bzzHash) + } + log.Debug("subtest terminated") +} + +func (ta *testAPI) createNewFileInsideNewDirectoryEncrypted(t *testing.T) { + log.Debug("Starting createNewFileInsideNewDirectoryEncrypted test") + ta.createNewFileInsideNewDirectory(t, true) + log.Debug("Test createNewFileInsideNewDirectoryEncrypted terminated") } -func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-mount") +func (ta *testAPI) createNewFileInsideNewDirectoryNonEncrypted(t *testing.T) { + log.Debug("Starting createNewFileInsideNewDirectoryNonEncrypted test") + ta.createNewFileInsideNewDirectory(t, false) + log.Debug("Test createNewFileInsideNewDirectoryNonEncrypted terminated") +} - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +//create a new directory in mount and a new file +func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("createNewFileInsideNewDirectory") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload") + dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() // Create a new file inside a existing dir and check - dirToCreate := filepath.Join(testMountDir, "one") - os.MkdirAll(dirToCreate, 0777) + dirToCreate, err2 := addDir(dat.testMountDir, "one") + if err2 != nil { + t.Fatalf("Error creating mount dir2: %v", err2) + } actualPath := filepath.Join(dirToCreate, "2.txt") d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) if err1 != nil { t.Fatalf("Could not create file %s : %v", actualPath, err1) } + defer d.Close() + log.Debug("File opened") contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() + _, err = rand.Read(contents) + if err != nil { + t.Fatalf("Error writing random bytes to byte array: %v", err) + } + log.Debug("content read") + _, err = d.Write(contents) + if err != nil { + t.Fatalf("Error writing to file: %v", err) + } + log.Debug("content written") + err = d.Close() + if err != nil { + t.Fatalf("Error closing file: %v", err) + } + log.Debug("File closed") - mi, err2 := swarmfs1.Unmount(testMountDir) + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") // mount again and see if things are okay - files["one/2.txt"] = fileInfo{0700, 333, 444, contents} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + dat.files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, dat.testMountDir) + log.Debug("Directory mounted again") + + checkFile(t, dat.testMountDir, "one/2.txt", contents) + _, err2 = dat.swarmfs.Unmount(dat.testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + log.Debug("subtest terminated") +} - checkFile(t, testMountDir, "one/2.txt", contents) +func (ta *testAPI) removeExistingFileEncrypted(t *testing.T) { + log.Debug("Starting removeExistingFileEncrypted test") + ta.removeExistingFile(t, true) + log.Debug("Test removeExistingFileEncrypted terminated") } -func (ta *testAPI) removeExistingFile(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") +func (ta *testAPI) removeExistingFileNonEncrypted(t *testing.T) { + log.Debug("Starting removeExistingFileNonEncrypted test") + ta.removeExistingFile(t, false) + log.Debug("Test removeExistingFileNonEncrypted terminated") +} - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +//remove existing file in mount +func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("removeExistingFile") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") + dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - // Remove a file in the root dir and check - actualPath := filepath.Join(testMountDir, "five.txt") - os.Remove(actualPath) + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() - mi, err2 := swarmfs1.Unmount(testMountDir) + // Remove a file in the root dir and check + actualPath := filepath.Join(dat.testMountDir, "five.txt") + err = os.Remove(actualPath) + if err != nil { + t.Fatalf("Error removing file! %v", err) + } + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") // mount again and see if things are okay - delete(files, "five.txt") - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + delete(dat.files, "five.txt") + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, dat.testMountDir) + _, err = os.Stat(actualPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + _, err2 = dat.swarmfs.Unmount(dat.testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + log.Debug("subtest terminated") +} + +func (ta *testAPI) removeExistingFileInsideDirEncrypted(t *testing.T) { + log.Debug("Starting removeExistingFileInsideDirEncrypted test") + ta.removeExistingFileInsideDir(t, true) + log.Debug("Test removeExistingFileInsideDirEncrypted terminated") +} + +func (ta *testAPI) removeExistingFileInsideDirNonEncrypted(t *testing.T) { + log.Debug("Starting removeExistingFileInsideDirNonEncrypted test") + ta.removeExistingFileInsideDir(t, false) + log.Debug("Test removeExistingFileInsideDirNonEncrypted terminated") } -func (ta *testAPI) removeExistingFileInsideDir(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") +//remove a file inside a directory inside a mount +func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("removeExistingFileInsideDir") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") + dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() // Remove a file in the root dir and check - actualPath := filepath.Join(testMountDir, "one/five.txt") - os.Remove(actualPath) - - mi, err2 := swarmfs1.Unmount(testMountDir) + actualPath := filepath.Join(dat.testMountDir, "one") + actualPath = filepath.Join(actualPath, "five.txt") + err = os.Remove(actualPath) + if err != nil { + t.Fatalf("Error removing file! %v", err) + } + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") // mount again and see if things are okay - delete(files, "one/five.txt") - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + delete(dat.files, "one/five.txt") + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, dat.testMountDir) + _, err = os.Stat(actualPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + + okPath := filepath.Join(dat.testMountDir, "one") + okPath = filepath.Join(okPath, "six.txt") + _, err = os.Stat(okPath) + if err != nil { + t.Fatal("Expected file to be present in re-mount after removal, but it is not there") + } + _, err2 = dat.swarmfs.Unmount(dat.testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + log.Debug("subtest terminated") +} + +func (ta *testAPI) removeNewlyAddedFileEncrypted(t *testing.T) { + log.Debug("Starting removeNewlyAddedFileEncrypted test") + ta.removeNewlyAddedFile(t, true) + log.Debug("Test removeNewlyAddedFileEncrypted terminated") } -func (ta *testAPI) removeNewlyAddedFile(t *testing.T) { +func (ta *testAPI) removeNewlyAddedFileNonEncrypted(t *testing.T) { + log.Debug("Starting removeNewlyAddedFileNonEncrypted test") + ta.removeNewlyAddedFile(t, false) + log.Debug("Test removeNewlyAddedFileNonEncrypted terminated") +} - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "removenew-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "removenew-mount") +//add a file in mount and then remove it; on remount file should not be there +func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("removeNewlyAddedFile") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload") + dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() - // Adda a new file and remove it - dirToCreate := filepath.Join(testMountDir, "one") - os.MkdirAll(dirToCreate, os.FileMode(0665)) + // Add a a new file and remove it + dirToCreate := filepath.Join(dat.testMountDir, "one") + err = os.MkdirAll(dirToCreate, os.FileMode(0665)) + if err != nil { + t.Fatalf("Error creating dir in mounted dir: %v", err) + } actualPath := filepath.Join(dirToCreate, "2.txt") d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) if err1 != nil { t.Fatalf("Could not create file %s : %v", actualPath, err1) } + defer d.Close() + log.Debug("file opened") contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() + _, err = rand.Read(contents) + if err != nil { + t.Fatalf("Error writing random bytes to byte array: %v", err) + } + log.Debug("content read") + _, err = d.Write(contents) + if err != nil { + t.Fatalf("Error writing random bytes to file: %v", err) + } + log.Debug("content written") + err = d.Close() + if err != nil { + t.Fatalf("Error closing file: %v", err) + } + log.Debug("file closed") - checkFile(t, testMountDir, "one/2.txt", contents) + checkFile(t, dat.testMountDir, "one/2.txt", contents) + log.Debug("file checked") - os.Remove(actualPath) + err = os.Remove(actualPath) + if err != nil { + t.Fatalf("Error removing file: %v", err) + } + log.Debug("file removed") - mi, err2 := swarmfs1.Unmount(testMountDir) + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") + testMountDir2, err3 := addDir(dat.testDir, "removenew-mount2") + if err3 != nil { + t.Fatalf("Error creating mount dir2: %v", err3) + } // mount again and see if things are okay - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, testMountDir2) + log.Debug("Directory mounted again") - if bzzHash != mi.LatestManifest { - t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + if dat.bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", dat.bzzHash, mi.LatestManifest) } + _, err2 = dat.swarmfs.Unmount(testMountDir2) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + log.Debug("subtest terminated") } -func (ta *testAPI) addNewFileAndModifyContents(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-mount") +func (ta *testAPI) addNewFileAndModifyContentsEncrypted(t *testing.T) { + log.Debug("Starting addNewFileAndModifyContentsEncrypted test") + ta.addNewFileAndModifyContents(t, true) + log.Debug("Test addNewFileAndModifyContentsEncrypted terminated") +} - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +func (ta *testAPI) addNewFileAndModifyContentsNonEncrypted(t *testing.T) { + log.Debug("Starting addNewFileAndModifyContentsNonEncrypted test") + ta.addNewFileAndModifyContents(t, false) + log.Debug("Test addNewFileAndModifyContentsNonEncrypted terminated") +} - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() +//add a new file and modify content; remount and check the modified file is intact +func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("addNewFileAndModifyContents") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - // Create a new file in the root dir and check - actualPath := filepath.Join(testMountDir, "2.txt") + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload") + dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() + + // Create a new file in the root dir + actualPath := filepath.Join(dat.testMountDir, "2.txt") d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) if err1 != nil { t.Fatalf("Could not create file %s : %v", actualPath, err1) } + defer d.Close() + //write some random data into the file + log.Debug("file opened") line1 := []byte("Line 1") - rand.Read(line1) - d.Write(line1) - d.Close() + _, err = rand.Read(line1) + if err != nil { + t.Fatalf("Error writing random bytes to byte array: %v", err) + } + log.Debug("line read") + _, err = d.Write(line1) + if err != nil { + t.Fatalf("Error writing random bytes to file: %v", err) + } + log.Debug("line written") + err = d.Close() + if err != nil { + t.Fatalf("Error closing file: %v", err) + } + log.Debug("file closed") - mi1, err2 := swarmfs1.Unmount(testMountDir) + //unmount the hash on the mounted dir + mi1, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v", err2) } + log.Debug("Directory unmounted") - // mount again and see if things are okay - files["2.txt"] = fileInfo{0700, 333, 444, line1} - swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - checkFile(t, testMountDir, "2.txt", line1) - - mi2, err3 := swarmfs2.Unmount(testMountDir) + //mount on a different dir to see if modified file is correct + testMountDir2, err3 := addDir(dat.testDir, "modifyfile-mount2") if err3 != nil { - t.Fatalf("Could not unmount %v", err3) + t.Fatalf("Error creating mount dir2: %v", err3) } + dat.files["2.txt"] = fileInfo{0700, 333, 444, line1} + _ = mountDir(t, ta.api, dat.files, mi1.LatestManifest, testMountDir2) + log.Debug("Directory mounted again") - // mount again and modify - swarmfs3 := mountDir(t, ta.api, files, mi2.LatestManifest, testMountDir) - defer swarmfs3.Stop() + checkFile(t, testMountDir2, "2.txt", line1) + log.Debug("file checked") - fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + //unmount second dir + mi2, err4 := dat.swarmfs.Unmount(testMountDir2) if err4 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err4) + t.Fatalf("Could not unmount %v", err4) } - line2 := []byte("Line 2") - rand.Read(line2) - fd.Seek(int64(len(line1)), 0) - fd.Write(line2) - fd.Close() + log.Debug("Directory unmounted again") + + //mount again on original dir and modify the file + //let's clean up the mounted dir first: remove... + err = os.RemoveAll(dat.testMountDir) + if err != nil { + t.Fatalf("Error cleaning up mount dir: %v", err) + } + //...and re-create + err = os.MkdirAll(dat.testMountDir, 0777) + if err != nil { + t.Fatalf("Error re-creating mount dir: %v", err) + } + //now remount + _ = mountDir(t, ta.api, dat.files, mi2.LatestManifest, dat.testMountDir) + log.Debug("Directory mounted yet again") - mi3, err5 := swarmfs3.Unmount(testMountDir) + //open the file.... + fd, err5 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) if err5 != nil { - t.Fatalf("Could not unmount %v", err5) + t.Fatalf("Could not create file %s : %v", actualPath, err5) + } + defer fd.Close() + log.Debug("file opened") + //...and modify something + line2 := []byte("Line 2") + _, err = rand.Read(line2) + if err != nil { + t.Fatalf("Error modifying random bytes to byte array: %v", err) + } + log.Debug("line read") + _, err = fd.Seek(int64(len(line1)), 0) + if err != nil { + t.Fatalf("Error seeking position for modification: %v", err) } + _, err = fd.Write(line2) + if err != nil { + t.Fatalf("Error modifying file: %v", err) + } + log.Debug("line written") + err = fd.Close() + if err != nil { + t.Fatalf("Error closing modified file; %v", err) + } + log.Debug("file closed") - // mount again and see if things are okay + //unmount the modified directory + mi3, err6 := dat.swarmfs.Unmount(dat.testMountDir) + if err6 != nil { + t.Fatalf("Could not unmount %v", err6) + } + log.Debug("Directory unmounted yet again") + + //now remount on a different dir and check that the modified file is ok + testMountDir4, err7 := addDir(dat.testDir, "modifyfile-mount4") + if err7 != nil { + t.Fatalf("Could not unmount %v", err7) + } b := [][]byte{line1, line2} line1and2 := bytes.Join(b, []byte("")) - files["2.txt"] = fileInfo{0700, 333, 444, line1and2} - swarmfs4 := mountDir(t, ta.api, files, mi3.LatestManifest, testMountDir) - defer swarmfs4.Stop() + dat.files["2.txt"] = fileInfo{0700, 333, 444, line1and2} + _ = mountDir(t, ta.api, dat.files, mi3.LatestManifest, testMountDir4) + log.Debug("Directory mounted final time") - checkFile(t, testMountDir, "2.txt", line1and2) + checkFile(t, testMountDir4, "2.txt", line1and2) + _, err = dat.swarmfs.Unmount(testMountDir4) + if err != nil { + t.Fatalf("Could not unmount %v", err) + } + log.Debug("subtest terminated") } -func (ta *testAPI) removeEmptyDir(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") +func (ta *testAPI) removeEmptyDirEncrypted(t *testing.T) { + log.Debug("Starting removeEmptyDirEncrypted test") + ta.removeEmptyDir(t, true) + log.Debug("Test removeEmptyDirEncrypted terminated") +} - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +func (ta *testAPI) removeEmptyDirNonEncrypted(t *testing.T) { + log.Debug("Starting removeEmptyDirNonEncrypted test") + ta.removeEmptyDir(t, false) + log.Debug("Test removeEmptyDirNonEncrypted terminated") +} - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() +//remove an empty dir inside mount +func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("removeEmptyDir") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - os.MkdirAll(filepath.Join(testMountDir, "newdir"), 0777) + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") + dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") + dat.files = make(map[string]fileInfo) + dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - mi, err3 := swarmfs1.Unmount(testMountDir) - if err3 != nil { - t.Fatalf("Could not unmount %v", err3) + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) } - if bzzHash != mi.LatestManifest { - t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + defer dat.swarmfs.Stop() + + _, err2 := addDir(dat.testMountDir, "newdir") + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + mi, err := dat.swarmfs.Unmount(dat.testMountDir) + if err != nil { + t.Fatalf("Could not unmount %v", err) + } + log.Debug("Directory unmounted") + //by just adding an empty dir, the hash doesn't change; test this + if dat.bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", dat.bzzHash, mi.LatestManifest) } + log.Debug("subtest terminated") } -func (ta *testAPI) removeDirWhichHasFiles(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") +func (ta *testAPI) removeDirWhichHasFilesEncrypted(t *testing.T) { + log.Debug("Starting removeDirWhichHasFilesEncrypted test") + ta.removeDirWhichHasFiles(t, true) + log.Debug("Test removeDirWhichHasFilesEncrypted terminated") +} +func (ta *testAPI) removeDirWhichHasFilesNonEncrypted(t *testing.T) { + log.Debug("Starting removeDirWhichHasFilesNonEncrypted test") + ta.removeDirWhichHasFiles(t, false) + log.Debug("Test removeDirWhichHasFilesNonEncrypted terminated") +} - files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +//remove a directory with a file; check on remount file isn't there +func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("removeDirWhichHasFiles") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") + dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") + dat.files = make(map[string]fileInfo) + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dirPath := filepath.Join(testMountDir, "two") - os.RemoveAll(dirPath) + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() - mi, err2 := swarmfs1.Unmount(testMountDir) + //delete a directory inside the mounted dir with all its files + dirPath := filepath.Join(dat.testMountDir, "two") + err = os.RemoveAll(dirPath) + if err != nil { + t.Fatalf("Error removing directory in mounted dir: %v", err) + } + + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v ", err2) } + log.Debug("Directory unmounted") - // mount again and see if things are okay - delete(files, "two/five.txt") - delete(files, "two/six.txt") + //we deleted files in the OS, so let's delete them also in the files map + delete(dat.files, "two/five.txt") + delete(dat.files, "two/six.txt") - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() + // mount again and see if deleted files have been deleted indeed + testMountDir2, err3 := addDir(dat.testDir, "remount-mount2") + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, testMountDir2) + log.Debug("Directory mounted") + actualPath := filepath.Join(dirPath, "five.txt") + _, err = os.Stat(actualPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + _, err = os.Stat(dirPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + _, err = dat.swarmfs.Unmount(testMountDir2) + if err != nil { + t.Fatalf("Could not unmount %v", err) + } + log.Debug("subtest terminated") } -func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-mount") - - files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} +func (ta *testAPI) removeDirWhichHasSubDirsEncrypted(t *testing.T) { + log.Debug("Starting removeDirWhichHasSubDirsEncrypted test") + ta.removeDirWhichHasSubDirs(t, true) + log.Debug("Test removeDirWhichHasSubDirsEncrypted terminated") +} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) +func (ta *testAPI) removeDirWhichHasSubDirsNonEncrypted(t *testing.T) { + log.Debug("Starting removeDirWhichHasSubDirsNonEncrypted test") + ta.removeDirWhichHasSubDirs(t, false) + log.Debug("Test removeDirWhichHasSubDirsNonEncrypted terminated") +} - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() +//remove a directory with subdirectories inside mount; on remount check they are not there +func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("removeDirWhichHasSubDirs") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) + + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload") + dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount") + dat.files = make(map[string]fileInfo) + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() - dirPath := filepath.Join(testMountDir, "two") - os.RemoveAll(dirPath) + dirPath := filepath.Join(dat.testMountDir, "two") + err = os.RemoveAll(dirPath) + if err != nil { + t.Fatalf("Error removing directory in mounted dir: %v", err) + } - mi, err2 := swarmfs1.Unmount(testMountDir) + //delete a directory inside the mounted dir with all its files + mi, err2 := dat.swarmfs.Unmount(dat.testMountDir) if err2 != nil { t.Fatalf("Could not unmount %v ", err2) } + log.Debug("Directory unmounted") + + //we deleted files in the OS, so let's delete them also in the files map + delete(dat.files, "two/three/2.txt") + delete(dat.files, "two/three/3.txt") + delete(dat.files, "two/four/5.txt") + delete(dat.files, "two/four/6.txt") + delete(dat.files, "two/four/six/7.txt") // mount again and see if things are okay - delete(files, "two/three/2.txt") - delete(files, "two/three/3.txt") - delete(files, "two/four/5.txt") - delete(files, "two/four/6.txt") - delete(files, "two/four/six/7.txt") + testMountDir2, err3 := addDir(dat.testDir, "remount-mount2") + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + _ = mountDir(t, ta.api, dat.files, mi.LatestManifest, testMountDir2) + log.Debug("Directory mounted again") + actualPath := filepath.Join(dirPath, "three") + actualPath = filepath.Join(actualPath, "2.txt") + _, err = os.Stat(actualPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + actualPath = filepath.Join(dirPath, "four") + _, err = os.Stat(actualPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + _, err = os.Stat(dirPath) + if err == nil { + t.Fatal("Expected file to not be present in re-mount after removal, but it is there") + } + _, err = dat.swarmfs.Unmount(testMountDir2) + if err != nil { + t.Fatalf("Could not unmount %v", err) + } + log.Debug("subtest terminated") +} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() +func (ta *testAPI) appendFileContentsToEndEncrypted(t *testing.T) { + log.Debug("Starting appendFileContentsToEndEncrypted test") + ta.appendFileContentsToEnd(t, true) + log.Debug("Test appendFileContentsToEndEncrypted terminated") } -func (ta *testAPI) appendFileContentsToEnd(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-mount") +func (ta *testAPI) appendFileContentsToEndNonEncrypted(t *testing.T) { + log.Debug("Starting appendFileContentsToEndNonEncrypted test") + ta.appendFileContentsToEnd(t, false) + log.Debug("Test appendFileContentsToEndNonEncrypted terminated") +} + +//append contents to the end of a file; remount and check it's intact +func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { + dat, err := ta.initSubtest("appendFileContentsToEnd") + if err != nil { + t.Fatalf("Couldn't initialize subtest dirs: %v", err) + } + defer os.RemoveAll(dat.testDir) + + dat.toEncrypt = toEncrypt + dat.testUploadDir = filepath.Join(dat.testDir, "appendlargefile-upload") + dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount") + dat.files = make(map[string]fileInfo) line1 := make([]byte, 10) - rand.Read(line1) - files["1.txt"] = fileInfo{0700, 333, 444, line1} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + _, err = rand.Read(line1) + if err != nil { + t.Fatalf("Error writing random bytes to byte array: %v", err) + } - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + dat.files["1.txt"] = fileInfo{0700, 333, 444, line1} - actualPath := filepath.Join(testMountDir, "1.txt") + dat, err = ta.uploadAndMount(dat, t) + if err != nil { + t.Fatalf("Error during upload of files to swarm / mount of swarm dir: %v", err) + } + defer dat.swarmfs.Stop() + + actualPath := filepath.Join(dat.testMountDir, "1.txt") fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) if err4 != nil { t.Fatalf("Could not create file %s : %v", actualPath, err4) } + defer fd.Close() + log.Debug("file opened") line2 := make([]byte, 5) - rand.Read(line2) - fd.Seek(int64(len(line1)), 0) - fd.Write(line2) - fd.Close() + _, err = rand.Read(line2) + if err != nil { + t.Fatalf("Error writing random bytes to byte array: %v", err) + } + log.Debug("line read") + _, err = fd.Seek(int64(len(line1)), 0) + if err != nil { + t.Fatalf("Error searching for position to append: %v", err) + } + _, err = fd.Write(line2) + if err != nil { + t.Fatalf("Error appending: %v", err) + } + log.Debug("line written") + err = fd.Close() + if err != nil { + t.Fatalf("Error closing file: %v", err) + } + log.Debug("file closed") - mi1, err5 := swarmfs1.Unmount(testMountDir) + mi1, err5 := dat.swarmfs.Unmount(dat.testMountDir) if err5 != nil { t.Fatalf("Could not unmount %v ", err5) } + log.Debug("Directory unmounted") - // mount again and see if things are okay + // mount again and see if appended file is correct b := [][]byte{line1, line2} line1and2 := bytes.Join(b, []byte("")) - files["1.txt"] = fileInfo{0700, 333, 444, line1and2} - swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir) - defer swarmfs2.Stop() + dat.files["1.txt"] = fileInfo{0700, 333, 444, line1and2} + testMountDir2, err6 := addDir(dat.testDir, "remount-mount2") + if err6 != nil { + t.Fatalf("Could not unmount %v", err6) + } + _ = mountDir(t, ta.api, dat.files, mi1.LatestManifest, testMountDir2) + log.Debug("Directory mounted") + + checkFile(t, testMountDir2, "1.txt", line1and2) - checkFile(t, testMountDir, "1.txt", line1and2) + _, err = dat.swarmfs.Unmount(testMountDir2) + if err != nil { + t.Fatalf("Could not unmount %v", err) + } + log.Debug("subtest terminated") } +//run all the tests func TestFUSE(t *testing.T) { + //create a data directory for swarm datadir, err := ioutil.TempDir("", "fuse") if err != nil { t.Fatalf("unable to create temp dir: %v", err) } - os.RemoveAll(datadir) + defer os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32)) if err != nil { t.Fatal(err) } - ta := &testAPI{api: api.NewApi(dpa, nil)} - dpa.Start() - defer dpa.Stop() - - t.Run("mountListAndUmount", ta.mountListAndUnmount) - t.Run("maxMounts", ta.maxMounts) - t.Run("remount", ta.remount) - t.Run("unmount", ta.unmount) - t.Run("unmountWhenResourceBusy", ta.unmountWhenResourceBusy) - t.Run("seekInMultiChunkFile", ta.seekInMultiChunkFile) - t.Run("createNewFile", ta.createNewFile) - t.Run("createNewFileInsideDirectory", ta.createNewFileInsideDirectory) - t.Run("createNewFileInsideNewDirectory", ta.createNewFileInsideNewDirectory) - t.Run("removeExistingFile", ta.removeExistingFile) - t.Run("removeExistingFileInsideDir", ta.removeExistingFileInsideDir) - t.Run("removeNewlyAddedFile", ta.removeNewlyAddedFile) - t.Run("addNewFileAndModifyContents", ta.addNewFileAndModifyContents) - t.Run("removeEmptyDir", ta.removeEmptyDir) - t.Run("removeDirWhichHasFiles", ta.removeDirWhichHasFiles) - t.Run("removeDirWhichHasSubDirs", ta.removeDirWhichHasSubDirs) - t.Run("appendFileContentsToEnd", ta.appendFileContentsToEnd) + ta := &testAPI{api: api.NewAPI(fileStore, nil, nil)} + + //run a short suite of tests + //approx time: 28s + t.Run("mountListAndUnmountEncrypted", ta.mountListAndUnmountEncrypted) + t.Run("remountEncrypted", ta.remountEncrypted) + t.Run("unmountWhenResourceBusyNonEncrypted", ta.unmountWhenResourceBusyNonEncrypted) + t.Run("removeExistingFileEncrypted", ta.removeExistingFileEncrypted) + t.Run("addNewFileAndModifyContentsNonEncrypted", ta.addNewFileAndModifyContentsNonEncrypted) + t.Run("removeDirWhichHasFilesNonEncrypted", ta.removeDirWhichHasFilesNonEncrypted) + t.Run("appendFileContentsToEndEncrypted", ta.appendFileContentsToEndEncrypted) + + //provide longrunning flag to execute all tests + //approx time with longrunning: 140s + if *longrunning { + t.Run("mountListAndUnmountNonEncrypted", ta.mountListAndUnmountNonEncrypted) + t.Run("maxMountsEncrypted", ta.maxMountsEncrypted) + t.Run("maxMountsNonEncrypted", ta.maxMountsNonEncrypted) + t.Run("remountNonEncrypted", ta.remountNonEncrypted) + t.Run("unmountEncrypted", ta.unmountEncrypted) + t.Run("unmountNonEncrypted", ta.unmountNonEncrypted) + t.Run("unmountWhenResourceBusyEncrypted", ta.unmountWhenResourceBusyEncrypted) + t.Run("unmountWhenResourceBusyNonEncrypted", ta.unmountWhenResourceBusyNonEncrypted) + t.Run("seekInMultiChunkFileEncrypted", ta.seekInMultiChunkFileEncrypted) + t.Run("seekInMultiChunkFileNonEncrypted", ta.seekInMultiChunkFileNonEncrypted) + t.Run("createNewFileEncrypted", ta.createNewFileEncrypted) + t.Run("createNewFileNonEncrypted", ta.createNewFileNonEncrypted) + t.Run("createNewFileInsideDirectoryEncrypted", ta.createNewFileInsideDirectoryEncrypted) + t.Run("createNewFileInsideDirectoryNonEncrypted", ta.createNewFileInsideDirectoryNonEncrypted) + t.Run("createNewFileInsideNewDirectoryEncrypted", ta.createNewFileInsideNewDirectoryEncrypted) + t.Run("createNewFileInsideNewDirectoryNonEncrypted", ta.createNewFileInsideNewDirectoryNonEncrypted) + t.Run("removeExistingFileNonEncrypted", ta.removeExistingFileNonEncrypted) + t.Run("removeExistingFileInsideDirEncrypted", ta.removeExistingFileInsideDirEncrypted) + t.Run("removeExistingFileInsideDirNonEncrypted", ta.removeExistingFileInsideDirNonEncrypted) + t.Run("removeNewlyAddedFileEncrypted", ta.removeNewlyAddedFileEncrypted) + t.Run("removeNewlyAddedFileNonEncrypted", ta.removeNewlyAddedFileNonEncrypted) + t.Run("addNewFileAndModifyContentsEncrypted", ta.addNewFileAndModifyContentsEncrypted) + t.Run("removeEmptyDirEncrypted", ta.removeEmptyDirEncrypted) + t.Run("removeEmptyDirNonEncrypted", ta.removeEmptyDirNonEncrypted) + t.Run("removeDirWhichHasFilesEncrypted", ta.removeDirWhichHasFilesEncrypted) + t.Run("removeDirWhichHasSubDirsEncrypted", ta.removeDirWhichHasSubDirsEncrypted) + t.Run("removeDirWhichHasSubDirsNonEncrypted", ta.removeDirWhichHasSubDirsNonEncrypted) + t.Run("appendFileContentsToEndNonEncrypted", ta.appendFileContentsToEndNonEncrypted) + } } diff --git a/swarm/fuse/swarmfs_unix.go b/swarm/fuse/swarmfs_unix.go index 75742845a..74dd84a90 100644 --- a/swarm/fuse/swarmfs_unix.go +++ b/swarm/fuse/swarmfs_unix.go @@ -30,15 +30,16 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/log" ) var ( - errEmptyMountPoint = errors.New("need non-empty mount point") - errMaxMountCount = errors.New("max FUSE mount count reached") - errMountTimeout = errors.New("mount timeout") - errAlreadyMounted = errors.New("mount point is already serving") + errEmptyMountPoint = errors.New("need non-empty mount point") + errNoRelativeMountPoint = errors.New("invalid path for mount point (need absolute path)") + errMaxMountCount = errors.New("max FUSE mount count reached") + errMountTimeout = errors.New("mount timeout") + errAlreadyMounted = errors.New("mount point is already serving") ) func isFUSEUnsupportedError(err error) bool { @@ -48,18 +49,20 @@ func isFUSEUnsupportedError(err error) bool { return err == fuse.ErrOSXFUSENotFound } -// information about every active mount +// MountInfo contains information about every active mount type MountInfo struct { MountPoint string StartManifest string LatestManifest string rootDir *SwarmDir fuseConnection *fuse.Conn - swarmApi *api.Api + swarmApi *api.API lock *sync.RWMutex + serveClose chan struct{} } -func NewMountInfo(mhash, mpoint string, sapi *api.Api) *MountInfo { +func NewMountInfo(mhash, mpoint string, sapi *api.API) *MountInfo { + log.Debug("swarmfs NewMountInfo", "hash", mhash, "mount point", mpoint) newMountInfo := &MountInfo{ MountPoint: mpoint, StartManifest: mhash, @@ -68,50 +71,57 @@ func NewMountInfo(mhash, mpoint string, sapi *api.Api) *MountInfo { fuseConnection: nil, swarmApi: sapi, lock: &sync.RWMutex{}, + serveClose: make(chan struct{}), } return newMountInfo } -func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { - +func (swarmfs *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { + log.Info("swarmfs", "mounting hash", mhash, "mount point", mountpoint) if mountpoint == "" { return nil, errEmptyMountPoint } + if !strings.HasPrefix(mountpoint, "/") { + return nil, errNoRelativeMountPoint + } cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint)) if err != nil { return nil, err } + log.Trace("swarmfs mount", "cleanedMountPoint", cleanedMountPoint) - self.swarmFsLock.Lock() - defer self.swarmFsLock.Unlock() + swarmfs.swarmFsLock.Lock() + defer swarmfs.swarmFsLock.Unlock() - noOfActiveMounts := len(self.activeMounts) + noOfActiveMounts := len(swarmfs.activeMounts) + log.Debug("swarmfs mount", "# active mounts", noOfActiveMounts) if noOfActiveMounts >= maxFuseMounts { return nil, errMaxMountCount } - if _, ok := self.activeMounts[cleanedMountPoint]; ok { + if _, ok := swarmfs.activeMounts[cleanedMountPoint]; ok { return nil, errAlreadyMounted } - log.Info(fmt.Sprintf("Attempting to mount %s ", cleanedMountPoint)) - _, manifestEntryMap, err := self.swarmApi.BuildDirectoryTree(mhash, true) + log.Trace("swarmfs mount: getting manifest tree") + _, manifestEntryMap, err := swarmfs.swarmApi.BuildDirectoryTree(mhash, true) if err != nil { return nil, err } - mi := NewMountInfo(mhash, cleanedMountPoint, self.swarmApi) + log.Trace("swarmfs mount: building mount info") + mi := NewMountInfo(mhash, cleanedMountPoint, swarmfs.swarmApi) dirTree := map[string]*SwarmDir{} rootDir := NewSwarmDir("/", mi) - dirTree["/"] = rootDir + log.Trace("swarmfs mount", "rootDir", rootDir) mi.rootDir = rootDir + log.Trace("swarmfs mount: traversing manifest map") for suffix, entry := range manifestEntryMap { - key := common.Hex2Bytes(entry.Hash) + addr := common.Hex2Bytes(entry.Hash) fullpath := "/" + suffix basepath := filepath.Dir(fullpath) - parentDir := rootDir dirUntilNow := "" paths := strings.Split(basepath, "/") @@ -128,105 +138,143 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { } else { parentDir = dirTree[dirUntilNow] } - } } thisFile := NewSwarmFile(basepath, filepath.Base(fullpath), mi) - thisFile.key = key + thisFile.addr = addr parentDir.files = append(parentDir.files, thisFile) } fconn, err := fuse.Mount(cleanedMountPoint, fuse.FSName("swarmfs"), fuse.VolumeName(mhash)) if isFUSEUnsupportedError(err) { - log.Warn("Fuse not installed", "mountpoint", cleanedMountPoint, "err", err) + log.Error("swarmfs error - FUSE not installed", "mountpoint", cleanedMountPoint, "err", err) return nil, err } else if err != nil { fuse.Unmount(cleanedMountPoint) - log.Warn("Error mounting swarm manifest", "mountpoint", cleanedMountPoint, "err", err) + log.Error("swarmfs error mounting swarm manifest", "mountpoint", cleanedMountPoint, "err", err) return nil, err } mi.fuseConnection = fconn serverr := make(chan error, 1) go func() { - log.Info(fmt.Sprintf("Serving %s at %s", mhash, cleanedMountPoint)) + log.Info("swarmfs", "serving hash", mhash, "at", cleanedMountPoint) filesys := &SwarmRoot{root: rootDir} + //start serving the actual file system; see note below if err := fs.Serve(fconn, filesys); err != nil { - log.Warn(fmt.Sprintf("Could not Serve SwarmFileSystem error: %v", err)) + log.Warn("swarmfs could not serve the requested hash", "error", err) serverr <- err } - + mi.serveClose <- struct{}{} }() + /* + IMPORTANT NOTE: the fs.Serve function is blocking; + Serve builds up the actual fuse file system by calling the + Attr functions on each SwarmFile, creating the file inodes; + specifically calling the swarm's LazySectionReader.Size() to set the file size. + + This can take some time, and it appears that if we access the fuse file system + too early, we can bring the tests to deadlock. The assumption so far is that + at this point, the fuse driver didn't finish to initialize the file system. + + Accessing files too early not only deadlocks the tests, but locks the access + of the fuse file completely, resulting in blocked resources at OS system level. + Even a simple `ls /tmp/testDir/testMountDir` could deadlock in a shell. + + Workaround so far is to wait some time to give the OS enough time to initialize + the fuse file system. During tests, this seemed to address the issue. + + HOWEVER IT SHOULD BE NOTED THAT THIS MAY ONLY BE AN EFFECT, + AND THE DEADLOCK CAUSED BY SOMETHING ELSE BLOCKING ACCESS DUE TO SOME RACE CONDITION + (caused in the bazil.org library and/or the SwarmRoot, SwarmDir and SwarmFile implementations) + */ + time.Sleep(2 * time.Second) + + timer := time.NewTimer(mountTimeout) + defer timer.Stop() // Check if the mount process has an error to report. select { - case <-time.After(mountTimeout): - fuse.Unmount(cleanedMountPoint) + case <-timer.C: + log.Warn("swarmfs timed out mounting over FUSE", "mountpoint", cleanedMountPoint, "err", err) + err := fuse.Unmount(cleanedMountPoint) + if err != nil { + return nil, err + } return nil, errMountTimeout - case err := <-serverr: - fuse.Unmount(cleanedMountPoint) - log.Warn("Error serving swarm FUSE FS", "mountpoint", cleanedMountPoint, "err", err) + log.Warn("swarmfs error serving over FUSE", "mountpoint", cleanedMountPoint, "err", err) + err = fuse.Unmount(cleanedMountPoint) return nil, err case <-fconn.Ready: - log.Info("Now serving swarm FUSE FS", "manifest", mhash, "mountpoint", cleanedMountPoint) + //this signals that the actual mount point from the fuse.Mount call is ready; + //it does not signal though that the file system from fs.Serve is actually fully built up + if err := fconn.MountError; err != nil { + log.Error("Mounting error from fuse driver: ", "err", err) + return nil, err + } + log.Info("swarmfs now served over FUSE", "manifest", mhash, "mountpoint", cleanedMountPoint) } - self.activeMounts[cleanedMountPoint] = mi + timer.Stop() + swarmfs.activeMounts[cleanedMountPoint] = mi return mi, nil } -func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) { - - self.swarmFsLock.Lock() - defer self.swarmFsLock.Unlock() +func (swarmfs *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) { + swarmfs.swarmFsLock.Lock() + defer swarmfs.swarmFsLock.Unlock() cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint)) if err != nil { return nil, err } - mountInfo := self.activeMounts[cleanedMountPoint] + mountInfo := swarmfs.activeMounts[cleanedMountPoint] if mountInfo == nil || mountInfo.MountPoint != cleanedMountPoint { - return nil, fmt.Errorf("%s is not mounted", cleanedMountPoint) + return nil, fmt.Errorf("swarmfs %s is not mounted", cleanedMountPoint) } err = fuse.Unmount(cleanedMountPoint) if err != nil { err1 := externalUnmount(cleanedMountPoint) if err1 != nil { - errStr := fmt.Sprintf("UnMount error: %v", err) + errStr := fmt.Sprintf("swarmfs unmount error: %v", err) log.Warn(errStr) return nil, err1 } } - mountInfo.fuseConnection.Close() - delete(self.activeMounts, cleanedMountPoint) + err = mountInfo.fuseConnection.Close() + if err != nil { + return nil, err + } + delete(swarmfs.activeMounts, cleanedMountPoint) - succString := fmt.Sprintf("UnMounting %v succeeded", cleanedMountPoint) + <-mountInfo.serveClose + + succString := fmt.Sprintf("swarmfs unmounting %v succeeded", cleanedMountPoint) log.Info(succString) return mountInfo, nil } -func (self *SwarmFS) Listmounts() []*MountInfo { - self.swarmFsLock.RLock() - defer self.swarmFsLock.RUnlock() - - rows := make([]*MountInfo, 0, len(self.activeMounts)) - for _, mi := range self.activeMounts { +func (swarmfs *SwarmFS) Listmounts() []*MountInfo { + swarmfs.swarmFsLock.RLock() + defer swarmfs.swarmFsLock.RUnlock() + rows := make([]*MountInfo, 0, len(swarmfs.activeMounts)) + for _, mi := range swarmfs.activeMounts { rows = append(rows, mi) } return rows } -func (self *SwarmFS) Stop() bool { - for mp := range self.activeMounts { - mountInfo := self.activeMounts[mp] - self.Unmount(mountInfo.MountPoint) +func (swarmfs *SwarmFS) Stop() bool { + for mp := range swarmfs.activeMounts { + mountInfo := swarmfs.activeMounts[mp] + swarmfs.Unmount(mountInfo.MountPoint) } return true } diff --git a/swarm/fuse/swarmfs_util.go b/swarm/fuse/swarmfs_util.go index 169b67487..9bbb0f6ac 100644 --- a/swarm/fuse/swarmfs_util.go +++ b/swarm/fuse/swarmfs_util.go @@ -24,7 +24,7 @@ import ( "os/exec" "runtime" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/log" ) func externalUnmount(mountPoint string) error { @@ -38,11 +38,11 @@ func externalUnmount(mountPoint string) error { // Try FUSE-specific commands if umount didn't work. switch runtime.GOOS { case "darwin": - return exec.CommandContext(ctx, "diskutil", "umount", "force", mountPoint).Run() + return exec.CommandContext(ctx, "diskutil", "umount", mountPoint).Run() case "linux": return exec.CommandContext(ctx, "fusermount", "-u", mountPoint).Run() default: - return fmt.Errorf("unmount: unimplemented") + return fmt.Errorf("swarmfs unmount: unimplemented") } } @@ -54,14 +54,14 @@ func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { sf.lock.Lock() defer sf.lock.Unlock() - sf.key = fkey + sf.addr = fkey sf.fileSize = int64(size) sf.mountInfo.lock.Lock() defer sf.mountInfo.lock.Unlock() sf.mountInfo.LatestManifest = mhash - log.Info("Added new file:", "fname", sf.name, "New Manifest hash", mhash) + log.Info("swarmfs added new file:", "fname", sf.name, "new Manifest hash", mhash) return nil } @@ -75,7 +75,7 @@ func removeFileFromSwarm(sf *SwarmFile) error { defer sf.mountInfo.lock.Unlock() sf.mountInfo.LatestManifest = mkey - log.Info("Removed file:", "fname", sf.name, "New Manifest hash", mkey) + log.Info("swarmfs removed file:", "fname", sf.name, "new Manifest hash", mkey) return nil } @@ -102,20 +102,20 @@ func removeDirectoryFromSwarm(sd *SwarmDir) error { } func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error { - fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.key, offset, length, true) + fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.addr, offset, length, true) if err != nil { return err } sf.lock.Lock() defer sf.lock.Unlock() - sf.key = fkey + sf.addr = fkey sf.fileSize = sf.fileSize + int64(len(content)) sf.mountInfo.lock.Lock() defer sf.mountInfo.lock.Unlock() sf.mountInfo.LatestManifest = mhash - log.Info("Appended file:", "fname", sf.name, "New Manifest hash", mhash) + log.Info("swarmfs appended file:", "fname", sf.name, "new Manifest hash", mhash) return nil } |