diff options
author | Yondon Fu <yondon.fu@gmail.com> | 2017-12-19 06:17:41 +0800 |
---|---|---|
committer | Yondon Fu <yondon.fu@gmail.com> | 2017-12-19 06:17:41 +0800 |
commit | 3857cdc267e3192697f561df0a0f827f65dfb6b5 (patch) | |
tree | 401c52c4972a68229ea283a394a0b0a5f3cfdc8e /swarm | |
parent | a5330fe0c569b75cb8a524f60f7e8dc06498262b (diff) | |
parent | fe070ab5c32702033489f1b9d1655ea1b894c29e (diff) | |
download | dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.tar dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.tar.gz dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.tar.bz2 dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.tar.lz dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.tar.xz dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.tar.zst dexon-3857cdc267e3192697f561df0a0f827f65dfb6b5.zip |
Merge branch 'master' into abi-offset-fixed-arrays
Diffstat (limited to 'swarm')
-rw-r--r-- | swarm/api/client/client_test.go | 38 | ||||
-rw-r--r-- | swarm/api/config.go | 131 | ||||
-rw-r--r-- | swarm/api/config_test.go | 116 | ||||
-rw-r--r-- | swarm/api/http/error.go | 2 | ||||
-rw-r--r-- | swarm/api/http/error_test.go | 2 | ||||
-rw-r--r-- | swarm/api/manifest.go | 16 | ||||
-rw-r--r-- | swarm/api/manifest_test.go | 78 | ||||
-rw-r--r-- | swarm/fuse/fuse_file.go | 11 | ||||
-rw-r--r-- | swarm/fuse/swarmfs_test.go | 10 | ||||
-rw-r--r-- | swarm/network/depo.go | 2 | ||||
-rw-r--r-- | swarm/network/hive.go | 12 | ||||
-rw-r--r-- | swarm/network/kademlia/address.go | 2 | ||||
-rw-r--r-- | swarm/network/kademlia/kaddb.go | 8 | ||||
-rw-r--r-- | swarm/network/kademlia/kademlia.go | 2 | ||||
-rw-r--r-- | swarm/network/kademlia/kademlia_test.go | 10 | ||||
-rw-r--r-- | swarm/network/protocol.go | 2 | ||||
-rw-r--r-- | swarm/network/syncer.go | 11 | ||||
-rw-r--r-- | swarm/services/swap/swap.go | 28 | ||||
-rw-r--r-- | swarm/storage/chunker.go | 19 | ||||
-rw-r--r-- | swarm/storage/chunker_test.go | 4 | ||||
-rw-r--r-- | swarm/storage/memstore.go | 2 | ||||
-rw-r--r-- | swarm/storage/netstore.go | 10 | ||||
-rw-r--r-- | swarm/storage/pyramid.go | 28 | ||||
-rw-r--r-- | swarm/swarm.go | 12 |
24 files changed, 260 insertions, 296 deletions
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index edf385dd0..c1d144e37 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -244,25 +244,25 @@ func TestClientFileList(t *testing.T) { } tests := map[string][]string{ - "": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"}, - "file": []string{"file1.txt", "file2.txt"}, - "file1": []string{"file1.txt"}, - "file2.txt": []string{"file2.txt"}, - "file12": []string{}, - "dir": []string{"dir1/", "dir2/"}, - "dir1": []string{"dir1/"}, - "dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"}, - "dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"}, - "dir1/file3.txt": []string{"dir1/file3.txt"}, - "dir1/file34": []string{}, - "dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, - "dir2/file": []string{"dir2/file5.txt"}, - "dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"}, - "dir2/dir3/": []string{"dir2/dir3/file6.txt"}, - "dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, - "dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, - "dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"}, - "dir2/dir4/file78": []string{}, + "": {"dir1/", "dir2/", "file1.txt", "file2.txt"}, + "file": {"file1.txt", "file2.txt"}, + "file1": {"file1.txt"}, + "file2.txt": {"file2.txt"}, + "file12": {}, + "dir": {"dir1/", "dir2/"}, + "dir1": {"dir1/"}, + "dir1/": {"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file": {"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file3.txt": {"dir1/file3.txt"}, + "dir1/file34": {}, + "dir2/": {"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, + "dir2/file": {"dir2/file5.txt"}, + "dir2/dir": {"dir2/dir3/", "dir2/dir4/"}, + "dir2/dir3/": {"dir2/dir3/file6.txt"}, + "dir2/dir4/": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file7.txt": {"dir2/dir4/file7.txt"}, + "dir2/dir4/file78": {}, } for prefix, expected := range tests { actual := ls(prefix) diff --git a/swarm/api/config.go b/swarm/api/config.go index d8d25b1c8..140c938ae 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -18,15 +18,15 @@ package api import ( "crypto/ecdsa" - "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/services/swap" "github.com/ethereum/go-ethereum/swarm/storage" @@ -46,101 +46,68 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams *network.SyncParams - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - EnsRoot common.Address - NetworkId uint64 + Contract common.Address + EnsRoot common.Address + EnsApi string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + SwapApi string + Cors string + BzzAccount string + BootNodes string } -// config is agnostic to where private key is coming from -// so managing accounts is outside swarm and left to wrappers -func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, networkId uint64) (self *Config, err error) { - address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address - dirpath := filepath.Join(path, "bzz-"+common.Bytes2Hex(address.Bytes())) - err = os.MkdirAll(dirpath, os.ModePerm) - if err != nil { - return - } - confpath := filepath.Join(dirpath, "config.json") - var data []byte - pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) - pubkeyhex := common.ToHex(pubkey) - keyhex := crypto.Keccak256Hash(pubkey).Hex() +//create a default config with all parameters to set to defaults +func NewDefaultConfig() (self *Config) { self = &Config{ - SyncParams: network.NewSyncParams(dirpath), - HiveParams: network.NewHiveParams(dirpath), + StoreParams: storage.NewDefaultStoreParams(), ChunkerParams: storage.NewChunkerParams(), - StoreParams: storage.NewStoreParams(dirpath), + HiveParams: network.NewDefaultHiveParams(), + SyncParams: network.NewDefaultSyncParams(), + Swap: swap.NewDefaultSwapParams(), ListenAddr: DefaultHTTPListenAddr, Port: DefaultHTTPPort, - Path: dirpath, - Swap: swap.DefaultSwapParams(contract, prvKey), - PublicKey: pubkeyhex, - BzzKey: keyhex, + Path: node.DefaultDataDir(), + EnsApi: node.DefaultIPCEndpoint("geth"), EnsRoot: ens.TestNetAddress, - NetworkId: networkId, - } - data, err = ioutil.ReadFile(confpath) - - // if not set in function param, then set default for swarm network, will be overwritten by config file if present - if networkId == 0 { - self.NetworkId = network.NetworkId + NetworkId: network.NetworkId, + SwapEnabled: false, + SyncEnabled: true, + SwapApi: "", + BootNodes: "", } - if err != nil { - if !os.IsNotExist(err) { - return - } + return +} - // file does not exist - // write out config file - err = self.Save() - if err != nil { - err = fmt.Errorf("error writing config: %v", err) - } - return - } +//some config params need to be initialized after the complete +//config building phase is completed (e.g. due to overriding flags) +func (self *Config) Init(prvKey *ecdsa.PrivateKey) { - // file exists, deserialise - err = json.Unmarshal(data, self) + address := crypto.PubkeyToAddress(prvKey.PublicKey) + self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes())) + err := os.MkdirAll(self.Path, os.ModePerm) if err != nil { - return nil, fmt.Errorf("unable to parse config: %v", err) - } - // check public key - if pubkeyhex != self.PublicKey { - return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey) - } - if keyhex != self.BzzKey { - return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) - } - - // if set in function param, replace id set from config file - if networkId != 0 { - self.NetworkId = networkId + log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err)) + return } - self.Swap.SetKey(prvKey) - - if (self.EnsRoot == common.Address{}) { - self.EnsRoot = ens.TestNetAddress - } + pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) + pubkeyhex := common.ToHex(pubkey) + keyhex := crypto.Keccak256Hash(pubkey).Hex() - return -} + self.PublicKey = pubkeyhex + self.BzzKey = keyhex -func (self *Config) Save() error { - data, err := json.MarshalIndent(self, "", " ") - if err != nil { - return err - } - err = os.MkdirAll(self.Path, os.ModePerm) - if err != nil { - return err - } - confpath := filepath.Join(self.Path, "config.json") - return ioutil.WriteFile(confpath, data, os.ModePerm) + self.Swap.Init(self.Contract, prvKey) + self.SyncParams.Init(self.Path) + self.HiveParams.Init(self.Path) + self.StoreParams.Init(self.Path) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 85d056270..5636b6daf 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -17,109 +17,53 @@ package api import ( - "io/ioutil" - "os" - "path/filepath" - "strings" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) -var ( - hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" - defaultConfig = `{ - "ChunkDbPath": "` + filepath.Join("TMPDIR", "chunks") + `", - "DbCapacity": 5000000, - "CacheCapacity": 5000, - "Radius": 0, - "Branches": 128, - "Hash": "SHA3", - "CallInterval": 3000000000, - "KadDbPath": "` + filepath.Join("TMPDIR", "bzz-peers.json") + `", - "MaxProx": 8, - "ProxBinSize": 2, - "BucketSize": 4, - "PurgeInterval": 151200000000000, - "InitialRetryInterval": 42000000, - "MaxIdleInterval": 42000000000, - "ConnRetryExp": 2, - "Swap": { - "BuyAt": 20000000000, - "SellAt": 20000000000, - "PayAt": 100, - "DropAt": 10000, - "AutoCashInterval": 300000000000, - "AutoCashThreshold": 50000000000000, - "AutoDepositInterval": 300000000000, - "AutoDepositThreshold": 50000000000000, - "AutoDepositBuffer": 100000000000000, - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "Contract": "0x0000000000000000000000000000000000000000", - "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e" - }, - "RequestDbPath": "` + filepath.Join("TMPDIR", "requests") + `", - "RequestDbBatchSize": 512, - "KeyBufferSize": 1024, - "SyncBatchSize": 128, - "SyncBufferSize": 128, - "SyncCacheSize": 1024, - "SyncPriorities": [ - 2, - 1, - 1, - 0, - 0 - ], - "SyncModes": [ - true, - true, - true, - true, - false - ], - "Path": "TMPDIR", - "ListenAddr": "127.0.0.1", - "Port": "8500", - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", - "EnsRoot": "0x112234455c3a32fd11230c42e7bccd4a84e02010", - "NetworkId": 323 -}` -) +func TestConfig(t *testing.T) { -func TestConfigWriteRead(t *testing.T) { - tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) + var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" prvkey, err := crypto.HexToECDSA(hexprvkey) if err != nil { t.Fatalf("failed to load private key: %v", err) } - orig, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + + one := NewDefaultConfig() + two := NewDefaultConfig() + + if equal := reflect.DeepEqual(one, two); !equal { + t.Fatal("Two default configs are not equal") } - data, err := ioutil.ReadFile(filepath.Join(orig.Path, "config.json")) - if err != nil { - t.Fatalf("default config file cannot be read: %v", err) + + one.Init(prvkey) + + //the init function should set the following fields + if one.BzzKey == "" { + t.Fatal("Expected BzzKey to be set") } - exp := strings.Replace(defaultConfig, "TMPDIR", orig.Path, -1) - exp = strings.Replace(exp, "\\", "\\\\", -1) - if string(data) != exp { - t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data)) + if one.PublicKey == "" { + t.Fatal("Expected PublicKey to be set") } - conf, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + //the Init function should append subdirs to the given path + if one.Swap.PayProfile.Beneficiary == (common.Address{}) { + t.Fatal("Failed to correctly initialize SwapParams") } - if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() { - t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex()) + + if one.SyncParams.RequestDbPath == one.Path { + t.Fatal("Failed to correctly initialize SyncParams") } + if one.HiveParams.KadDbPath == one.Path { + t.Fatal("Failed to correctly initialize HiveParams") + } + + if one.StoreParams.ChunkDbPath == one.Path { + t.Fatal("Failed to correctly initialize StoreParams") + } } diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index b4d46b3c4..dbd97182f 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -72,7 +72,7 @@ func initErrHandling() { //ShowMultipeChoices is used when a user requests a resource in a manifest which results //in ambiguous results. It returns a HTML page with clickable links of each of the entry //in the manifest which fits the request URI ambiguity. -//For example, if the user requests bzz:/<hash>/read and that manifest containes entries +//For example, if the user requests bzz:/<hash>/read and that manifest contains entries //"readme.md" and "readinglist.txt", a HTML page is returned with this two links. //This only applies if the manifest has no default entry func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) { diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go index 465fbf4ca..ed52bafbd 100644 --- a/swarm/api/http/error_test.go +++ b/swarm/api/http/error_test.go @@ -132,7 +132,7 @@ func TestJsonResponse(t *testing.T) { } if !isJSON(string(respbody)) { - t.Fatalf("Expected repsonse to be JSON, received invalid JSON: %s", string(respbody)) + t.Fatalf("Expected response to be JSON, received invalid JSON: %s", string(respbody)) } } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index d3eced198..685a300fc 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -238,7 +238,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { return } - b := byte(entry.Path[0]) + b := entry.Path[0] oldentry := self.entries[b] if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) { self.entries[b] = entry @@ -294,7 +294,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { return } - b := byte(path[0]) + b := path[0] entry := self.entries[b] if entry == nil { return @@ -425,7 +425,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man } //see if first char is in manifest entries - b := byte(path[0]) + b := path[0] entry = self.entries[b] if entry == nil { return self.entries[256], 0 @@ -436,6 +436,16 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man if len(path) <= epl { if entry.Path[:len(path)] == path { if entry.ContentType == ManifestType { + err := self.loadSubTrie(entry, quitC) + if err == nil && entry.subtrie != nil { + subentries := entry.subtrie.entries + for i := 0; i < len(subentries); i++ { + sub := subentries[i] + if sub != nil && sub.Path == "" { + return sub, len(path) + } + } + } entry.Status = http.StatusMultipleChoices } pos = len(path) diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index f048627c5..7098ca16f 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "strings" "testing" @@ -39,17 +40,17 @@ func manifest(paths ...string) (manifestReader storage.LazySectionReader) { } } -func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie { +func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie { quitC := make(chan bool) trie, err := readManifest(manifest(paths...), nil, nil, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } - checkEntry(t, path, match, trie) + checkEntry(t, path, match, multiple, trie) return trie } -func checkEntry(t *testing.T, path, match string, trie *manifestTrie) { +func checkEntry(t *testing.T, path, match string, multiple bool, trie *manifestTrie) { entry, fullpath := trie.getEntry(path) if match == "-" && entry != nil { t.Errorf("expected no match for '%s', got '%s'", path, fullpath) @@ -60,32 +61,55 @@ func checkEntry(t *testing.T, path, match string, trie *manifestTrie) { } else if fullpath != match { t.Errorf("incorrect entry retrieved for '%s'. expected path '%v', got '%s'", path, match, fullpath) } + + if multiple && entry.Status != http.StatusMultipleChoices { + t.Errorf("Expected %d Multiple Choices Status for path %s, match %s, got %d", http.StatusMultipleChoices, path, match, entry.Status) + } else if !multiple && entry != nil && entry.Status == http.StatusMultipleChoices { + t.Errorf("Were not expecting %d Multiple Choices Status for path %s, match %s, but got it", http.StatusMultipleChoices, path, match) + } } func TestGetEntry(t *testing.T) { // file system manifest always contains regularized paths - testGetEntry(t, "a", "a", "a") - testGetEntry(t, "b", "-", "a") - testGetEntry(t, "/a//", "a", "a") + testGetEntry(t, "a", "a", false, "a") + testGetEntry(t, "b", "-", false, "a") + testGetEntry(t, "/a//", "a", false, "a") // fallback - testGetEntry(t, "/a", "", "") - testGetEntry(t, "/a/b", "a/b", "a/b") + testGetEntry(t, "/a", "", false, "") + testGetEntry(t, "/a/b", "a/b", false, "a/b") // longest/deepest math - testGetEntry(t, "read", "read", "readme.md", "readit.md") - testGetEntry(t, "rf", "-", "readme.md", "readit.md") - testGetEntry(t, "readme", "readme", "readme.md") - testGetEntry(t, "readme", "-", "readit.md") - testGetEntry(t, "readme.md", "readme.md", "readme.md") - testGetEntry(t, "readme.md", "-", "readit.md") - testGetEntry(t, "readmeAmd", "-", "readit.md") - testGetEntry(t, "readme.mdffff", "-", "readme.md") - testGetEntry(t, "ab", "ab", "ab/cefg", "ab/cedh", "ab/kkkkkk") - testGetEntry(t, "ab/ce", "ab/ce", "ab/cefg", "ab/cedh", "ab/ceuuuuuuuuuu") - testGetEntry(t, "abc", "abc", "abcd", "abczzzzef", "abc/def", "abc/e/g") - testGetEntry(t, "a/b", "a/b", "a", "a/bc", "a/ba", "a/b/c") - testGetEntry(t, "a/b", "a/b", "a", "a/b", "a/bb", "a/b/c") - testGetEntry(t, "//a//b//", "a/b", "a", "a/b", "a/bb", "a/b/c") + testGetEntry(t, "read", "read", true, "readme.md", "readit.md") + testGetEntry(t, "rf", "-", false, "readme.md", "readit.md") + testGetEntry(t, "readme", "readme", false, "readme.md") + testGetEntry(t, "readme", "-", false, "readit.md") + testGetEntry(t, "readme.md", "readme.md", false, "readme.md") + testGetEntry(t, "readme.md", "-", false, "readit.md") + testGetEntry(t, "readmeAmd", "-", false, "readit.md") + testGetEntry(t, "readme.mdffff", "-", false, "readme.md") + testGetEntry(t, "ab", "ab", true, "ab/cefg", "ab/cedh", "ab/kkkkkk") + testGetEntry(t, "ab/ce", "ab/ce", true, "ab/cefg", "ab/cedh", "ab/ceuuuuuuuuuu") + testGetEntry(t, "abc", "abc", true, "abcd", "abczzzzef", "abc/def", "abc/e/g") + testGetEntry(t, "a/b", "a/b", true, "a", "a/bc", "a/ba", "a/b/c") + testGetEntry(t, "a/b", "a/b", false, "a", "a/b", "a/bb", "a/b/c") + testGetEntry(t, "//a//b//", "a/b", false, "a", "a/b", "a/bb", "a/b/c") } + +func TestExactMatch(t *testing.T) { + quitC := make(chan bool) + mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") + trie, err := readManifest(mf, nil, nil, quitC) + if err != nil { + t.Errorf("unexpected error making manifest: %v", err) + } + entry, _ := trie.getEntry("shouldBeExactMatch.css") + if entry.Path != "" { + t.Errorf("Expected entry to match %s, got: %s", "shouldBeExactMatch.css", entry.Path) + } + if entry.Status == http.StatusMultipleChoices { + t.Errorf("Got status %d, which is unexepcted", http.StatusMultipleChoices) + } +} + func TestDeleteEntry(t *testing.T) { } @@ -108,15 +132,15 @@ func TestAddFileWithManifestPath(t *testing.T) { if err != nil { t.Fatal(err) } - checkEntry(t, "ab", "ab", trie) - checkEntry(t, "ac", "ac", trie) + checkEntry(t, "ab", "ab", false, trie) + checkEntry(t, "ac", "ac", false, trie) // now add path "a" and check we can still get "ab" and "ac" entry := &manifestTrieEntry{} entry.Path = "a" entry.Hash = "a" trie.addEntry(entry, nil) - checkEntry(t, "ab", "ab", trie) - checkEntry(t, "ac", "ac", trie) - checkEntry(t, "a", "a", trie) + checkEntry(t, "ab", "ab", false, trie) + checkEntry(t, "ac", "ac", false, trie) + checkEntry(t, "a", "a", false, trie) } diff --git a/swarm/fuse/fuse_file.go b/swarm/fuse/fuse_file.go index 32c85d02f..c94a0773f 100644 --- a/swarm/fuse/fuse_file.go +++ b/swarm/fuse/fuse_file.go @@ -19,15 +19,16 @@ package fuse import ( + "errors" + "io" + "os" + "sync" + "bazil.org/fuse" "bazil.org/fuse/fs" - "errors" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" "golang.org/x/net/context" - "io" - "os" - "sync" ) const ( @@ -87,7 +88,7 @@ func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { if err != nil { log.Warn("Couldnt get size of file %s : %v", file.path, err) } - file.fileSize = int64(size) + file.fileSize = size } a.Size = uint64(file.fileSize) return nil diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 93f1d4c2f..42af36345 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -95,7 +95,7 @@ func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash str } // Test listMounts - if found == false { + if !found { t.Fatalf("Error getting mounts information for %v: %v", mountDir, err) } @@ -185,10 +185,8 @@ func isDirEmpty(name string) bool { defer f.Close() _, err = f.Readdirnames(1) - if err == io.EOF { - return true - } - return false + + return err == io.EOF } type testAPI struct { @@ -388,7 +386,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T) { d.Read(contents) finfo := files["1.txt"] - if bytes.Compare(finfo.contents[:6024][5000:], contents) != 0 { + if !bytes.Equal(finfo.contents[:6024][5000:], contents) { t.Fatalf("File seek contents mismatch") } d.Close() diff --git a/swarm/network/depo.go b/swarm/network/depo.go index 8695bf5d9..17540d2f9 100644 --- a/swarm/network/depo.go +++ b/swarm/network/depo.go @@ -55,7 +55,7 @@ func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error var err error for _, req := range unsynced { // skip keys that are found, - chunk, err = self.localStore.Get(storage.Key(req.Key[:])) + chunk, err = self.localStore.Get(req.Key[:]) if err != nil || chunk.SData == nil { missing = append(missing, req) } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index d37b7e400..2504a4610 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -70,19 +70,25 @@ type HiveParams struct { *kademlia.KadParams } -func NewHiveParams(path string) *HiveParams { - kad := kademlia.NewKadParams() +//create default params +func NewDefaultHiveParams() *HiveParams { + kad := kademlia.NewDefaultKadParams() // kad.BucketSize = bucketSize // kad.MaxProx = maxProx // kad.ProxBinSize = proxBinSize return &HiveParams{ CallInterval: callInterval, - KadDbPath: filepath.Join(path, "bzz-peers.json"), KadParams: kad, } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *HiveParams) Init(path string) { + self.KadDbPath = filepath.Join(path, "bzz-peers.json") +} + func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive { kad := kademlia.New(kademlia.Address(addr), params.KadParams) return &Hive{ diff --git a/swarm/network/kademlia/address.go b/swarm/network/kademlia/address.go index 16c5ce532..4c38a846f 100644 --- a/swarm/network/kademlia/address.go +++ b/swarm/network/kademlia/address.go @@ -67,7 +67,7 @@ func proximity(one, other Address) (ret int) { for i := 0; i < len(one); i++ { oxo := one[i] ^ other[i] for j := 0; j < 8; j++ { - if (uint8(oxo)>>uint8(7-j))&0x01 != 0 { + if (oxo>>uint8(7-j))&0x01 != 0 { return i*8 + j } } diff --git a/swarm/network/kademlia/kaddb.go b/swarm/network/kademlia/kaddb.go index 3e4b261fd..b37ced5ba 100644 --- a/swarm/network/kademlia/kaddb.go +++ b/swarm/network/kademlia/kaddb.go @@ -211,12 +211,12 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe } // if node is scheduled to connect - if time.Time(node.After).After(time.Now()) { + if node.After.After(time.Now()) { log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) skipped. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After)) continue ROW } - delta = time.Since(time.Time(node.Seen)) + delta = time.Since(node.Seen) if delta < self.initialRetryInterval { delta = self.initialRetryInterval } @@ -230,7 +230,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) ready to be tried. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After)) // scheduling next check - interval = time.Duration(delta * time.Duration(self.connRetryExp)) + interval = delta * time.Duration(self.connRetryExp) after = time.Now().Add(interval) log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval)) @@ -330,7 +330,7 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro } } n++ - if (node.After == time.Time{}) { + if node.After.IsZero() { node.After = time.Now() } self.index[node.Addr] = node diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index bf976a3e1..0abc42a19 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -52,7 +52,7 @@ type KadParams struct { ConnRetryExp int } -func NewKadParams() *KadParams { +func NewDefaultKadParams() *KadParams { return &KadParams{ MaxProx: maxProx, ProxBinSize: proxBinSize, diff --git a/swarm/network/kademlia/kademlia_test.go b/swarm/network/kademlia/kademlia_test.go index 417ccecae..88858908a 100644 --- a/swarm/network/kademlia/kademlia_test.go +++ b/swarm/network/kademlia/kademlia_test.go @@ -63,7 +63,7 @@ func TestOn(t *testing.T) { if !ok1 || !ok2 { t.Errorf("oops") } - kad := New(addr, NewKadParams()) + kad := New(addr, NewDefaultKadParams()) err := kad.On(&testNode{addr: other}, nil) _ = err } @@ -72,7 +72,7 @@ func TestBootstrap(t *testing.T) { test := func(test *bootstrapTest) bool { // for any node kad.le, Target and N - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = test.MaxProx params.BucketSize = test.BucketSize params.ProxBinSize = test.BucketSize @@ -127,7 +127,7 @@ func TestFindClosest(t *testing.T) { test := func(test *FindClosestTest) bool { // for any node kad.le, Target and N - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(test.Self, params) var err error @@ -198,7 +198,7 @@ var ( func TestProxAdjust(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) self := gen(Address{}, r).(Address) - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(self, params) @@ -232,7 +232,7 @@ func TestSaveLoad(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) addresses := gen([]Address{}, r).([]Address) self := RandomAddress() - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(self, params) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 2f880df57..a418c1dbb 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -309,7 +309,7 @@ func (self *bzz) handleStatus() (err error) { Version: uint64(Version), ID: "honey", Addr: self.selfAddr(), - NetworkId: uint64(self.NetworkId), + NetworkId: self.NetworkId, Swap: &bzzswap.SwapProfile{ Profile: self.swapParams.Profile, PayProfile: self.swapParams.PayProfile, diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index a3814e423..6d729fcb9 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -131,9 +131,8 @@ type SyncParams struct { } // constructor with default values -func NewSyncParams(bzzdir string) *SyncParams { +func NewDefaultSyncParams() *SyncParams { return &SyncParams{ - RequestDbPath: filepath.Join(bzzdir, "requests"), RequestDbBatchSize: requestDbBatchSize, KeyBufferSize: keyBufferSize, SyncBufferSize: syncBufferSize, @@ -144,6 +143,12 @@ func NewSyncParams(bzzdir string) *SyncParams { } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *SyncParams) Init(path string) { + self.RequestDbPath = filepath.Join(path, "requests") +} + // syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding type syncer struct { *SyncParams // sync parameters @@ -378,7 +383,7 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} { } select { // blocking until history channel is read from - case history <- storage.Key(key): + case history <- key: n++ log.Trace(fmt.Sprintf("syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n)) state.Latest = key diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index 093892e8d..1f9b22b90 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -80,17 +80,10 @@ type PayProfile struct { lock sync.RWMutex } -func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapParams { - pubkey := &prvkey.PublicKey +//create params with default values +func NewDefaultSwapParams() *SwapParams { return &SwapParams{ - PayProfile: &PayProfile{ - PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)), - Contract: contract, - Beneficiary: crypto.PubkeyToAddress(*pubkey), - privateKey: prvkey, - publicKey: pubkey, - owner: crypto.PubkeyToAddress(*pubkey), - }, + PayProfile: &PayProfile{}, Params: &swap.Params{ Profile: &swap.Profile{ BuyAt: buyAt, @@ -109,6 +102,21 @@ func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapP } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) { + pubkey := &prvkey.PublicKey + + self.PayProfile = &PayProfile{ + PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)), + Contract: contract, + Beneficiary: crypto.PubkeyToAddress(*pubkey), + privateKey: prvkey, + publicKey: pubkey, + owner: crypto.PubkeyToAddress(*pubkey), + } +} + // swap constructor, parameters // * global chequebook, assume deployed service and // * the balance is at buffer. diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 0454828b9..98cd6e75e 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -50,7 +50,6 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} The underlying hash function is configurable */ - /* Tree chunker is a concrete implementation of data chunking. This chunker works in a simple way, it builds a tree out of the document so that each node either represents a chunk of real data or a chunk of data representing an branching non-leaf node of the tree. In particular each such non-leaf chunk will represent is a concatenation of the hash of its respective children. This scheme simultaneously guarantees data integrity as well as self addressing. Abstract nodes are transparent since their represented size component is strictly greater than their maximum data size, since they encode a subtree. @@ -61,17 +60,17 @@ The hashing itself does use extra copies and allocation though, since it does ne var ( errAppendOppNotSuported = errors.New("Append operation not supported") - errOperationTimedOut = errors.New("operation timed out") + errOperationTimedOut = errors.New("operation timed out") ) type TreeChunker struct { branches int64 hashFunc SwarmHasher // calculated - hashSize int64 // self.hashFunc.New().Size() - chunkSize int64 // hashSize* branches - workerCount int64 // the number of worker routines used - workerLock sync.RWMutex // lock for the worker count + hashSize int64 // self.hashFunc.New().Size() + chunkSize int64 // hashSize* branches + workerCount int64 // the number of worker routines used + workerLock sync.RWMutex // lock for the worker count } func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) { @@ -124,7 +123,6 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s panic("chunker must be initialised") } - jobC := make(chan *hashJob, 2*ChunkProcessors) wg := &sync.WaitGroup{} errC := make(chan error) @@ -164,7 +162,6 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s close(errC) }() - defer close(quitC) select { case err := <-errC: @@ -172,7 +169,7 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s return nil, err } case <-time.NewTimer(splitTimeout).C: - return nil,errOperationTimedOut + return nil, errOperationTimedOut } return key, nil @@ -208,9 +205,9 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade } // dept > 0 // intermediate chunk containing child nodes hashes - branchCnt := int64((size + treeSize - 1) / treeSize) + branchCnt := (size + treeSize - 1) / treeSize - var chunk []byte = make([]byte, branchCnt*self.hashSize+8) + var chunk = make([]byte, branchCnt*self.hashSize+8) var pos, i int64 binary.LittleEndian.PutUint64(chunk[0:8], uint64(size)) diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index b41d7dd33..6178a7bb1 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -77,7 +77,7 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c key, err = chunker.Split(data, size, chunkC, swg, nil) if err != nil && expectedError == nil { - err = errors.New(fmt.Sprintf("Split error: %v", err)) + err = fmt.Errorf("Split error: %v", err) } if chunkC != nil { @@ -123,7 +123,7 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, key, err = chunker.Append(rootKey, data, chunkC, swg, nil) if err != nil && expectedError == nil { - err = errors.New(fmt.Sprintf("Append error: %v", err)) + err = fmt.Errorf("Append error: %v", err) } if chunkC != nil { diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 155dd0088..3cb25ac62 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -78,7 +78,7 @@ type memTree struct { func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) { node = new(memTree) node.bits = b - node.width = 1 << uint(b) + node.width = 1 << b node.subtree = make([]*memTree, node.width) node.access = make([]uint64, node.width-1) node.parent = parent diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 7b0612edc..5d4f17deb 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -57,15 +57,21 @@ type StoreParams struct { Radius int } -func NewStoreParams(path string) (self *StoreParams) { +//create params with default values +func NewDefaultStoreParams() (self *StoreParams) { return &StoreParams{ - ChunkDbPath: filepath.Join(path, "chunks"), DbCapacity: defaultDbCapacity, CacheCapacity: defaultCacheCapacity, Radius: defaultRadius, } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *StoreParams) Init(path string) { + self.ChunkDbPath = filepath.Join(path, "chunks") +} + // netstore contructor, takes path argument that is used to initialise dbStore, // the persistent (disk) storage component of LocalStore // the second argument is the hive, the connection/logistics manager for the node diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index e3be2a987..f2e85cb5b 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -27,7 +27,7 @@ import ( /* The main idea of a pyramid chunker is to process the input data without knowing the entire size apriori. For this to be achieved, the chunker tree is built from the ground up until the data is exhausted. - This opens up new aveneus such as easy append and other sort of modifications to the tree therby avoiding + This opens up new aveneus such as easy append and other sort of modifications to the tree thereby avoiding duplication of data chunks. @@ -123,7 +123,7 @@ type PyramidChunker struct { hashSize int64 branches int64 workerCount int64 - workerLock sync.RWMutex + workerLock sync.RWMutex } func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) { @@ -327,7 +327,7 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC // Add the root chunk entry branchCount := int64(len(chunk.SData)-8) / self.hashSize newEntry := &TreeEntry{ - level: int(depth - 1), + level: depth - 1, branchCount: branchCount, subtreeSize: uint64(chunk.Size), chunk: chunk.SData, @@ -352,7 +352,7 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC } bewBranchCount := int64(len(newChunk.SData)-8) / self.hashSize newEntry := &TreeEntry{ - level: int(lvl - 1), + level: lvl - 1, branchCount: bewBranchCount, subtreeSize: uint64(newChunk.Size), chunk: newChunk.SData, @@ -391,7 +391,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt parent := NewTreeEntry(self) var unFinishedChunk *Chunk - if isAppend == true && len(chunkLevel[0]) != 0 { + if isAppend && len(chunkLevel[0]) != 0 { lastIndex := len(chunkLevel[0]) - 1 ent := chunkLevel[0][lastIndex] @@ -451,7 +451,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt } } - // Data ended in chunk boundry.. just signal to start bulding tree + // Data ended in chunk boundary.. just signal to start bulding tree if n == 0 { self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey) break @@ -512,7 +512,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, } } - if compress == false && last == false { + if !compress && !last { return } @@ -522,7 +522,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, for lvl := int64(ent.level); lvl < endLvl; lvl++ { lvlCount := int64(len(chunkLevel[lvl])) - if lvlCount == 1 && last == true { + if lvlCount == 1 && last { copy(rootKey, chunkLevel[lvl][0].key) return } @@ -540,7 +540,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, nextLvlCount = int64(len(chunkLevel[lvl+1]) - 1) tempEntry = chunkLevel[lvl+1][nextLvlCount] } - if isAppend == true && tempEntry != nil && tempEntry.updatePending == true { + if isAppend && tempEntry != nil && tempEntry.updatePending { updateEntry := &TreeEntry{ level: int(lvl + 1), branchCount: 0, @@ -585,9 +585,9 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, } - if isAppend == false { + if !isAppend { chunkWG.Wait() - if compress == true { + if compress { chunkLevel[lvl] = nil } } @@ -599,7 +599,7 @@ func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *Tre if ent != nil { // wait for data chunks to get over before processing the tree chunk - if last == true { + if last { chunkWG.Wait() } @@ -612,7 +612,7 @@ func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *Tre } // Update or append based on weather it is a new entry or being reused - if ent.updatePending == true { + if ent.updatePending { chunkWG.Wait() chunkLevel[ent.level][ent.index] = ent } else { @@ -634,4 +634,4 @@ func (self *PyramidChunker) enqueueDataChunk(chunkData []byte, size uint64, pare return pkey -}
\ No newline at end of file +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 9db15325a..3be3660b5 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -220,7 +220,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { // stops all component services. func (self *Swarm) Stop() error { self.dpa.Stop() - self.hive.Stop() + err := self.hive.Stop() if ch := self.config.Swap.Chequebook(); ch != nil { ch.Stop() ch.Save() @@ -230,7 +230,7 @@ func (self *Swarm) Stop() error { self.lstore.DbStore.Close() } self.sfs.Stop() - return self.config.Save() + return err } // implements the node.Service interface @@ -301,7 +301,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error { return err } log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex())) - self.config.Save() self.hive.DropAll() return nil } @@ -314,10 +313,9 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { return } - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) - if err != nil { - return - } + config := api.NewDefaultConfig() + config.Path = datadir + config.Init(prvKey) config.Port = port dpa, err := storage.NewLocalDPA(datadir) |