From 03ec3fed2b7363700571a83ef41de3ee9208d94e Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Wed, 18 Oct 2017 08:33:50 -0500 Subject: swarm/api: bug fix exact match for manifest --- swarm/api/manifest.go | 10 ++++++ swarm/api/manifest_test.go | 78 ++++++++++++++++++++++++++++++---------------- 2 files changed, 61 insertions(+), 27 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index d3eced198..5cd84e28e 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -436,6 +436,16 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man if len(path) <= epl { if entry.Path[:len(path)] == path { if entry.ContentType == ManifestType { + err := self.loadSubTrie(entry, quitC) + if err == nil && entry.subtrie != nil { + subentries := entry.subtrie.entries + for i := 0; i < len(subentries); i++ { + sub := subentries[i] + if sub != nil && sub.Path == "" { + return sub, len(path) + } + } + } entry.Status = http.StatusMultipleChoices } pos = len(path) diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index f048627c5..7098ca16f 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "strings" "testing" @@ -39,17 +40,17 @@ func manifest(paths ...string) (manifestReader storage.LazySectionReader) { } } -func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie { +func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie { quitC := make(chan bool) trie, err := readManifest(manifest(paths...), nil, nil, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } - checkEntry(t, path, match, trie) + checkEntry(t, path, match, multiple, trie) return trie } -func checkEntry(t *testing.T, path, match string, trie *manifestTrie) { +func checkEntry(t *testing.T, path, match string, multiple bool, trie *manifestTrie) { entry, fullpath := trie.getEntry(path) if match == "-" && entry != nil { t.Errorf("expected no match for '%s', got '%s'", path, fullpath) @@ -60,32 +61,55 @@ func checkEntry(t *testing.T, path, match string, trie *manifestTrie) { } else if fullpath != match { t.Errorf("incorrect entry retrieved for '%s'. expected path '%v', got '%s'", path, match, fullpath) } + + if multiple && entry.Status != http.StatusMultipleChoices { + t.Errorf("Expected %d Multiple Choices Status for path %s, match %s, got %d", http.StatusMultipleChoices, path, match, entry.Status) + } else if !multiple && entry != nil && entry.Status == http.StatusMultipleChoices { + t.Errorf("Were not expecting %d Multiple Choices Status for path %s, match %s, but got it", http.StatusMultipleChoices, path, match) + } } func TestGetEntry(t *testing.T) { // file system manifest always contains regularized paths - testGetEntry(t, "a", "a", "a") - testGetEntry(t, "b", "-", "a") - testGetEntry(t, "/a//", "a", "a") + testGetEntry(t, "a", "a", false, "a") + testGetEntry(t, "b", "-", false, "a") + testGetEntry(t, "/a//", "a", false, "a") // fallback - testGetEntry(t, "/a", "", "") - testGetEntry(t, "/a/b", "a/b", "a/b") + testGetEntry(t, "/a", "", false, "") + testGetEntry(t, "/a/b", "a/b", false, "a/b") // longest/deepest math - testGetEntry(t, "read", "read", "readme.md", "readit.md") - testGetEntry(t, "rf", "-", "readme.md", "readit.md") - testGetEntry(t, "readme", "readme", "readme.md") - testGetEntry(t, "readme", "-", "readit.md") - testGetEntry(t, "readme.md", "readme.md", "readme.md") - testGetEntry(t, "readme.md", "-", "readit.md") - testGetEntry(t, "readmeAmd", "-", "readit.md") - testGetEntry(t, "readme.mdffff", "-", "readme.md") - testGetEntry(t, "ab", "ab", "ab/cefg", "ab/cedh", "ab/kkkkkk") - testGetEntry(t, "ab/ce", "ab/ce", "ab/cefg", "ab/cedh", "ab/ceuuuuuuuuuu") - testGetEntry(t, "abc", "abc", "abcd", "abczzzzef", "abc/def", "abc/e/g") - testGetEntry(t, "a/b", "a/b", "a", "a/bc", "a/ba", "a/b/c") - testGetEntry(t, "a/b", "a/b", "a", "a/b", "a/bb", "a/b/c") - testGetEntry(t, "//a//b//", "a/b", "a", "a/b", "a/bb", "a/b/c") + testGetEntry(t, "read", "read", true, "readme.md", "readit.md") + testGetEntry(t, "rf", "-", false, "readme.md", "readit.md") + testGetEntry(t, "readme", "readme", false, "readme.md") + testGetEntry(t, "readme", "-", false, "readit.md") + testGetEntry(t, "readme.md", "readme.md", false, "readme.md") + testGetEntry(t, "readme.md", "-", false, "readit.md") + testGetEntry(t, "readmeAmd", "-", false, "readit.md") + testGetEntry(t, "readme.mdffff", "-", false, "readme.md") + testGetEntry(t, "ab", "ab", true, "ab/cefg", "ab/cedh", "ab/kkkkkk") + testGetEntry(t, "ab/ce", "ab/ce", true, "ab/cefg", "ab/cedh", "ab/ceuuuuuuuuuu") + testGetEntry(t, "abc", "abc", true, "abcd", "abczzzzef", "abc/def", "abc/e/g") + testGetEntry(t, "a/b", "a/b", true, "a", "a/bc", "a/ba", "a/b/c") + testGetEntry(t, "a/b", "a/b", false, "a", "a/b", "a/bb", "a/b/c") + testGetEntry(t, "//a//b//", "a/b", false, "a", "a/b", "a/bb", "a/b/c") } + +func TestExactMatch(t *testing.T) { + quitC := make(chan bool) + mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") + trie, err := readManifest(mf, nil, nil, quitC) + if err != nil { + t.Errorf("unexpected error making manifest: %v", err) + } + entry, _ := trie.getEntry("shouldBeExactMatch.css") + if entry.Path != "" { + t.Errorf("Expected entry to match %s, got: %s", "shouldBeExactMatch.css", entry.Path) + } + if entry.Status == http.StatusMultipleChoices { + t.Errorf("Got status %d, which is unexepcted", http.StatusMultipleChoices) + } +} + func TestDeleteEntry(t *testing.T) { } @@ -108,15 +132,15 @@ func TestAddFileWithManifestPath(t *testing.T) { if err != nil { t.Fatal(err) } - checkEntry(t, "ab", "ab", trie) - checkEntry(t, "ac", "ac", trie) + checkEntry(t, "ab", "ab", false, trie) + checkEntry(t, "ac", "ac", false, trie) // now add path "a" and check we can still get "ab" and "ac" entry := &manifestTrieEntry{} entry.Path = "a" entry.Hash = "a" trie.addEntry(entry, nil) - checkEntry(t, "ab", "ab", trie) - checkEntry(t, "ac", "ac", trie) - checkEntry(t, "a", "a", trie) + checkEntry(t, "ab", "ab", false, trie) + checkEntry(t, "ac", "ac", false, trie) + checkEntry(t, "a", "a", false, trie) } -- cgit v1.2.3 From 32516c768ec09e2a71cab5983d2c8b8ae5d92fc7 Mon Sep 17 00:00:00 2001 From: holisticode Date: Mon, 11 Dec 2017 16:56:06 -0500 Subject: cmd/swarm: add config file (#15548) This commit adds a TOML configuration option to swarm. It reuses the TOML configuration structure used in geth with swarm customized items. The commit: * Adds a "dumpconfig" command to the swarm executable which allows printing the (default) configuration to stdout, which then can be redirected to a file in order to customize it. * Adds a "--config " option to the swarm executable which will allow to load a configuration file in TOML format from the specified location in order to initialize the Swarm node The override priorities are like follows: environment variables override command line arguments override config file override default config. --- swarm/api/config.go | 131 ++++++++++++++++++----------------------------- swarm/api/config_test.go | 116 +++++++++++------------------------------ 2 files changed, 79 insertions(+), 168 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/config.go b/swarm/api/config.go index d8d25b1c8..140c938ae 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -18,15 +18,15 @@ package api import ( "crypto/ecdsa" - "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/services/swap" "github.com/ethereum/go-ethereum/swarm/storage" @@ -46,101 +46,68 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams *network.SyncParams - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - EnsRoot common.Address - NetworkId uint64 + Contract common.Address + EnsRoot common.Address + EnsApi string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + SwapApi string + Cors string + BzzAccount string + BootNodes string } -// config is agnostic to where private key is coming from -// so managing accounts is outside swarm and left to wrappers -func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, networkId uint64) (self *Config, err error) { - address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address - dirpath := filepath.Join(path, "bzz-"+common.Bytes2Hex(address.Bytes())) - err = os.MkdirAll(dirpath, os.ModePerm) - if err != nil { - return - } - confpath := filepath.Join(dirpath, "config.json") - var data []byte - pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) - pubkeyhex := common.ToHex(pubkey) - keyhex := crypto.Keccak256Hash(pubkey).Hex() +//create a default config with all parameters to set to defaults +func NewDefaultConfig() (self *Config) { self = &Config{ - SyncParams: network.NewSyncParams(dirpath), - HiveParams: network.NewHiveParams(dirpath), + StoreParams: storage.NewDefaultStoreParams(), ChunkerParams: storage.NewChunkerParams(), - StoreParams: storage.NewStoreParams(dirpath), + HiveParams: network.NewDefaultHiveParams(), + SyncParams: network.NewDefaultSyncParams(), + Swap: swap.NewDefaultSwapParams(), ListenAddr: DefaultHTTPListenAddr, Port: DefaultHTTPPort, - Path: dirpath, - Swap: swap.DefaultSwapParams(contract, prvKey), - PublicKey: pubkeyhex, - BzzKey: keyhex, + Path: node.DefaultDataDir(), + EnsApi: node.DefaultIPCEndpoint("geth"), EnsRoot: ens.TestNetAddress, - NetworkId: networkId, - } - data, err = ioutil.ReadFile(confpath) - - // if not set in function param, then set default for swarm network, will be overwritten by config file if present - if networkId == 0 { - self.NetworkId = network.NetworkId + NetworkId: network.NetworkId, + SwapEnabled: false, + SyncEnabled: true, + SwapApi: "", + BootNodes: "", } - if err != nil { - if !os.IsNotExist(err) { - return - } + return +} - // file does not exist - // write out config file - err = self.Save() - if err != nil { - err = fmt.Errorf("error writing config: %v", err) - } - return - } +//some config params need to be initialized after the complete +//config building phase is completed (e.g. due to overriding flags) +func (self *Config) Init(prvKey *ecdsa.PrivateKey) { - // file exists, deserialise - err = json.Unmarshal(data, self) + address := crypto.PubkeyToAddress(prvKey.PublicKey) + self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes())) + err := os.MkdirAll(self.Path, os.ModePerm) if err != nil { - return nil, fmt.Errorf("unable to parse config: %v", err) - } - // check public key - if pubkeyhex != self.PublicKey { - return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey) - } - if keyhex != self.BzzKey { - return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) - } - - // if set in function param, replace id set from config file - if networkId != 0 { - self.NetworkId = networkId + log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err)) + return } - self.Swap.SetKey(prvKey) - - if (self.EnsRoot == common.Address{}) { - self.EnsRoot = ens.TestNetAddress - } + pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) + pubkeyhex := common.ToHex(pubkey) + keyhex := crypto.Keccak256Hash(pubkey).Hex() - return -} + self.PublicKey = pubkeyhex + self.BzzKey = keyhex -func (self *Config) Save() error { - data, err := json.MarshalIndent(self, "", " ") - if err != nil { - return err - } - err = os.MkdirAll(self.Path, os.ModePerm) - if err != nil { - return err - } - confpath := filepath.Join(self.Path, "config.json") - return ioutil.WriteFile(confpath, data, os.ModePerm) + self.Swap.Init(self.Contract, prvKey) + self.SyncParams.Init(self.Path) + self.HiveParams.Init(self.Path) + self.StoreParams.Init(self.Path) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 85d056270..2e03a610e 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -17,109 +17,53 @@ package api import ( - "io/ioutil" - "os" - "path/filepath" - "strings" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) -var ( - hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" - defaultConfig = `{ - "ChunkDbPath": "` + filepath.Join("TMPDIR", "chunks") + `", - "DbCapacity": 5000000, - "CacheCapacity": 5000, - "Radius": 0, - "Branches": 128, - "Hash": "SHA3", - "CallInterval": 3000000000, - "KadDbPath": "` + filepath.Join("TMPDIR", "bzz-peers.json") + `", - "MaxProx": 8, - "ProxBinSize": 2, - "BucketSize": 4, - "PurgeInterval": 151200000000000, - "InitialRetryInterval": 42000000, - "MaxIdleInterval": 42000000000, - "ConnRetryExp": 2, - "Swap": { - "BuyAt": 20000000000, - "SellAt": 20000000000, - "PayAt": 100, - "DropAt": 10000, - "AutoCashInterval": 300000000000, - "AutoCashThreshold": 50000000000000, - "AutoDepositInterval": 300000000000, - "AutoDepositThreshold": 50000000000000, - "AutoDepositBuffer": 100000000000000, - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "Contract": "0x0000000000000000000000000000000000000000", - "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e" - }, - "RequestDbPath": "` + filepath.Join("TMPDIR", "requests") + `", - "RequestDbBatchSize": 512, - "KeyBufferSize": 1024, - "SyncBatchSize": 128, - "SyncBufferSize": 128, - "SyncCacheSize": 1024, - "SyncPriorities": [ - 2, - 1, - 1, - 0, - 0 - ], - "SyncModes": [ - true, - true, - true, - true, - false - ], - "Path": "TMPDIR", - "ListenAddr": "127.0.0.1", - "Port": "8500", - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", - "EnsRoot": "0x112234455c3a32fd11230c42e7bccd4a84e02010", - "NetworkId": 323 -}` -) +func TestConfig(t *testing.T) { -func TestConfigWriteRead(t *testing.T) { - tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) + var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" prvkey, err := crypto.HexToECDSA(hexprvkey) if err != nil { t.Fatalf("failed to load private key: %v", err) } - orig, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + + one := NewDefaultConfig() + two := NewDefaultConfig() + + if equal := reflect.DeepEqual(one, two); equal == false { + t.Fatal("Two default configs are not equal") } - data, err := ioutil.ReadFile(filepath.Join(orig.Path, "config.json")) - if err != nil { - t.Fatalf("default config file cannot be read: %v", err) + + one.Init(prvkey) + + //the init function should set the following fields + if one.BzzKey == "" { + t.Fatal("Expected BzzKey to be set") } - exp := strings.Replace(defaultConfig, "TMPDIR", orig.Path, -1) - exp = strings.Replace(exp, "\\", "\\\\", -1) - if string(data) != exp { - t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data)) + if one.PublicKey == "" { + t.Fatal("Expected PublicKey to be set") } - conf, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + //the Init function should append subdirs to the given path + if one.Swap.PayProfile.Beneficiary == (common.Address{}) { + t.Fatal("Failed to correctly initialize SwapParams") } - if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() { - t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex()) + + if one.SyncParams.RequestDbPath == one.Path { + t.Fatal("Failed to correctly initialize SyncParams") } + if one.HiveParams.KadDbPath == one.Path { + t.Fatal("Failed to correctly initialize HiveParams") + } + + if one.StoreParams.ChunkDbPath == one.Path { + t.Fatal("Failed to correctly initialize StoreParams") + } } -- cgit v1.2.3