diff options
122 files changed, 1810 insertions, 1257 deletions
diff --git a/.travis.yml b/.travis.yml index 7f7168854..5f3ff9d16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,6 @@ matrix: include: - os: linux dist: trusty - go: 1.5.4 - env: - - GO15VENDOREXPERIMENT=1 - - os: linux - dist: trusty - go: 1.6.2 - - os: linux - dist: trusty go: 1.7.5 # These are the latest Go versions, only run go vet and misspell on these @@ -158,7 +150,7 @@ matrix: # Build the iOS framework and upload it to CocoaPods and Azure - gem uninstall cocoapods -a - - gem install cocoapods --pre + - gem install cocoapods - mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak - sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb @@ -40,6 +40,15 @@ test: all clean: rm -fr build/_workspace/pkg/ $(GOBIN)/* +# The devtools target installs tools required for 'go generate'. +# You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'. + +devtools: + go get -u golang.org/x/tools/cmd/stringer + go get -u github.com/jteeuwen/go-bindata/go-bindata + go get -u github.com/fjl/gencodec + go install ./cmd/abigen + # Cross Compilation Targets (xgo) geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios @@ -16,7 +16,7 @@ For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki. -Building geth requires both a Go and a C compiler. +Building geth requires both a Go (version 1.7 or later) and a C compiler. You can install them using your favourite package manager. Once the dependencies are installed, run diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index 4509e222d..25b61928e 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -17,13 +17,13 @@ package bind import ( + "context" "errors" "math/big" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "golang.org/x/net/context" ) var ( diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 2e6796de6..2f5719c78 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -17,6 +17,7 @@ package backends import ( + "context" "errors" "fmt" "math/big" @@ -34,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/pow" - "golang.org/x/net/context" ) // Default chain configuration which sets homestead phase at block 0 (i.e. no frontier) diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index 1f11827dd..b40bd65e8 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -17,6 +17,7 @@ package bind import ( + "context" "errors" "fmt" "math/big" @@ -26,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "golang.org/x/net/context" ) // SignerFn is a signer function callback when a contract requires a method to @@ -35,7 +35,8 @@ type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Tra // CallOpts is the collection of options to fine tune a contract call request. type CallOpts struct { - Pending bool // Whether to operate on the pending state or the last known one + Pending bool // Whether to operate on the pending state or the last known one + From common.Address // Optional the sender address, otherwise the first account is used Context context.Context // Network context to support cancellation and timeouts (nil = no timeout) } @@ -108,7 +109,7 @@ func (c *BoundContract) Call(opts *CallOpts, result interface{}, method string, return err } var ( - msg = ethereum.CallMsg{To: &c.address, Data: input} + msg = ethereum.CallMsg{From: opts.From, To: &c.address, Data: input} ctx = ensureContext(opts.Context) code []byte output []byte diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index eb46bc081..8ac4aa44e 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -408,6 +408,45 @@ var bindTests = []struct { } `, }, + // Test that constant functions can be called from an (optional) specified address + { + `CallFrom`, + ` + contract CallFrom { + function callFrom() constant returns(address) { + return msg.sender; + } + } + `, `6060604052346000575b6086806100176000396000f300606060405263ffffffff60e060020a60003504166349f8e98281146022575b6000565b34600057602c6055565b6040805173ffffffffffffffffffffffffffffffffffffffff9092168252519081900360200190f35b335b905600a165627a7a72305820aef6b7685c0fa24ba6027e4870404a57df701473fe4107741805c19f5138417c0029`, + `[{"constant":true,"inputs":[],"name":"callFrom","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"}]`, + ` + // Generate a new random account and a funded simulator + key, _ := crypto.GenerateKey() + auth := bind.NewKeyedTransactor(key) + sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)}) + + // Deploy a sender tester contract and execute a structured call on it + _, _, callfrom, err := DeployCallFrom(auth, sim) + if err != nil { + t.Fatalf("Failed to deploy sender contract: %v", err) + } + sim.Commit() + + if res, err := callfrom.CallFrom(nil); err != nil { + t.Errorf("Failed to call constant function: %v", err) + } else if res != (common.Address{}) { + t.Errorf("Invalid address returned, want: %x, got: %x", (common.Address{}), res) + } + + for _, addr := range []common.Address{common.Address{}, common.Address{1}, common.Address{2}} { + if res, err := callfrom.CallFrom(&bind.CallOpts{From: addr}); err != nil { + t.Fatalf("Failed to call constant function: %v", err) + } else if res != addr { + t.Fatalf("Invalid address returned, want: %x, got: %x", addr, res) + } + } + `, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/accounts/abi/bind/util.go b/accounts/abi/bind/util.go index 8348f6980..d129993ca 100644 --- a/accounts/abi/bind/util.go +++ b/accounts/abi/bind/util.go @@ -17,13 +17,13 @@ package bind import ( + "context" "fmt" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "golang.org/x/net/context" ) // WaitMined waits for tx to be mined on the blockchain. diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index f31dbfc29..b37a69cfc 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -17,6 +17,7 @@ package bind_test import ( + "context" "math/big" "testing" "time" @@ -27,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "golang.org/x/net/context" ) var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") diff --git a/accounts/usbwallet/ledger_wallet.go b/accounts/usbwallet/ledger_wallet.go index c3d0f0ac8..698e85f48 100644 --- a/accounts/usbwallet/ledger_wallet.go +++ b/accounts/usbwallet/ledger_wallet.go @@ -21,6 +21,7 @@ package usbwallet import ( + "context" "encoding/binary" "encoding/hex" "errors" @@ -38,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/karalabe/hid" - "golang.org/x/net/context" ) // Maximum time between wallet health checks to detect USB unplugs. diff --git a/build/_vendor/src/golang.org/x/net/LICENSE b/build/_vendor/src/golang.org/x/net/LICENSE deleted file mode 100644 index 6a66aea5e..000000000 --- a/build/_vendor/src/golang.org/x/net/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/build/ci-notes.md b/build/ci-notes.md index 92e7c54d0..cd2ba8bb2 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -1,18 +1,3 @@ -# Vendored Dependencies - -Dependencies are almost all vendored in at the standard Go `/vendor` path. This allows -people to build go-ethereum using the standard toolchain without any particular package -manager. It also plays nicely with `go get`, not requiring external code to be relied on. - -The one single dependent package missing from `vendor` is `golang.org/x/net/context`. As -this is a package exposed via public library APIs, it must not be vendored as dependent -code woulnd't be able to instantiate. - -To allow reproducible builds of go-ethereum nonetheless that don't need network access -during build time to fetch `golang.org/x/net/context`, a version was copied into our repo -at the very specific `/build/_vendor` path, which is added automatically by all CI build -scripts and the makefile too. - # Debian Packaging Tagged releases and develop branch commits are available as installable Debian packages diff --git a/build/ci.go b/build/ci.go index 914ce9eae..cb9c7a335 100644 --- a/build/ci.go +++ b/build/ci.go @@ -162,9 +162,9 @@ func doInstall(cmdline []string) { // Check Go version. People regularly open issues about compilation // failure with outdated Go. This should save them the trouble. - if runtime.Version() < "go1.4" && !strings.HasPrefix(runtime.Version(), "devel") { + if runtime.Version() < "go1.7" && !strings.HasPrefix(runtime.Version(), "devel") { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.4 and cannot") + log.Println("go-ethereum requires at least Go version 1.7 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } @@ -215,20 +215,9 @@ func doInstall(cmdline []string) { } func buildFlags(env build.Environment) (flags []string) { - if os.Getenv("GO_OPENCL") != "" { - flags = append(flags, "-tags", "opencl") - } - - // Since Go 1.5, the separator char for link time assignments - // is '=' and using ' ' prints a warning. However, Go < 1.5 does - // not support using '='. - sep := " " - if runtime.Version() > "go1.5" || strings.Contains(runtime.Version(), "devel") { - sep = "=" - } // Set gitCommit constant via link-time assignment. if env.Commit != "" { - flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+env.Commit) + flags = append(flags, "-ldflags", "-X main.gitCommit="+env.Commit) } return flags } @@ -249,10 +238,7 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd { cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...) } } - cmd.Env = []string{ - "GO15VENDOREXPERIMENT=1", - "GOPATH=" + build.GOPATH(), - } + cmd.Env = []string{"GOPATH=" + build.GOPATH()} if arch == "" || arch == runtime.GOARCH { cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) } else { diff --git a/build/env.sh b/build/env.sh index af560305b..3914555d1 100755 --- a/build/env.sh +++ b/build/env.sh @@ -20,8 +20,7 @@ fi # Set up the environment to use the workspace. GOPATH="$workspace" -GO15VENDOREXPERIMENT=1 -export GOPATH GO15VENDOREXPERIMENT +export GOPATH # Run the command inside the workspace. cd "$ethdir/go-ethereum" diff --git a/contracts/chequebook/cheque.go b/contracts/chequebook/cheque.go index 945e56e86..7e0f7eafc 100644 --- a/contracts/chequebook/cheque.go +++ b/contracts/chequebook/cheque.go @@ -26,6 +26,7 @@ package chequebook import ( "bytes" + "context" "crypto/ecdsa" "encoding/json" "fmt" @@ -43,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/services/swap/swap" - "golang.org/x/net/context" ) // TODO(zelig): watch peer solvency and notify of bouncing cheques diff --git a/contracts/release/release.go b/contracts/release/release.go index 613e62aa9..28a35381d 100644 --- a/contracts/release/release.go +++ b/contracts/release/release.go @@ -20,6 +20,7 @@ package release //go:generate abigen --sol ./contract.sol --pkg release --out ./contract.go import ( + "context" "fmt" "strings" "time" @@ -33,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // Interval to check for new releases @@ -116,47 +116,49 @@ func (r *ReleaseService) checker() { for { select { - // If the time arrived, check for a new release case <-timer.C: // Rechedule the timer before continuing timer.Reset(releaseRecheckInterval) - - // Retrieve the current version, and handle missing contracts gracefully - ctx, _ := context.WithTimeout(context.Background(), time.Second*5) - opts := &bind.CallOpts{Context: ctx} - version, err := r.oracle.CurrentVersion(opts) - if err != nil { - if err == bind.ErrNoCode { - log.Debug("Release oracle not found", "contract", r.config.Oracle) - continue - } - log.Error("Failed to retrieve current release", "err", err) - continue - } - // Version was successfully retrieved, notify if newer than ours - if version.Major > r.config.Major || - (version.Major == r.config.Major && version.Minor > r.config.Minor) || - (version.Major == r.config.Major && version.Minor == r.config.Minor && version.Patch > r.config.Patch) { - - warning := fmt.Sprintf("Client v%d.%d.%d-%x seems older than the latest upstream release v%d.%d.%d-%x", - r.config.Major, r.config.Minor, r.config.Patch, r.config.Commit[:4], version.Major, version.Minor, version.Patch, version.Commit[:4]) - howtofix := fmt.Sprintf("Please check https://github.com/ethereum/go-ethereum/releases for new releases") - separator := strings.Repeat("-", len(warning)) - - log.Warn(separator) - log.Warn(warning) - log.Warn(howtofix) - log.Warn(separator) - } else { - log.Debug("Client seems up to date with upstream", - "local", fmt.Sprintf("v%d.%d.%d-%x", r.config.Major, r.config.Minor, r.config.Patch, r.config.Commit[:4]), - "upstream", fmt.Sprintf("v%d.%d.%d-%x", version.Major, version.Minor, version.Patch, version.Commit[:4])) - } - - // If termination was requested, return + r.checkVersion() case errc := <-r.quit: errc <- nil return } } } + +func (r *ReleaseService) checkVersion() { + // Retrieve the current version, and handle missing contracts gracefully + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + opts := &bind.CallOpts{Context: ctx} + defer cancel() + + version, err := r.oracle.CurrentVersion(opts) + if err != nil { + if err == bind.ErrNoCode { + log.Debug("Release oracle not found", "contract", r.config.Oracle) + } else { + log.Error("Failed to retrieve current release", "err", err) + } + return + } + // Version was successfully retrieved, notify if newer than ours + if version.Major > r.config.Major || + (version.Major == r.config.Major && version.Minor > r.config.Minor) || + (version.Major == r.config.Major && version.Minor == r.config.Minor && version.Patch > r.config.Patch) { + + warning := fmt.Sprintf("Client v%d.%d.%d-%x seems older than the latest upstream release v%d.%d.%d-%x", + r.config.Major, r.config.Minor, r.config.Patch, r.config.Commit[:4], version.Major, version.Minor, version.Patch, version.Commit[:4]) + howtofix := fmt.Sprintf("Please check https://github.com/ethereum/go-ethereum/releases for new releases") + separator := strings.Repeat("-", len(warning)) + + log.Warn(separator) + log.Warn(warning) + log.Warn(howtofix) + log.Warn(separator) + } else { + log.Debug("Client seems up to date with upstream", + "local", fmt.Sprintf("v%d.%d.%d-%x", r.config.Major, r.config.Minor, r.config.Patch, r.config.Commit[:4]), + "upstream", fmt.Sprintf("v%d.%d.%d-%x", version.Major, version.Minor, version.Patch, version.Commit[:4])) + } +} diff --git a/core/blockchain.go b/core/blockchain.go index 765a4b318..a57832df0 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -182,16 +182,25 @@ func (self *BlockChain) loadLastState() error { head := GetHeadBlockHash(self.chainDb) if head == (common.Hash{}) { // Corrupt or empty database, init from scratch - self.Reset() - } else { - if block := self.GetBlockByHash(head); block != nil { - // Block found, set as the current head - self.currentBlock = block - } else { - // Corrupt or empty database, init from scratch - self.Reset() - } + log.Warn("Empty database, resetting chain") + return self.Reset() + } + // Make sure the entire head block is available + currentBlock := self.GetBlockByHash(head) + if currentBlock == nil { + // Corrupt or empty database, init from scratch + log.Warn("Head block missing, resetting chain", "hash", head) + return self.Reset() + } + // Make sure the state associated with the block is available + if _, err := state.New(currentBlock.Root(), self.chainDb); err != nil { + // Dangling block without a state associated, init from scratch + log.Warn("Head state missing, resetting chain", "number", currentBlock.Number(), "hash", currentBlock.Hash()) + return self.Reset() } + // Everything seems to be fine, set as the head block + self.currentBlock = currentBlock + // Restore the last known head header currentHeader := self.currentBlock.Header() if head := GetHeadHeaderHash(self.chainDb); head != (common.Hash{}) { @@ -200,6 +209,7 @@ func (self *BlockChain) loadLastState() error { } } self.hc.SetCurrentHeader(currentHeader) + // Restore the last known head fast block self.currentFastBlock = self.currentBlock if head := GetHeadFastBlockHash(self.chainDb); head != (common.Hash{}) { @@ -233,14 +243,18 @@ func (self *BlockChain) loadLastState() error { // above the new head will be deleted and the new one set. In the case of blocks // though, the head may be further rewound if block bodies are missing (non-archive // nodes after a fast sync). -func (bc *BlockChain) SetHead(head uint64) { +func (bc *BlockChain) SetHead(head uint64) error { + log.Warn("Rewinding blockchain", "target", head) + bc.mu.Lock() defer bc.mu.Unlock() + // Rewind the header chain, deleting all block bodies until then delFn := func(hash common.Hash, num uint64) { DeleteBody(bc.chainDb, hash, num) } bc.hc.SetHead(head, delFn) + currentHeader := bc.hc.CurrentHeader() // Clear out any stale content from the caches bc.bodyCache.Purge() @@ -248,29 +262,34 @@ func (bc *BlockChain) SetHead(head uint64) { bc.blockCache.Purge() bc.futureBlocks.Purge() - // Update all computed fields to the new head - currentHeader := bc.hc.CurrentHeader() + // Rewind the block chain, ensuring we don't end up with a stateless head block if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() { bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } + if bc.currentBlock != nil { + if _, err := state.New(bc.currentBlock.Root(), bc.chainDb); err != nil { + // Rewound state missing, rolled back to before pivot, reset to genesis + bc.currentBlock = nil + } + } + // Rewind the fast block in a simpleton way to the target head if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() { bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } - + // If either blocks reached nil, reset to the genesis state if bc.currentBlock == nil { bc.currentBlock = bc.genesisBlock } if bc.currentFastBlock == nil { bc.currentFastBlock = bc.genesisBlock } - if err := WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()); err != nil { log.Crit("Failed to reset head full block", "err", err) } if err := WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()); err != nil { log.Crit("Failed to reset head fast block", "err", err) } - bc.loadLastState() + return bc.loadLastState() } // FastSyncCommitHead sets the current head block to the one defined by the hash @@ -378,16 +397,17 @@ func (self *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { } // Reset purges the entire blockchain, restoring it to its genesis state. -func (bc *BlockChain) Reset() { - bc.ResetWithGenesisBlock(bc.genesisBlock) +func (bc *BlockChain) Reset() error { + return bc.ResetWithGenesisBlock(bc.genesisBlock) } // ResetWithGenesisBlock purges the entire blockchain, restoring it to the // specified genesis state. -func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) { +func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { // Dump the entire block chain and purge the caches - bc.SetHead(0) - + if err := bc.SetHead(0); err != nil { + return err + } bc.mu.Lock() defer bc.mu.Unlock() @@ -404,6 +424,8 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) { bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentFastBlock = bc.genesisBlock + + return nil } // Export writes the active chain to the given writer. @@ -790,12 +812,15 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain } // Update the head fast sync block if better self.mu.Lock() + head := blockChain[len(errs)-1] - if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(self.GetTd(head.Hash(), head.NumberU64())) < 0 { - if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil { - log.Crit("Failed to update head fast block hash", "err", err) + if td := self.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case + if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(td) < 0 { + if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil { + log.Crit("Failed to update head fast block hash", "err", err) + } + self.currentFastBlock = head } - self.currentFastBlock = head } self.mu.Unlock() @@ -1288,6 +1313,11 @@ Error: %v // of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. func (self *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { + start := time.Now() + if i, err := self.hc.ValidateHeaderChain(chain, checkFreq); err != nil { + return i, err + } + // Make sure only one thread manipulates the chain at once self.chainmu.Lock() defer self.chainmu.Unlock() @@ -1303,7 +1333,7 @@ func (self *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) return err } - return self.hc.InsertHeaderChain(chain, checkFreq, whFunc) + return self.hc.InsertHeaderChain(chain, whFunc, start) } // writeHeader writes a header into the local chain, given that its parent is diff --git a/core/headerchain.go b/core/headerchain.go index a3d622087..57da9771b 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -219,7 +219,8 @@ type WhCallback func(*types.Header) error // should be done or not. The reason behind the optional check is because some // of the header retrieval mechanisms already need to verfy nonces, as well as // because nonces can be verified sparsely, not needing to check each. -func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, writeHeader WhCallback) (int, error) { + +func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) (int, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].Number.Uint64() != chain[i-1].Number.Uint64()+1 || chain[i].ParentHash != chain[i-1].Hash() { @@ -231,9 +232,6 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, w chain[i-1].Hash().Bytes()[:4], i, chain[i].Number, chain[i].Hash().Bytes()[:4], chain[i].ParentHash[:4]) } } - // Collect some import statistics to report on - stats := struct{ processed, ignored int }{} - start := time.Now() // Generate the list of headers that should be POW verified verify := make([]bool, len(chain)) @@ -309,6 +307,13 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, w } } } + + return 0, nil +} + +func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCallback, start time.Time) (int, error) { + // Collect some import statistics to report on + stats := struct{ processed, ignored int }{} // All headers passed verification, import them into the database for i, header := range chain { // Short circuit insertion if shutting down diff --git a/core/types/block.go b/core/types/block.go index 1dae87962..b699ba686 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -19,8 +19,6 @@ package types import ( "encoding/binary" - "encoding/json" - "errors" "fmt" "io" "math/big" @@ -39,12 +37,6 @@ var ( EmptyUncleHash = CalcUncleHash(nil) ) -var ( - errMissingHeaderMixDigest = errors.New("missing mixHash in JSON block header") - errMissingHeaderFields = errors.New("missing required JSON block header fields") - errBadNonceSize = errors.New("invalid block nonce size, want 8 bytes") -) - // A BlockNonce is a 64-bit hash which proves (combined with the // mix-hash) that a sufficient amount of computation has been carried // out on a block. @@ -72,41 +64,35 @@ func (n *BlockNonce) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) } +//go:generate gencodec -type Header -field-override headerMarshaling -out gen_header_json.go + // Header represents a block header in the Ethereum blockchain. type Header struct { - ParentHash common.Hash // Hash to the previous block - UncleHash common.Hash // Uncles of this block - Coinbase common.Address // The coin base address - Root common.Hash // Block Trie state - TxHash common.Hash // Tx sha - ReceiptHash common.Hash // Receipt sha - Bloom Bloom // Bloom - Difficulty *big.Int // Difficulty for the current block - Number *big.Int // The block number - GasLimit *big.Int // Gas limit - GasUsed *big.Int // Gas used - Time *big.Int // Creation time - Extra []byte // Extra data - MixDigest common.Hash // for quick difficulty verification - Nonce BlockNonce -} - -type jsonHeader struct { - ParentHash *common.Hash `json:"parentHash"` - UncleHash *common.Hash `json:"sha3Uncles"` - Coinbase *common.Address `json:"miner"` - Root *common.Hash `json:"stateRoot"` - TxHash *common.Hash `json:"transactionsRoot"` - ReceiptHash *common.Hash `json:"receiptsRoot"` - Bloom *Bloom `json:"logsBloom"` - Difficulty *hexutil.Big `json:"difficulty"` - Number *hexutil.Big `json:"number"` - GasLimit *hexutil.Big `json:"gasLimit"` - GasUsed *hexutil.Big `json:"gasUsed"` - Time *hexutil.Big `json:"timestamp"` - Extra *hexutil.Bytes `json:"extraData"` - MixDigest *common.Hash `json:"mixHash"` - Nonce *BlockNonce `json:"nonce"` + ParentHash common.Hash `json:"parentHash"` + UncleHash common.Hash `json:"sha3Uncles"` + Coinbase common.Address `json:"miner"` + Root common.Hash `json:"stateRoot"` + TxHash common.Hash `json:"transactionsRoot"` + ReceiptHash common.Hash `json:"receiptsRoot"` + Bloom Bloom `json:"logsBloom"` + Difficulty *big.Int `json:"difficulty"` + Number *big.Int `json:"number"` + GasLimit *big.Int `json:"gasLimit"` + GasUsed *big.Int `json:"gasUsed"` + Time *big.Int `json:"timestamp"` + Extra []byte `json:"extraData"` + MixDigest common.Hash `json:"mixHash"` + Nonce BlockNonce `json:"nonce"` +} + +// field type overrides for gencodec +type headerMarshaling struct { + Difficulty *hexutil.Big + Number *hexutil.Big + GasLimit *hexutil.Big + GasUsed *hexutil.Big + Time *hexutil.Big + Extra hexutil.Bytes } // Hash returns the block hash of the header, which is simply the keccak256 hash of its @@ -134,65 +120,6 @@ func (h *Header) HashNoNonce() common.Hash { }) } -// MarshalJSON encodes headers into the web3 RPC response block format. -func (h *Header) MarshalJSON() ([]byte, error) { - return json.Marshal(&jsonHeader{ - ParentHash: &h.ParentHash, - UncleHash: &h.UncleHash, - Coinbase: &h.Coinbase, - Root: &h.Root, - TxHash: &h.TxHash, - ReceiptHash: &h.ReceiptHash, - Bloom: &h.Bloom, - Difficulty: (*hexutil.Big)(h.Difficulty), - Number: (*hexutil.Big)(h.Number), - GasLimit: (*hexutil.Big)(h.GasLimit), - GasUsed: (*hexutil.Big)(h.GasUsed), - Time: (*hexutil.Big)(h.Time), - Extra: (*hexutil.Bytes)(&h.Extra), - MixDigest: &h.MixDigest, - Nonce: &h.Nonce, - }) -} - -// UnmarshalJSON decodes headers from the web3 RPC response block format. -func (h *Header) UnmarshalJSON(input []byte) error { - var dec jsonHeader - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - // Ensure that all fields are set. MixDigest is checked separately because - // it is a recent addition to the spec (as of August 2016) and older RPC server - // implementations might not provide it. - if dec.MixDigest == nil { - return errMissingHeaderMixDigest - } - if dec.ParentHash == nil || dec.UncleHash == nil || dec.Coinbase == nil || - dec.Root == nil || dec.TxHash == nil || dec.ReceiptHash == nil || - dec.Bloom == nil || dec.Difficulty == nil || dec.Number == nil || - dec.GasLimit == nil || dec.GasUsed == nil || dec.Time == nil || - dec.Extra == nil || dec.Nonce == nil { - return errMissingHeaderFields - } - // Assign all values. - h.ParentHash = *dec.ParentHash - h.UncleHash = *dec.UncleHash - h.Coinbase = *dec.Coinbase - h.Root = *dec.Root - h.TxHash = *dec.TxHash - h.ReceiptHash = *dec.ReceiptHash - h.Bloom = *dec.Bloom - h.Difficulty = (*big.Int)(dec.Difficulty) - h.Number = (*big.Int)(dec.Number) - h.GasLimit = (*big.Int)(dec.GasLimit) - h.GasUsed = (*big.Int)(dec.GasUsed) - h.Time = (*big.Int)(dec.Time) - h.Extra = *dec.Extra - h.MixDigest = *dec.MixDigest - h.Nonce = *dec.Nonce - return nil -} - func rlpHash(x interface{}) (h common.Hash) { hw := sha3.NewKeccak256() rlp.Encode(hw, x) diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go new file mode 100644 index 000000000..860622e6e --- /dev/null +++ b/core/types/gen_header_json.go @@ -0,0 +1,136 @@ +// generated by github.com/fjl/gencodec, do not edit. + +package types + +import ( + "encoding/json" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (h *Header) MarshalJSON() ([]byte, error) { + type HeaderJSON struct { + ParentHash *common.Hash `json:"parentHash"` + UncleHash *common.Hash `json:"sha3Uncles"` + Coinbase *common.Address `json:"miner"` + Root *common.Hash `json:"stateRoot"` + TxHash *common.Hash `json:"transactionsRoot"` + ReceiptHash *common.Hash `json:"receiptsRoot"` + Bloom *Bloom `json:"logsBloom"` + Difficulty *hexutil.Big `json:"difficulty"` + Number *hexutil.Big `json:"number"` + GasLimit *hexutil.Big `json:"gasLimit"` + GasUsed *hexutil.Big `json:"gasUsed"` + Time *hexutil.Big `json:"timestamp"` + Extra hexutil.Bytes `json:"extraData"` + MixDigest *common.Hash `json:"mixHash"` + Nonce *BlockNonce `json:"nonce"` + } + var enc HeaderJSON + enc.ParentHash = &h.ParentHash + enc.UncleHash = &h.UncleHash + enc.Coinbase = &h.Coinbase + enc.Root = &h.Root + enc.TxHash = &h.TxHash + enc.ReceiptHash = &h.ReceiptHash + enc.Bloom = &h.Bloom + enc.Difficulty = (*hexutil.Big)(h.Difficulty) + enc.Number = (*hexutil.Big)(h.Number) + enc.GasLimit = (*hexutil.Big)(h.GasLimit) + enc.GasUsed = (*hexutil.Big)(h.GasUsed) + enc.Time = (*hexutil.Big)(h.Time) + enc.Extra = h.Extra + enc.MixDigest = &h.MixDigest + enc.Nonce = &h.Nonce + return json.Marshal(&enc) +} + +func (h *Header) UnmarshalJSON(input []byte) error { + type HeaderJSON struct { + ParentHash *common.Hash `json:"parentHash"` + UncleHash *common.Hash `json:"sha3Uncles"` + Coinbase *common.Address `json:"miner"` + Root *common.Hash `json:"stateRoot"` + TxHash *common.Hash `json:"transactionsRoot"` + ReceiptHash *common.Hash `json:"receiptsRoot"` + Bloom *Bloom `json:"logsBloom"` + Difficulty *hexutil.Big `json:"difficulty"` + Number *hexutil.Big `json:"number"` + GasLimit *hexutil.Big `json:"gasLimit"` + GasUsed *hexutil.Big `json:"gasUsed"` + Time *hexutil.Big `json:"timestamp"` + Extra hexutil.Bytes `json:"extraData"` + MixDigest *common.Hash `json:"mixHash"` + Nonce *BlockNonce `json:"nonce"` + } + var dec HeaderJSON + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + var x Header + if dec.ParentHash == nil { + return errors.New("missing required field 'parentHash' for Header") + } + x.ParentHash = *dec.ParentHash + if dec.UncleHash == nil { + return errors.New("missing required field 'sha3Uncles' for Header") + } + x.UncleHash = *dec.UncleHash + if dec.Coinbase == nil { + return errors.New("missing required field 'miner' for Header") + } + x.Coinbase = *dec.Coinbase + if dec.Root == nil { + return errors.New("missing required field 'stateRoot' for Header") + } + x.Root = *dec.Root + if dec.TxHash == nil { + return errors.New("missing required field 'transactionsRoot' for Header") + } + x.TxHash = *dec.TxHash + if dec.ReceiptHash == nil { + return errors.New("missing required field 'receiptsRoot' for Header") + } + x.ReceiptHash = *dec.ReceiptHash + if dec.Bloom == nil { + return errors.New("missing required field 'logsBloom' for Header") + } + x.Bloom = *dec.Bloom + if dec.Difficulty == nil { + return errors.New("missing required field 'difficulty' for Header") + } + x.Difficulty = (*big.Int)(dec.Difficulty) + if dec.Number == nil { + return errors.New("missing required field 'number' for Header") + } + x.Number = (*big.Int)(dec.Number) + if dec.GasLimit == nil { + return errors.New("missing required field 'gasLimit' for Header") + } + x.GasLimit = (*big.Int)(dec.GasLimit) + if dec.GasUsed == nil { + return errors.New("missing required field 'gasUsed' for Header") + } + x.GasUsed = (*big.Int)(dec.GasUsed) + if dec.Time == nil { + return errors.New("missing required field 'timestamp' for Header") + } + x.Time = (*big.Int)(dec.Time) + if dec.Extra == nil { + return errors.New("missing required field 'extraData' for Header") + } + x.Extra = dec.Extra + if dec.MixDigest == nil { + return errors.New("missing required field 'mixHash' for Header") + } + x.MixDigest = *dec.MixDigest + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' for Header") + } + x.Nonce = *dec.Nonce + *h = x + return nil +} diff --git a/core/types/gen_log_json.go b/core/types/gen_log_json.go new file mode 100644 index 000000000..ef2cdfd89 --- /dev/null +++ b/core/types/gen_log_json.go @@ -0,0 +1,90 @@ +// generated by github.com/fjl/gencodec, do not edit. + +package types + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (l *Log) MarshalJSON() ([]byte, error) { + type LogJSON struct { + Address *common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + BlockNumber *hexutil.Uint64 `json:"blockNumber" optional:"yes"` + TxHash *common.Hash `json:"transactionHash"` + TxIndex *hexutil.Uint `json:"transactionIndex"` + BlockHash *common.Hash `json:"blockHash" optional:"yes"` + Index *hexutil.Uint `json:"logIndex"` + Removed *bool `json:"removed" optional:"yes"` + } + var enc LogJSON + enc.Address = &l.Address + enc.Topics = l.Topics + enc.Data = l.Data + enc.BlockNumber = (*hexutil.Uint64)(&l.BlockNumber) + enc.TxHash = &l.TxHash + enc.TxIndex = (*hexutil.Uint)(&l.TxIndex) + enc.BlockHash = &l.BlockHash + enc.Index = (*hexutil.Uint)(&l.Index) + enc.Removed = &l.Removed + return json.Marshal(&enc) +} + +func (l *Log) UnmarshalJSON(input []byte) error { + type LogJSON struct { + Address *common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + BlockNumber *hexutil.Uint64 `json:"blockNumber" optional:"yes"` + TxHash *common.Hash `json:"transactionHash"` + TxIndex *hexutil.Uint `json:"transactionIndex"` + BlockHash *common.Hash `json:"blockHash" optional:"yes"` + Index *hexutil.Uint `json:"logIndex"` + Removed *bool `json:"removed" optional:"yes"` + } + var dec LogJSON + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + var x Log + if dec.Address == nil { + return errors.New("missing required field 'address' for Log") + } + x.Address = *dec.Address + if dec.Topics == nil { + return errors.New("missing required field 'topics' for Log") + } + x.Topics = dec.Topics + if dec.Data == nil { + return errors.New("missing required field 'data' for Log") + } + x.Data = dec.Data + if dec.BlockNumber != nil { + x.BlockNumber = uint64(*dec.BlockNumber) + } + if dec.TxHash == nil { + return errors.New("missing required field 'transactionHash' for Log") + } + x.TxHash = *dec.TxHash + if dec.TxIndex == nil { + return errors.New("missing required field 'transactionIndex' for Log") + } + x.TxIndex = uint(*dec.TxIndex) + if dec.BlockHash != nil { + x.BlockHash = *dec.BlockHash + } + if dec.Index == nil { + return errors.New("missing required field 'logIndex' for Log") + } + x.Index = uint(*dec.Index) + if dec.Removed != nil { + x.Removed = *dec.Removed + } + *l = x + return nil +} diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go new file mode 100644 index 000000000..b9e9bee26 --- /dev/null +++ b/core/types/gen_receipt_json.go @@ -0,0 +1,79 @@ +// generated by github.com/fjl/gencodec, do not edit. + +package types + +import ( + "encoding/json" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (r *Receipt) MarshalJSON() ([]byte, error) { + type ReceiptJSON struct { + PostState hexutil.Bytes `json:"root"` + CumulativeGasUsed *hexutil.Big `json:"cumulativeGasUsed"` + Bloom *Bloom `json:"logsBloom"` + Logs []*Log `json:"logs"` + TxHash *common.Hash `json:"transactionHash"` + ContractAddress *common.Address `json:"contractAddress" optional:"true"` + GasUsed *hexutil.Big `json:"gasUsed"` + } + var enc ReceiptJSON + enc.PostState = r.PostState + enc.CumulativeGasUsed = (*hexutil.Big)(r.CumulativeGasUsed) + enc.Bloom = &r.Bloom + enc.Logs = r.Logs + enc.TxHash = &r.TxHash + enc.ContractAddress = &r.ContractAddress + enc.GasUsed = (*hexutil.Big)(r.GasUsed) + return json.Marshal(&enc) +} + +func (r *Receipt) UnmarshalJSON(input []byte) error { + type ReceiptJSON struct { + PostState hexutil.Bytes `json:"root"` + CumulativeGasUsed *hexutil.Big `json:"cumulativeGasUsed"` + Bloom *Bloom `json:"logsBloom"` + Logs []*Log `json:"logs"` + TxHash *common.Hash `json:"transactionHash"` + ContractAddress *common.Address `json:"contractAddress" optional:"true"` + GasUsed *hexutil.Big `json:"gasUsed"` + } + var dec ReceiptJSON + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + var x Receipt + if dec.PostState == nil { + return errors.New("missing required field 'root' for Receipt") + } + x.PostState = dec.PostState + if dec.CumulativeGasUsed == nil { + return errors.New("missing required field 'cumulativeGasUsed' for Receipt") + } + x.CumulativeGasUsed = (*big.Int)(dec.CumulativeGasUsed) + if dec.Bloom == nil { + return errors.New("missing required field 'logsBloom' for Receipt") + } + x.Bloom = *dec.Bloom + if dec.Logs == nil { + return errors.New("missing required field 'logs' for Receipt") + } + x.Logs = dec.Logs + if dec.TxHash == nil { + return errors.New("missing required field 'transactionHash' for Receipt") + } + x.TxHash = *dec.TxHash + if dec.ContractAddress != nil { + x.ContractAddress = *dec.ContractAddress + } + if dec.GasUsed == nil { + return errors.New("missing required field 'gasUsed' for Receipt") + } + x.GasUsed = (*big.Int)(dec.GasUsed) + *r = x + return nil +} diff --git a/core/types/gen_tx_json.go b/core/types/gen_tx_json.go new file mode 100644 index 000000000..8bbe629d7 --- /dev/null +++ b/core/types/gen_tx_json.go @@ -0,0 +1,99 @@ +// generated by github.com/fjl/gencodec, do not edit. + +package types + +import ( + "encoding/json" + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (t *txdata) MarshalJSON() ([]byte, error) { + type txdataJSON struct { + AccountNonce *hexutil.Uint64 `json:"nonce"` + Price *hexutil.Big `json:"gasPrice"` + GasLimit *hexutil.Big `json:"gasLimit"` + Recipient *common.Address `json:"to" optional:"yes" rlp:"nil"` + Amount *hexutil.Big `json:"value"` + Payload hexutil.Bytes `json:"input"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + Hash *common.Hash `json:"hash" optional:"yes" rlp:"-"` + } + var enc txdataJSON + enc.AccountNonce = (*hexutil.Uint64)(&t.AccountNonce) + enc.Price = (*hexutil.Big)(t.Price) + enc.GasLimit = (*hexutil.Big)(t.GasLimit) + enc.Recipient = t.Recipient + enc.Amount = (*hexutil.Big)(t.Amount) + enc.Payload = t.Payload + enc.V = (*hexutil.Big)(t.V) + enc.R = (*hexutil.Big)(t.R) + enc.S = (*hexutil.Big)(t.S) + enc.Hash = t.Hash + return json.Marshal(&enc) +} + +func (t *txdata) UnmarshalJSON(input []byte) error { + type txdataJSON struct { + AccountNonce *hexutil.Uint64 `json:"nonce"` + Price *hexutil.Big `json:"gasPrice"` + GasLimit *hexutil.Big `json:"gasLimit"` + Recipient *common.Address `json:"to" optional:"yes" rlp:"nil"` + Amount *hexutil.Big `json:"value"` + Payload hexutil.Bytes `json:"input"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + Hash *common.Hash `json:"hash" optional:"yes" rlp:"-"` + } + var dec txdataJSON + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + var x txdata + if dec.AccountNonce == nil { + return errors.New("missing required field 'nonce' for txdata") + } + x.AccountNonce = uint64(*dec.AccountNonce) + if dec.Price == nil { + return errors.New("missing required field 'gasPrice' for txdata") + } + x.Price = (*big.Int)(dec.Price) + if dec.GasLimit == nil { + return errors.New("missing required field 'gasLimit' for txdata") + } + x.GasLimit = (*big.Int)(dec.GasLimit) + if dec.Recipient != nil { + x.Recipient = dec.Recipient + } + if dec.Amount == nil { + return errors.New("missing required field 'value' for txdata") + } + x.Amount = (*big.Int)(dec.Amount) + if dec.Payload == nil { + return errors.New("missing required field 'input' for txdata") + } + x.Payload = dec.Payload + if dec.V == nil { + return errors.New("missing required field 'v' for txdata") + } + x.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' for txdata") + } + x.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' for txdata") + } + x.S = (*big.Int)(dec.S) + if dec.Hash != nil { + x.Hash = dec.Hash + } + *t = x + return nil +} diff --git a/core/types/log.go b/core/types/log.go index 7efb06b5c..57fc7b363 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -17,8 +17,6 @@ package types import ( - "encoding/json" - "errors" "fmt" "io" @@ -27,27 +25,42 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -var errMissingLogFields = errors.New("missing required JSON log fields") +//go:generate gencodec -type Log -field-override logMarshaling -out gen_log_json.go // Log represents a contract log event. These events are generated by the LOG opcode and // stored/indexed by the node. type Log struct { - // Consensus fields. - Address common.Address // address of the contract that generated the event - Topics []common.Hash // list of topics provided by the contract. - Data []byte // supplied by the contract, usually ABI-encoded + // Consensus fields: + // address of the contract that generated the event + Address common.Address `json:"address"` + // list of topics provided by the contract. + Topics []common.Hash `json:"topics"` + // supplied by the contract, usually ABI-encoded + Data []byte `json:"data"` // Derived fields. These fields are filled in by the node // but not secured by consensus. - BlockNumber uint64 // block in which the transaction was included - TxHash common.Hash // hash of the transaction - TxIndex uint // index of the transaction in the block - BlockHash common.Hash // hash of the block in which the transaction was included - Index uint // index of the log in the receipt + // block in which the transaction was included + BlockNumber uint64 `json:"blockNumber" optional:"yes"` + // hash of the transaction + TxHash common.Hash `json:"transactionHash"` + // index of the transaction in the block + TxIndex uint `json:"transactionIndex"` + // hash of the block in which the transaction was included + BlockHash common.Hash `json:"blockHash" optional:"yes"` + // index of the log in the receipt + Index uint `json:"logIndex"` // The Removed field is true if this log was reverted due to a chain reorganisation. // You must pay attention to this field if you receive logs through a filter query. - Removed bool + Removed bool `json:"removed" optional:"yes"` +} + +type logMarshaling struct { + Data hexutil.Bytes + BlockNumber hexutil.Uint64 + TxIndex hexutil.Uint + Index hexutil.Uint } type rlpLog struct { @@ -67,18 +80,6 @@ type rlpStorageLog struct { Index uint } -type jsonLog struct { - Address *common.Address `json:"address"` - Topics *[]common.Hash `json:"topics"` - Data *hexutil.Bytes `json:"data"` - BlockNumber *hexutil.Uint64 `json:"blockNumber"` - TxIndex *hexutil.Uint `json:"transactionIndex"` - TxHash *common.Hash `json:"transactionHash"` - BlockHash *common.Hash `json:"blockHash"` - Index *hexutil.Uint `json:"logIndex"` - Removed bool `json:"removed"` -} - // EncodeRLP implements rlp.Encoder. func (l *Log) EncodeRLP(w io.Writer) error { return rlp.Encode(w, rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}) @@ -98,54 +99,6 @@ func (l *Log) String() string { return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index) } -// MarshalJSON implements json.Marshaler. -func (l *Log) MarshalJSON() ([]byte, error) { - jslog := &jsonLog{ - Address: &l.Address, - Topics: &l.Topics, - Data: (*hexutil.Bytes)(&l.Data), - TxIndex: (*hexutil.Uint)(&l.TxIndex), - TxHash: &l.TxHash, - Index: (*hexutil.Uint)(&l.Index), - Removed: l.Removed, - } - // Set block information for mined logs. - if (l.BlockHash != common.Hash{}) { - jslog.BlockHash = &l.BlockHash - jslog.BlockNumber = (*hexutil.Uint64)(&l.BlockNumber) - } - return json.Marshal(jslog) -} - -// UnmarshalJSON implements json.Umarshaler. -func (l *Log) UnmarshalJSON(input []byte) error { - var dec jsonLog - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - if dec.Address == nil || dec.Topics == nil || dec.Data == nil || - dec.TxIndex == nil || dec.TxHash == nil || dec.Index == nil { - return errMissingLogFields - } - declog := Log{ - Address: *dec.Address, - Topics: *dec.Topics, - Data: *dec.Data, - TxHash: *dec.TxHash, - TxIndex: uint(*dec.TxIndex), - Index: uint(*dec.Index), - Removed: dec.Removed, - } - // Block information may be missing if the log is received through - // the pending log filter, so it's handled specially here. - if dec.BlockHash != nil && dec.BlockNumber != nil { - declog.BlockHash = *dec.BlockHash - declog.BlockNumber = uint64(*dec.BlockNumber) - } - *l = declog - return nil -} - // LogForStorage is a wrapper around a Log that flattens and parses the entire content of // a log including non-consensus fields. type LogForStorage Log diff --git a/core/types/log_test.go b/core/types/log_test.go index bf742ccac..0e56acfe4 100644 --- a/core/types/log_test.go +++ b/core/types/log_test.go @@ -18,6 +18,7 @@ package types import ( "encoding/json" + "fmt" "reflect" "testing" @@ -96,7 +97,7 @@ var unmarshalLogTests = map[string]struct { }, "missing data": { input: `{"address":"0xecf8f87f810ecf450940c9f60066b4a7a501d6a7","blockHash":"0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056","blockNumber":"0x1ecfa4","logIndex":"0x2","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615","0x000000000000000000000000f9dff387dcb5cc4cca5b91adb07a95f54e9f1bb6"],"transactionHash":"0x3b198bfd5d2907285af009e9ae84a0ecd63677110d89d7e030251acb87f6487e","transactionIndex":"0x3"}`, - wantError: errMissingLogFields, + wantError: fmt.Errorf("missing required field 'data' for Log"), }, } diff --git a/core/types/receipt.go b/core/types/receipt.go index 0a6a35e33..5bfcb15fc 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -17,8 +17,6 @@ package types import ( - "encoding/json" - "errors" "fmt" "io" "math/big" @@ -28,33 +26,26 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -var ( - errMissingReceiptPostState = errors.New("missing post state root in JSON receipt") - errMissingReceiptFields = errors.New("missing required JSON receipt fields") -) +//go:generate gencodec -type Receipt -field-override receiptMarshaling -out gen_receipt_json.go // Receipt represents the results of a transaction. type Receipt struct { // Consensus fields - PostState []byte - CumulativeGasUsed *big.Int - Bloom Bloom - Logs []*Log + PostState []byte `json:"root"` + CumulativeGasUsed *big.Int `json:"cumulativeGasUsed"` + Bloom Bloom `json:"logsBloom"` + Logs []*Log `json:"logs"` // Implementation fields (don't reorder!) - TxHash common.Hash - ContractAddress common.Address - GasUsed *big.Int + TxHash common.Hash `json:"transactionHash"` + ContractAddress common.Address `json:"contractAddress" optional:"true"` + GasUsed *big.Int `json:"gasUsed"` } -type jsonReceipt struct { - PostState *common.Hash `json:"root"` - CumulativeGasUsed *hexutil.Big `json:"cumulativeGasUsed"` - Bloom *Bloom `json:"logsBloom"` - Logs []*Log `json:"logs"` - TxHash *common.Hash `json:"transactionHash"` - ContractAddress *common.Address `json:"contractAddress"` - GasUsed *hexutil.Big `json:"gasUsed"` +type receiptMarshaling struct { + PostState hexutil.Bytes + CumulativeGasUsed *hexutil.Big + GasUsed *hexutil.Big } // NewReceipt creates a barebone transaction receipt, copying the init fields. @@ -84,51 +75,6 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error { return nil } -// MarshalJSON encodes receipts into the web3 RPC response block format. -func (r *Receipt) MarshalJSON() ([]byte, error) { - root := common.BytesToHash(r.PostState) - - return json.Marshal(&jsonReceipt{ - PostState: &root, - CumulativeGasUsed: (*hexutil.Big)(r.CumulativeGasUsed), - Bloom: &r.Bloom, - Logs: r.Logs, - TxHash: &r.TxHash, - ContractAddress: &r.ContractAddress, - GasUsed: (*hexutil.Big)(r.GasUsed), - }) -} - -// UnmarshalJSON decodes the web3 RPC receipt format. -func (r *Receipt) UnmarshalJSON(input []byte) error { - var dec jsonReceipt - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - // Ensure that all fields are set. PostState is checked separately because it is a - // recent addition to the RPC spec (as of August 2016) and older implementations might - // not provide it. Note that ContractAddress is not checked because it can be null. - if dec.PostState == nil { - return errMissingReceiptPostState - } - if dec.CumulativeGasUsed == nil || dec.Bloom == nil || - dec.Logs == nil || dec.TxHash == nil || dec.GasUsed == nil { - return errMissingReceiptFields - } - *r = Receipt{ - PostState: (*dec.PostState)[:], - CumulativeGasUsed: (*big.Int)(dec.CumulativeGasUsed), - Bloom: *dec.Bloom, - Logs: dec.Logs, - TxHash: *dec.TxHash, - GasUsed: (*big.Int)(dec.GasUsed), - } - if dec.ContractAddress != nil { - r.ContractAddress = *dec.ContractAddress - } - return nil -} - // String implements the Stringer interface. func (r *Receipt) String() string { return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs) diff --git a/core/types/transaction.go b/core/types/transaction.go index ab0bba4dc..a02f9ed00 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -18,7 +18,6 @@ package types import ( "container/heap" - "encoding/json" "errors" "fmt" "io" @@ -32,12 +31,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -var ErrInvalidSig = errors.New("invalid transaction v, r, s values") +//go:generate gencodec -type txdata -field-override txdataMarshaling -out gen_tx_json.go var ( - errMissingTxSignatureFields = errors.New("missing required JSON transaction signature fields") - errMissingTxFields = errors.New("missing required JSON transaction fields") - errNoSigner = errors.New("missing signing methods") + ErrInvalidSig = errors.New("invalid transaction v, r, s values") + errNoSigner = errors.New("missing signing methods") ) // deriveSigner makes a *best* guess about which signer to use. @@ -58,26 +56,31 @@ type Transaction struct { } type txdata struct { - AccountNonce uint64 - Price, GasLimit *big.Int - Recipient *common.Address `rlp:"nil"` // nil means contract creation - Amount *big.Int - Payload []byte - V *big.Int // signature - R, S *big.Int // signature -} - -type jsonTransaction struct { - Hash *common.Hash `json:"hash"` - AccountNonce *hexutil.Uint64 `json:"nonce"` - Price *hexutil.Big `json:"gasPrice"` - GasLimit *hexutil.Big `json:"gas"` - Recipient *common.Address `json:"to"` - Amount *hexutil.Big `json:"value"` - Payload *hexutil.Bytes `json:"input"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` + AccountNonce uint64 `json:"nonce"` + Price *big.Int `json:"gasPrice"` + GasLimit *big.Int `json:"gasLimit"` + Recipient *common.Address `json:"to" optional:"yes" rlp:"nil"` // nil means contract creation + Amount *big.Int `json:"value"` + Payload []byte `json:"input"` + + // Signature values + V *big.Int `json:"v"` + R *big.Int `json:"r"` + S *big.Int `json:"s"` + + // This is only used when marshaling to JSON. + Hash *common.Hash `json:"hash" optional:"yes" rlp:"-"` +} + +type txdataMarshaling struct { + AccountNonce hexutil.Uint64 + Price *hexutil.Big + GasLimit *hexutil.Big + Amount *hexutil.Big + Payload hexutil.Bytes + V *hexutil.Big + R *hexutil.Big + S *hexutil.Big } func NewTransaction(nonce uint64, to common.Address, amount, gasLimit, gasPrice *big.Int, data []byte) *Transaction { @@ -164,66 +167,30 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { return err } -// MarshalJSON encodes transactions into the web3 RPC response block format. func (tx *Transaction) MarshalJSON() ([]byte, error) { hash := tx.Hash() - - return json.Marshal(&jsonTransaction{ - Hash: &hash, - AccountNonce: (*hexutil.Uint64)(&tx.data.AccountNonce), - Price: (*hexutil.Big)(tx.data.Price), - GasLimit: (*hexutil.Big)(tx.data.GasLimit), - Recipient: tx.data.Recipient, - Amount: (*hexutil.Big)(tx.data.Amount), - Payload: (*hexutil.Bytes)(&tx.data.Payload), - V: (*hexutil.Big)(tx.data.V), - R: (*hexutil.Big)(tx.data.R), - S: (*hexutil.Big)(tx.data.S), - }) + data := tx.data + data.Hash = &hash + return data.MarshalJSON() } // UnmarshalJSON decodes the web3 RPC transaction format. func (tx *Transaction) UnmarshalJSON(input []byte) error { - var dec jsonTransaction - if err := json.Unmarshal(input, &dec); err != nil { + var dec txdata + if err := dec.UnmarshalJSON(input); err != nil { return err } - // Ensure that all fields are set. V, R, S are checked separately because they're a - // recent addition to the RPC spec (as of August 2016) and older implementations might - // not provide them. Note that Recipient is not checked because it can be missing for - // contract creations. - if dec.V == nil || dec.R == nil || dec.S == nil { - return errMissingTxSignatureFields - } - var V byte - if isProtectedV((*big.Int)(dec.V)) { - chainId := deriveChainId((*big.Int)(dec.V)).Uint64() - V = byte(dec.V.ToInt().Uint64() - 35 - 2*chainId) + if isProtectedV(dec.V) { + chainId := deriveChainId(dec.V).Uint64() + V = byte(dec.V.Uint64() - 35 - 2*chainId) } else { - V = byte(((*big.Int)(dec.V)).Uint64() - 27) + V = byte(dec.V.Uint64() - 27) } - if !crypto.ValidateSignatureValues(V, (*big.Int)(dec.R), (*big.Int)(dec.S), false) { + if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) { return ErrInvalidSig } - - if dec.AccountNonce == nil || dec.Price == nil || dec.GasLimit == nil || dec.Amount == nil || dec.Payload == nil { - return errMissingTxFields - } - // Assign the fields. This is not atomic but reusing transactions - // for decoding isn't thread safe anyway. - *tx = Transaction{} - tx.data = txdata{ - AccountNonce: uint64(*dec.AccountNonce), - Recipient: dec.Recipient, - Amount: (*big.Int)(dec.Amount), - GasLimit: (*big.Int)(dec.GasLimit), - Price: (*big.Int)(dec.Price), - Payload: *dec.Payload, - V: (*big.Int)(dec.V), - R: (*big.Int)(dec.R), - S: (*big.Int)(dec.S), - } + *tx = Transaction{data: dec} return nil } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 7d12ebc05..8ee9d3ca7 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -18,7 +18,6 @@ package vm import ( "fmt" - "math/big" "sync/atomic" "time" @@ -117,9 +116,7 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e if err != nil && evm.cfg.Debug { // XXX For debugging //fmt.Printf("%04d: %8v cost = %-8d stack = %-8d ERR = %v\n", pc, op, cost, stack.len(), err) - // TODO update the tracer - g, c := new(big.Int).SetUint64(contract.Gas), new(big.Int).SetUint64(cost) - evm.cfg.Tracer.CaptureState(evm.env, pc, op, g, c, mem, stack, contract, evm.env.depth, err) + evm.cfg.Tracer.CaptureState(evm.env, pc, op, contract.Gas, cost, mem, stack, contract, evm.env.depth, err) } }() @@ -177,8 +174,7 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e } if evm.cfg.Debug { - g, c := new(big.Int).SetUint64(contract.Gas), new(big.Int).SetUint64(cost) - evm.cfg.Tracer.CaptureState(evm.env, pc, op, g, c, mem, stack, contract, evm.env.depth, err) + evm.cfg.Tracer.CaptureState(evm.env, pc, op, contract.Gas, cost, mem, stack, contract, evm.env.depth, err) } // XXX For debugging //fmt.Printf("%04d: %8v cost = %-8d stack = %-8d\n", pc, op, cost, stack.len()) diff --git a/core/vm/logger.go b/core/vm/logger.go index 3d7e1c95f..825025b05 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -52,8 +52,8 @@ type LogConfig struct { type StructLog struct { Pc uint64 Op OpCode - Gas *big.Int - GasCost *big.Int + Gas uint64 + GasCost uint64 Memory []byte Stack []*big.Int Storage map[common.Hash]common.Hash @@ -67,7 +67,7 @@ type StructLog struct { // Note that reference types are actual VM data structures; make copies // if you need to retain them beyond the current call. type Tracer interface { - CaptureState(env *EVM, pc uint64, op OpCode, gas, cost *big.Int, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error + CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error } // StructLogger is an EVM state logger and implements Tracer. @@ -96,7 +96,7 @@ func NewStructLogger(cfg *LogConfig) *StructLogger { // captureState logs a new structured log message and pushes it out to the environment // // captureState also tracks SSTORE ops to track dirty values. -func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost *big.Int, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { +func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { // check if already accumulated the specified number of logs if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) { return ErrTraceLimitReached @@ -158,7 +158,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost *b } } // create a new snaptshot of the EVM. - log := StructLog{pc, op, new(big.Int).Set(gas), cost, mem, stck, storage, env.depth, err} + log := StructLog{pc, op, gas, cost, mem, stck, storage, env.depth, err} l.logs = append(l.logs, log) return nil diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index e755a18e2..b6fa31132 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -59,7 +59,7 @@ func TestStoreCapture(t *testing.T) { var index common.Hash - logger.CaptureState(env, 0, SSTORE, new(big.Int), new(big.Int), mem, stack, contract, 0, nil) + logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, contract, 0, nil) if len(logger.changedValues[contract.Address()]) == 0 { t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()])) } @@ -81,13 +81,13 @@ func TestStorageCapture(t *testing.T) { stack = newstack() ) - logger.CaptureState(env, 0, STOP, new(big.Int), new(big.Int), mem, stack, contract, 0, nil) + logger.CaptureState(env, 0, STOP, 0, 0, mem, stack, contract, 0, nil) if ref.calledForEach { t.Error("didn't expect for each to be called") } logger = NewStructLogger(&LogConfig{FullStorage: true}) - logger.CaptureState(env, 0, STOP, new(big.Int), new(big.Int), mem, stack, contract, 0, nil) + logger.CaptureState(env, 0, STOP, 0, 0, mem, stack, contract, 0, nil) if !ref.calledForEach { t.Error("expected for each to be called") } diff --git a/eth/api.go b/eth/api.go index b17968ebb..b64153fd7 100644 --- a/eth/api.go +++ b/eth/api.go @@ -19,6 +19,7 @@ package eth import ( "bytes" "compress/gzip" + "context" "errors" "fmt" "io" @@ -39,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/net/context" ) const defaultTraceTimeout = 5 * time.Second diff --git a/eth/api_backend.go b/eth/api_backend.go index 5a5c4c532..418a34435 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -17,6 +17,7 @@ package eth import ( + "context" "math/big" "github.com/ethereum/go-ethereum/accounts" @@ -33,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // EthApiBackend implements ethapi.Backend for full nodes @@ -51,6 +51,7 @@ func (b *EthApiBackend) CurrentBlock() *types.Block { } func (b *EthApiBackend) SetHead(number uint64) { + b.eth.protocolManager.downloader.Cancel() b.eth.blockchain.SetHead(number) } diff --git a/eth/bind.go b/eth/bind.go index 2ee9f2bf7..245934183 100644 --- a/eth/bind.go +++ b/eth/bind.go @@ -17,6 +17,7 @@ package eth import ( + "context" "math/big" "github.com/ethereum/go-ethereum" @@ -26,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // ContractBackend implements bind.ContractBackend with direct calls to Ethereum diff --git a/eth/downloader/api.go b/eth/downloader/api.go index e41376810..d496fa6a4 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -17,12 +17,12 @@ package downloader import ( + "context" "sync" ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // PublicDownloaderAPI provides an API which gives information about the current synchronisation status. diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f7aca031a..d26995782 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -277,7 +277,7 @@ func (d *Downloader) UnregisterPeer(id string) error { d.cancelLock.RUnlock() if master { - d.cancel() + d.Cancel() } return nil } @@ -352,7 +352,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode d.cancelPeer = id d.cancelLock.Unlock() - defer d.cancel() // No matter what, we can't leave the cancel channel open + defer d.Cancel() // No matter what, we can't leave the cancel channel open // Set the requested sync mode, unless it's forbidden d.mode = mode @@ -473,7 +473,7 @@ func (d *Downloader) spawnSync(origin uint64, fetchers ...func() error) error { } } d.queue.Close() - d.cancel() + d.Cancel() wg.Wait() // If sync failed in the critical section, bump the fail counter @@ -483,9 +483,9 @@ func (d *Downloader) spawnSync(origin uint64, fetchers ...func() error) error { return err } -// cancel cancels all of the operations and resets the queue. It returns true +// Cancel cancels all of the operations and resets the queue. It returns true // if the cancel operation was completed. -func (d *Downloader) cancel() { +func (d *Downloader) Cancel() { // Close the current cancel channel d.cancelLock.Lock() if d.cancelCh != nil { @@ -512,7 +512,7 @@ func (d *Downloader) Terminate() { d.quitLock.Unlock() // Cancel any pending download requests - d.cancel() + d.Cancel() } // fetchHeight retrieves the head header of the remote peer to aid in estimating @@ -945,7 +945,7 @@ func (d *Downloader) fetchNodeData() error { if err != nil { // If the node data processing failed, the root hash is very wrong, abort log.Error("State processing failed", "peer", packet.PeerId(), "err", err) - d.cancel() + d.Cancel() return } // Processing succeeded, notify state fetcher of continuation @@ -1208,7 +1208,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { if atomic.LoadUint32(&d.fsPivotFails) == 0 { for _, header := range rollback { if header.Number.Uint64() == pivot { - log.Warn("Fast-sync critical section failure, locked pivot to header", "number", pivot, "hash", header.Hash()) + log.Warn("Fast-sync pivot locked in", "number", pivot, "hash", header.Hash()) d.fsPivotLock = header } } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a9ea797ea..267a0def9 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -982,7 +982,7 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) // Make sure canceling works with a pristine downloader - tester.downloader.cancel() + tester.downloader.Cancel() if !tester.downloader.queue.Idle() { t.Errorf("download queue not idle") } @@ -990,7 +990,7 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - tester.downloader.cancel() + tester.downloader.Cancel() if !tester.downloader.queue.Idle() { t.Errorf("download queue not idle") } diff --git a/eth/filters/api.go b/eth/filters/api.go index 02a544ce1..61647a5d0 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -17,6 +17,7 @@ package filters import ( + "context" "encoding/json" "errors" "fmt" @@ -24,8 +25,6 @@ import ( "sync" "time" - "golang.org/x/net/context" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 9a8e2fd70..0a0b81224 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -17,10 +17,10 @@ package filters import ( + "context" "math" - "time" - "math/big" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) type Backend interface { diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 3adf8111a..7abace1e6 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -19,6 +19,7 @@ package filters import ( + "context" "errors" "fmt" "sync" @@ -29,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // Type determines the kind of filter and is used to put the filter in to @@ -372,7 +372,8 @@ func (es *EventSystem) lightFilterNewHead(newHeader *types.Header, callBack func func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.Address, topics [][]common.Hash, remove bool) []*types.Log { if bloomFilter(header.Bloom, addresses, topics) { // Get the logs of the block - ctx, _ := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() receipts, err := es.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 1cfced7e4..d9c245a85 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -17,13 +17,12 @@ package filters import ( + "context" "math/big" "reflect" "testing" "time" - "golang.org/x/net/context" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 83ff3e9ce..c2dc2b842 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -17,13 +17,12 @@ package filters import ( + "context" "io/ioutil" "math/big" "os" "testing" - "golang.org/x/net/context" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" diff --git a/eth/gasprice/lightprice.go b/eth/gasprice/lightprice.go index 8886d32d7..562c7dd97 100644 --- a/eth/gasprice/lightprice.go +++ b/eth/gasprice/lightprice.go @@ -17,6 +17,7 @@ package gasprice import ( + "context" "math/big" "sort" "sync" @@ -24,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) const ( diff --git a/eth/sync.go b/eth/sync.go index 6e2c7c432..f2cae6c19 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -175,6 +175,14 @@ func (pm *ProtocolManager) synchronise(peer *peer) { // Otherwise try to sync with the downloader mode := downloader.FullSync if atomic.LoadUint32(&pm.fastSync) == 1 { + // Fast sync was explicitly requested, and explicitly granted + mode = downloader.FastSync + } else if currentBlock.NumberU64() == 0 && pm.blockchain.CurrentFastBlock().NumberU64() > 0 { + // The database seems empty as the current block is the genesis. Yet the fast + // block is ahead, so fast sync was enabled for this node at a certain point. + // The only scenario where this can happen is if the user manually (or via a + // bad block) rolled back a fast sync node below the sync point. In this case + // however it's safe to reenable fast sync. mode = downloader.FastSync } if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 1d04d9e03..59f60d659 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -18,6 +18,7 @@ package ethclient import ( + "context" "encoding/json" "fmt" "math/big" @@ -28,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // Client defines typed wrappers for the Ethereum RPC API. diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 7b3a3439e..4d38d3a50 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -22,10 +22,11 @@ import ( "errors" "fmt" "math/big" + "net" + "net/url" "regexp" "runtime" "strconv" - "strings" "time" "github.com/ethereum/go-ethereum/common" @@ -123,17 +124,34 @@ func (s *Service) loop() { // Loop reporting until termination for { - // Establish a websocket connection to the server and authenticate the node - url := fmt.Sprintf("%s/api", s.host) - if !strings.Contains(url, "://") { - url = "wss://" + url + // Resolve the URL, defaulting to TLS, but falling back to none too + path := fmt.Sprintf("%s/api", s.host) + urls := []string{path} + + if parsed, err := url.Parse(path); err == nil && !parsed.IsAbs() { + urls = []string{"wss://" + path, "ws://" + path} + } + // Establish a websocket connection to the server on any supported URL + var ( + conf *websocket.Config + conn *websocket.Conn + err error + ) + for _, url := range urls { + if conf, err = websocket.NewConfig(url, "http://localhost/"); err != nil { + continue + } + conf.Dialer = &net.Dialer{Timeout: 3 * time.Second} + if conn, err = websocket.DialConfig(conf); err == nil { + break + } } - conn, err := websocket.Dial(url, "", "http://localhost/") if err != nil { log.Warn("Stats server unreachable", "err", err) time.Sleep(10 * time.Second) continue } + // Authenticate the client with the server in := json.NewDecoder(conn) out := json.NewEncoder(conn) @@ -244,12 +262,12 @@ func (s *Service) readLoop(conn *websocket.Conn, in *json.Decoder) { // Make sure the request is valid and doesn't crash us request, ok := msg["emit"][1].(map[string]interface{}) if !ok { - log.Warn("Invalid history request", "msg", msg["emit"][1]) - return + log.Warn("Invalid stats history request", "msg", msg["emit"][1]) + continue // Ethstats sometime sends invalid history requests, ignore those } list, ok := request["list"].([]interface{}) if !ok { - log.Warn("Invalid history block list", "list", request["list"]) + log.Warn("Invalid stats history block list", "list", request["list"]) return } // Convert the block number list to an integer list @@ -257,7 +275,7 @@ func (s *Service) readLoop(conn *websocket.Conn, in *json.Decoder) { for i, num := range list { n, ok := num.(float64) if !ok { - log.Warn("Invalid history block number", "number", num) + log.Warn("Invalid stats history block number", "number", num) return } numbers[i] = uint64(n) diff --git a/event/subscription.go b/event/subscription.go index 83bd21213..02d7b9d7d 100644 --- a/event/subscription.go +++ b/event/subscription.go @@ -17,11 +17,11 @@ package event import ( + "context" "sync" "time" "github.com/ethereum/go-ethereum/common/mclock" - "golang.org/x/net/context" ) // Subscription represents a stream of events. The carrier of the events is typically a diff --git a/event/subscription_test.go b/event/subscription_test.go index a4fe30298..aa6d98984 100644 --- a/event/subscription_test.go +++ b/event/subscription_test.go @@ -17,11 +17,10 @@ package event import ( + "context" "errors" "testing" "time" - - "golang.org/x/net/context" ) var errInts = errors.New("error in subscribeInts") diff --git a/interfaces.go b/interfaces.go index f7e71a317..744f07b95 100644 --- a/interfaces.go +++ b/interfaces.go @@ -18,12 +18,12 @@ package ethereum import ( + "context" "errors" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "golang.org/x/net/context" ) // NotFound is returned by API methods if the requested item does not exist. diff --git a/internal/build/util.go b/internal/build/util.go index 1523a067b..4df7b9138 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -52,19 +52,10 @@ func MustRunCommand(cmd string, args ...string) { // GOPATH returns the value that the GOPATH environment // variable should be set to. func GOPATH() string { - path := filepath.SplitList(os.Getenv("GOPATH")) - if len(path) == 0 { + if os.Getenv("GOPATH") == "" { log.Fatal("GOPATH is not set") } - // Ensure that our internal vendor folder is on GOPATH - vendor, _ := filepath.Abs(filepath.Join("build", "_vendor")) - for _, dir := range path { - if dir == vendor { - return strings.Join(path, string(filepath.ListSeparator)) - } - } - newpath := append(path[:1], append([]string{vendor}, path[1:]...)...) - return strings.Join(newpath, string(filepath.ListSeparator)) + return os.Getenv("GOPATH") } // VERSION returns the content of the VERSION file. diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index c8374fe18..ccb7ec80b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -18,6 +18,7 @@ package ethapi import ( "bytes" + "context" "encoding/hex" "errors" "fmt" @@ -43,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" - "golang.org/x/net/context" ) const ( @@ -695,8 +695,8 @@ type ExecutionResult struct { type StructLogRes struct { Pc uint64 `json:"pc"` Op string `json:"op"` - Gas *big.Int `json:"gas"` - GasCost *big.Int `json:"gasCost"` + Gas uint64 `json:"gas"` + GasCost uint64 `json:"gasCost"` Depth int `json:"depth"` Error error `json:"error"` Stack []string `json:"stack"` diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index e10fb14ff..50cd3801b 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -18,6 +18,7 @@ package ethapi import ( + "context" "math/big" "github.com/ethereum/go-ethereum/accounts" @@ -30,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // Backend interface provides the common API services (that are provided by diff --git a/internal/ethapi/tracer.go b/internal/ethapi/tracer.go index ef107fc42..fe2685375 100644 --- a/internal/ethapi/tracer.go +++ b/internal/ethapi/tracer.go @@ -278,7 +278,7 @@ func wrapError(context string, err error) error { } // CaptureState implements the Tracer interface to trace a single step of VM execution -func (jst *JavascriptTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost *big.Int, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { +func (jst *JavascriptTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { if jst.err == nil { jst.memory.memory = memory jst.stack.stack = stack @@ -288,8 +288,8 @@ func (jst *JavascriptTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, jst.log["pc"] = pc jst.log["op"] = ocw.toValue(jst.vm) - jst.log["gas"] = gas.Int64() - jst.log["gasPrice"] = cost.Int64() + jst.log["gas"] = gas + jst.log["gasPrice"] = cost jst.log["memory"] = jst.memvalue jst.log["stack"] = jst.stackvalue jst.log["depth"] = depth diff --git a/internal/ethapi/tracer_test.go b/internal/ethapi/tracer_test.go index 693afe802..0ef450ce3 100644 --- a/internal/ethapi/tracer_test.go +++ b/internal/ethapi/tracer_test.go @@ -136,10 +136,10 @@ func TestHaltBetweenSteps(t *testing.T) { env := vm.NewEVM(vm.Context{}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) - tracer.CaptureState(env, 0, 0, big.NewInt(0), big.NewInt(0), nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil) timeout := errors.New("stahp") tracer.Stop(timeout) - tracer.CaptureState(env, 0, 0, big.NewInt(0), big.NewInt(0), nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil) if _, err := tracer.GetResult(); err.Error() != "stahp in server-side tracer function 'step'" { t.Errorf("Expected timeout error, got %v", err) diff --git a/internal/jsre/deps/bindata.go b/internal/jsre/deps/bindata.go index 5f6a2b873..782d4df80 100644 --- a/internal/jsre/deps/bindata.go +++ b/internal/jsre/deps/bindata.go @@ -84,7 +84,7 @@ func bignumberJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "bignumber.js", size: 17314, mode: os.FileMode(420), modTime: time.Unix(1484232218, 0)} + info := bindataFileInfo{name: "bignumber.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -104,7 +104,7 @@ func web3Js() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "web3.js", size: 491740, mode: os.FileMode(420), modTime: time.Unix(1484232456, 0)} + info := bindataFileInfo{name: "web3.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/internal/jsre/deps/deps.go b/internal/jsre/deps/deps.go index 8d0e1a400..fe2e6f2fa 100644 --- a/internal/jsre/deps/deps.go +++ b/internal/jsre/deps/deps.go @@ -17,4 +17,5 @@ // Package deps contains the console JavaScript dependencies Go embedded. package deps -//go:generate go-bindata -o bindata.go bignumber.js web3.js +//go:generate go-bindata -nometadata -pkg deps -o bindata.go bignumber.js web3.js +//go:generate gofmt -w -s bindata.go diff --git a/les/api_backend.go b/les/api_backend.go index 264b381f5..df2782f78 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -17,6 +17,7 @@ package les import ( + "context" "math/big" "github.com/ethereum/go-ethereum/accounts" @@ -33,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) type LesApiBackend struct { @@ -50,6 +50,7 @@ func (b *LesApiBackend) CurrentBlock() *types.Block { } func (b *LesApiBackend) SetHead(number uint64) { + b.eth.protocolManager.downloader.Cancel() b.eth.blockchain.SetHead(number) } diff --git a/les/backend.go b/les/backend.go index 404728c0e..3cab75f33 100644 --- a/les/backend.go +++ b/les/backend.go @@ -107,6 +107,8 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.LightMode, config.NetworkId, eth.eventMux, eth.pow, eth.blockchain, nil, chainDb, odr, relay); err != nil { return nil, err } + relay.ps = eth.protocolManager.peers + relay.reqDist = eth.protocolManager.reqDist eth.ApiBackend = &LesApiBackend{eth, nil} eth.ApiBackend.gpo = gasprice.NewLightPriceOracle(eth.ApiBackend) diff --git a/les/distributor.go b/les/distributor.go new file mode 100644 index 000000000..c59b36146 --- /dev/null +++ b/les/distributor.go @@ -0,0 +1,259 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// Package light implements on-demand retrieval capable state and chain objects +// for the Ethereum Light Client. +package les + +import ( + "container/list" + "errors" + "sync" + "time" +) + +// ErrNoPeers is returned if no peers capable of serving a queued request are available +var ErrNoPeers = errors.New("no suitable peers available") + +// requestDistributor implements a mechanism that distributes requests to +// suitable peers, obeying flow control rules and prioritizing them in creation +// order (even when a resend is necessary). +type requestDistributor struct { + reqQueue *list.List + lastReqOrder uint64 + stopChn, loopChn chan struct{} + loopNextSent bool + lock sync.Mutex + + getAllPeers func() map[distPeer]struct{} +} + +// distPeer is an LES server peer interface for the request distributor. +// waitBefore returns either the necessary waiting time before sending a request +// with the given upper estimated cost or the estimated remaining relative buffer +// value after sending such a request (in which case the request can be sent +// immediately). At least one of these values is always zero. +type distPeer interface { + waitBefore(uint64) (time.Duration, float64) + canQueue() bool + queueSend(f func()) +} + +// distReq is the request abstraction used by the distributor. It is based on +// three callback functions: +// - getCost returns the upper estimate of the cost of sending the request to a given peer +// - canSend tells if the server peer is suitable to serve the request +// - request prepares sending the request to the given peer and returns a function that +// does the actual sending. Request order should be preserved but the callback itself should not +// block until it is sent because other peers might still be able to receive requests while +// one of them is blocking. Instead, the returned function is put in the peer's send queue. +type distReq struct { + getCost func(distPeer) uint64 + canSend func(distPeer) bool + request func(distPeer) func() + + reqOrder uint64 + sentChn chan distPeer + element *list.Element +} + +// newRequestDistributor creates a new request distributor +func newRequestDistributor(getAllPeers func() map[distPeer]struct{}, stopChn chan struct{}) *requestDistributor { + r := &requestDistributor{ + reqQueue: list.New(), + loopChn: make(chan struct{}, 2), + stopChn: stopChn, + getAllPeers: getAllPeers, + } + go r.loop() + return r +} + +// distMaxWait is the maximum waiting time after which further necessary waiting +// times are recalculated based on new feedback from the servers +const distMaxWait = time.Millisecond * 10 + +// main event loop +func (d *requestDistributor) loop() { + for { + select { + case <-d.stopChn: + d.lock.Lock() + elem := d.reqQueue.Front() + for elem != nil { + close(elem.Value.(*distReq).sentChn) + elem = elem.Next() + } + d.lock.Unlock() + return + case <-d.loopChn: + d.lock.Lock() + d.loopNextSent = false + loop: + for { + peer, req, wait := d.nextRequest() + if req != nil && wait == 0 { + chn := req.sentChn // save sentChn because remove sets it to nil + d.remove(req) + send := req.request(peer) + if send != nil { + peer.queueSend(send) + } + chn <- peer + close(chn) + } else { + if wait == 0 { + // no request to send and nothing to wait for; the next + // queued request will wake up the loop + break loop + } + d.loopNextSent = true // a "next" signal has been sent, do not send another one until this one has been received + if wait > distMaxWait { + // waiting times may be reduced by incoming request replies, if it is too long, recalculate it periodically + wait = distMaxWait + } + go func() { + time.Sleep(wait) + d.loopChn <- struct{}{} + }() + break loop + } + } + d.lock.Unlock() + } + } +} + +// selectPeerItem represents a peer to be selected for a request by weightedRandomSelect +type selectPeerItem struct { + peer distPeer + req *distReq + weight int64 +} + +// Weight implements wrsItem interface +func (sp selectPeerItem) Weight() int64 { + return sp.weight +} + +// nextRequest returns the next possible request from any peer, along with the +// associated peer and necessary waiting time +func (d *requestDistributor) nextRequest() (distPeer, *distReq, time.Duration) { + peers := d.getAllPeers() + + elem := d.reqQueue.Front() + var ( + bestPeer distPeer + bestReq *distReq + bestWait time.Duration + sel *weightedRandomSelect + ) + + for (len(peers) > 0 || elem == d.reqQueue.Front()) && elem != nil { + req := elem.Value.(*distReq) + canSend := false + for peer, _ := range peers { + if peer.canQueue() && req.canSend(peer) { + canSend = true + cost := req.getCost(peer) + wait, bufRemain := peer.waitBefore(cost) + if wait == 0 { + if sel == nil { + sel = newWeightedRandomSelect() + } + sel.update(selectPeerItem{peer: peer, req: req, weight: int64(bufRemain*1000000) + 1}) + } else { + if bestReq == nil || wait < bestWait { + bestPeer = peer + bestReq = req + bestWait = wait + } + } + delete(peers, peer) + } + } + next := elem.Next() + if !canSend && elem == d.reqQueue.Front() { + close(req.sentChn) + d.remove(req) + } + elem = next + } + + if sel != nil { + c := sel.choose().(selectPeerItem) + return c.peer, c.req, 0 + } + return bestPeer, bestReq, bestWait +} + +// queue adds a request to the distribution queue, returns a channel where the +// receiving peer is sent once the request has been sent (request callback returned). +// If the request is cancelled or timed out without suitable peers, the channel is +// closed without sending any peer references to it. +func (d *requestDistributor) queue(r *distReq) chan distPeer { + d.lock.Lock() + defer d.lock.Unlock() + + if r.reqOrder == 0 { + d.lastReqOrder++ + r.reqOrder = d.lastReqOrder + } + + back := d.reqQueue.Back() + if back == nil || r.reqOrder > back.Value.(*distReq).reqOrder { + r.element = d.reqQueue.PushBack(r) + } else { + before := d.reqQueue.Front() + for before.Value.(*distReq).reqOrder < r.reqOrder { + before = before.Next() + } + r.element = d.reqQueue.InsertBefore(r, before) + } + + if !d.loopNextSent { + d.loopNextSent = true + d.loopChn <- struct{}{} + } + + r.sentChn = make(chan distPeer, 1) + return r.sentChn +} + +// cancel removes a request from the queue if it has not been sent yet (returns +// false if it has been sent already). It is guaranteed that the callback functions +// will not be called after cancel returns. +func (d *requestDistributor) cancel(r *distReq) bool { + d.lock.Lock() + defer d.lock.Unlock() + + if r.sentChn == nil { + return false + } + + close(r.sentChn) + d.remove(r) + return true +} + +// remove removes a request from the queue +func (d *requestDistributor) remove(r *distReq) { + r.sentChn = nil + if r.element != nil { + d.reqQueue.Remove(r.element) + r.element = nil + } +} diff --git a/les/distributor_test.go b/les/distributor_test.go new file mode 100644 index 000000000..f2eb80729 --- /dev/null +++ b/les/distributor_test.go @@ -0,0 +1,192 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// Package light implements on-demand retrieval capable state and chain objects +// for the Ethereum Light Client. +package les + +import ( + "math/rand" + "sync" + "testing" + "time" +) + +type testDistReq struct { + cost, procTime, order uint64 + canSendTo map[*testDistPeer]struct{} +} + +func (r *testDistReq) getCost(dp distPeer) uint64 { + return r.cost +} + +func (r *testDistReq) canSend(dp distPeer) bool { + _, ok := r.canSendTo[dp.(*testDistPeer)] + return ok +} + +func (r *testDistReq) request(dp distPeer) func() { + return func() { dp.(*testDistPeer).send(r) } +} + +type testDistPeer struct { + sent []*testDistReq + sumCost uint64 + lock sync.RWMutex +} + +func (p *testDistPeer) send(r *testDistReq) { + p.lock.Lock() + defer p.lock.Unlock() + + p.sent = append(p.sent, r) + p.sumCost += r.cost +} + +func (p *testDistPeer) worker(t *testing.T, checkOrder bool, stop chan struct{}) { + var last uint64 + for { + wait := time.Millisecond + p.lock.Lock() + if len(p.sent) > 0 { + rq := p.sent[0] + wait = time.Duration(rq.procTime) + p.sumCost -= rq.cost + if checkOrder { + if rq.order <= last { + t.Errorf("Requests processed in wrong order") + } + last = rq.order + } + p.sent = p.sent[1:] + } + p.lock.Unlock() + select { + case <-stop: + return + case <-time.After(wait): + } + } +} + +const ( + testDistBufLimit = 10000000 + testDistMaxCost = 1000000 + testDistPeerCount = 5 + testDistReqCount = 50000 + testDistMaxResendCount = 3 +) + +func (p *testDistPeer) waitBefore(cost uint64) (time.Duration, float64) { + p.lock.RLock() + sumCost := p.sumCost + cost + p.lock.RUnlock() + if sumCost < testDistBufLimit { + return 0, float64(testDistBufLimit-sumCost) / float64(testDistBufLimit) + } else { + return time.Duration(sumCost - testDistBufLimit), 0 + } +} + +func (p *testDistPeer) canQueue() bool { + return true +} + +func (p *testDistPeer) queueSend(f func()) { + f() +} + +func TestRequestDistributor(t *testing.T) { + testRequestDistributor(t, false) +} + +func TestRequestDistributorResend(t *testing.T) { + testRequestDistributor(t, true) +} + +func testRequestDistributor(t *testing.T, resend bool) { + stop := make(chan struct{}) + defer close(stop) + + var peers [testDistPeerCount]*testDistPeer + for i, _ := range peers { + peers[i] = &testDistPeer{} + go peers[i].worker(t, !resend, stop) + } + + dist := newRequestDistributor(func() map[distPeer]struct{} { + m := make(map[distPeer]struct{}) + for _, peer := range peers { + m[peer] = struct{}{} + } + return m + }, stop) + + var wg sync.WaitGroup + + for i := 1; i <= testDistReqCount; i++ { + cost := uint64(rand.Int63n(testDistMaxCost)) + procTime := uint64(rand.Int63n(int64(cost + 1))) + rq := &testDistReq{ + cost: cost, + procTime: procTime, + order: uint64(i), + canSendTo: make(map[*testDistPeer]struct{}), + } + for _, peer := range peers { + if rand.Intn(2) != 0 { + rq.canSendTo[peer] = struct{}{} + } + } + + wg.Add(1) + req := &distReq{ + getCost: rq.getCost, + canSend: rq.canSend, + request: rq.request, + } + chn := dist.queue(req) + go func() { + cnt := 1 + if resend && len(rq.canSendTo) != 0 { + cnt = rand.Intn(testDistMaxResendCount) + 1 + } + for i := 0; i < cnt; i++ { + if i != 0 { + chn = dist.queue(req) + } + p := <-chn + if p == nil { + if len(rq.canSendTo) != 0 { + t.Errorf("Request that could have been sent was dropped") + } + } else { + peer := p.(*testDistPeer) + if _, ok := rq.canSendTo[peer]; !ok { + t.Errorf("Request sent to wrong peer") + } + } + } + wg.Done() + }() + if rand.Intn(1000) == 0 { + time.Sleep(time.Duration(rand.Intn(5000000))) + } + } + + wg.Wait() +} diff --git a/les/execqueue.go b/les/execqueue.go new file mode 100644 index 000000000..ac779003b --- /dev/null +++ b/les/execqueue.go @@ -0,0 +1,71 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package les + +import ( + "sync/atomic" +) + +// ExecQueue implements a queue that executes function calls in a single thread, +// in the same order as they have been queued. +type execQueue struct { + chn chan func() + cnt, stop, capacity int32 +} + +// NewExecQueue creates a new execution queue. +func newExecQueue(capacity int32) *execQueue { + q := &execQueue{ + chn: make(chan func(), capacity), + capacity: capacity, + } + go q.loop() + return q +} + +func (q *execQueue) loop() { + for f := range q.chn { + atomic.AddInt32(&q.cnt, -1) + if atomic.LoadInt32(&q.stop) != 0 { + return + } + f() + } +} + +// CanQueue returns true if more function calls can be added to the execution queue. +func (q *execQueue) canQueue() bool { + return atomic.LoadInt32(&q.stop) == 0 && atomic.LoadInt32(&q.cnt) < q.capacity +} + +// Queue adds a function call to the execution queue. Returns true if successful. +func (q *execQueue) queue(f func()) bool { + if atomic.LoadInt32(&q.stop) != 0 { + return false + } + if atomic.AddInt32(&q.cnt, 1) > q.capacity { + atomic.AddInt32(&q.cnt, -1) + return false + } + q.chn <- f + return true +} + +// Stop stops the exec queue. +func (q *execQueue) quit() { + atomic.StoreInt32(&q.stop, 1) +} diff --git a/les/fetcher.go b/les/fetcher.go index f9e517d25..353e91932 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -135,35 +135,38 @@ func (f *lightFetcher) syncLoop() { f.lock.Lock() s := requesting requesting = false + var ( + rq *distReq + reqID uint64 + ) if !f.syncing && !(newAnnounce && s) { - reqID := getNextReqID() - if peer, node, amount, retry := f.nextRequest(reqID); node != nil { - requesting = true - if reqID, ok := f.request(peer, reqID, node, amount); ok { - go func() { - time.Sleep(softRequestTimeout) - f.reqMu.Lock() - req, ok := f.requested[reqID] - if ok { - req.timeout = true - f.requested[reqID] = req - } - f.reqMu.Unlock() - // keep starting new requests while possible - f.requestChn <- false - }() - } - } else { - if retry { - requesting = true - go func() { - time.Sleep(time.Millisecond * 100) - f.requestChn <- false - }() - } - } + rq, reqID = f.nextRequest() } + syncing := f.syncing f.lock.Unlock() + + if rq != nil { + requesting = true + _, ok := <-f.pm.reqDist.queue(rq) + if !ok { + f.requestChn <- false + } + + if !syncing { + go func() { + time.Sleep(softRequestTimeout) + f.reqMu.Lock() + req, ok := f.requested[reqID] + if ok { + req.timeout = true + f.requested[reqID] = req + } + f.reqMu.Unlock() + // keep starting new requests while possible + f.requestChn <- false + }() + } + } case reqID := <-f.timeoutChn: f.reqMu.Lock() req, ok := f.requested[reqID] @@ -334,6 +337,12 @@ func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bo f.lock.Lock() defer f.lock.Unlock() + if f.syncing { + // always return true when syncing + // false positives are acceptable, a more sophisticated condition can be implemented later + return true + } + fp := f.peers[p] if fp == nil || fp.root == nil { return false @@ -346,43 +355,13 @@ func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bo f.chain.LockChain() defer f.chain.UnlockChain() // if it's older than the peer's block tree root but it's in the same canonical chain - // than the root, we can still be sure the peer knows it + // as the root, we can still be sure the peer knows it + // + // when syncing, just check if it is part of the known chain, there is nothing better we + // can do since we do not know the most recent block hash yet return core.GetCanonicalHash(f.pm.chainDb, fp.root.number) == fp.root.hash && core.GetCanonicalHash(f.pm.chainDb, number) == hash } -// request initiates a header download request from a certain peer -func (f *lightFetcher) request(p *peer, reqID uint64, n *fetcherTreeNode, amount uint64) (uint64, bool) { - fp := f.peers[p] - if fp == nil { - p.Log().Debug("Requesting from unknown peer") - p.fcServer.DeassignRequest(reqID) - return 0, false - } - if fp.bestConfirmed == nil || fp.root == nil || !f.checkKnownNode(p, fp.root) { - f.syncing = true - go func() { - p.Log().Debug("Synchronisation started") - f.pm.synchronise(p) - f.syncDone <- p - }() - p.fcServer.DeassignRequest(reqID) - return 0, false - } - - n.requested = true - cost := p.GetRequestCost(GetBlockHeadersMsg, int(amount)) - p.fcServer.SendRequest(reqID, cost) - f.reqMu.Lock() - f.requested[reqID] = fetchRequest{hash: n.hash, amount: amount, peer: p, sent: mclock.Now()} - f.reqMu.Unlock() - go p.RequestHeadersByHash(reqID, cost, n.hash, int(amount), 0, true) - go func() { - time.Sleep(hardRequestTimeout) - f.timeoutChn <- reqID - }() - return reqID, true -} - // requestAmount calculates the amount of headers to be downloaded starting // from a certain head backwards func (f *lightFetcher) requestAmount(p *peer, n *fetcherTreeNode) uint64 { @@ -408,12 +387,13 @@ func (f *lightFetcher) requestedID(reqID uint64) bool { // nextRequest selects the peer and announced head to be requested next, amount // to be downloaded starting from the head backwards is also returned -func (f *lightFetcher) nextRequest(reqID uint64) (*peer, *fetcherTreeNode, uint64, bool) { +func (f *lightFetcher) nextRequest() (*distReq, uint64) { var ( bestHash common.Hash bestAmount uint64 ) bestTd := f.maxConfirmedTd + bestSyncing := false for p, fp := range f.peers { for hash, n := range fp.nodeByHash { @@ -423,29 +403,83 @@ func (f *lightFetcher) nextRequest(reqID uint64) (*peer, *fetcherTreeNode, uint6 bestHash = hash bestAmount = amount bestTd = n.td + bestSyncing = fp.bestConfirmed == nil || fp.root == nil || !f.checkKnownNode(p, fp.root) } } } } if bestTd == f.maxConfirmedTd { - return nil, nil, 0, false - } - - peer, _, locked := f.pm.serverPool.selectPeer(reqID, func(p *peer) (bool, time.Duration) { - fp := f.peers[p] - if fp == nil || fp.nodeByHash[bestHash] == nil { - return false, 0 + return nil, 0 + } + + f.syncing = bestSyncing + + var rq *distReq + reqID := getNextReqID() + if f.syncing { + rq = &distReq{ + getCost: func(dp distPeer) uint64 { + return 0 + }, + canSend: func(dp distPeer) bool { + p := dp.(*peer) + fp := f.peers[p] + return fp != nil && fp.nodeByHash[bestHash] != nil + }, + request: func(dp distPeer) func() { + go func() { + p := dp.(*peer) + p.Log().Debug("Synchronisation started") + f.pm.synchronise(p) + f.syncDone <- p + }() + return nil + }, + } + } else { + rq = &distReq{ + getCost: func(dp distPeer) uint64 { + p := dp.(*peer) + return p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) + }, + canSend: func(dp distPeer) bool { + p := dp.(*peer) + f.lock.Lock() + defer f.lock.Unlock() + + fp := f.peers[p] + if fp == nil { + return false + } + n := fp.nodeByHash[bestHash] + return n != nil && !n.requested + }, + request: func(dp distPeer) func() { + p := dp.(*peer) + f.lock.Lock() + fp := f.peers[p] + if fp != nil { + n := fp.nodeByHash[bestHash] + if n != nil { + n.requested = true + } + } + f.lock.Unlock() + + cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) + p.fcServer.QueueRequest(reqID, cost) + f.reqMu.Lock() + f.requested[reqID] = fetchRequest{hash: bestHash, amount: bestAmount, peer: p, sent: mclock.Now()} + f.reqMu.Unlock() + go func() { + time.Sleep(hardRequestTimeout) + f.timeoutChn <- reqID + }() + return func() { p.RequestHeadersByHash(reqID, cost, bestHash, int(bestAmount), 0, true) } + }, } - return true, p.fcServer.CanSend(p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount))) - }) - if !locked { - return nil, nil, 0, true - } - var node *fetcherTreeNode - if peer != nil { - node = f.peers[peer].nodeByHash[bestHash] } - return peer, node, bestAmount, false + return rq, reqID } // deliverHeaders delivers header download request responses for processing diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index e45537cf5..e40e69346 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -94,14 +94,12 @@ func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) { } type ServerNode struct { - bufEstimate uint64 - lastTime mclock.AbsTime - params *ServerParams - sumCost uint64 // sum of req costs sent to this server - pending map[uint64]uint64 // value = sumCost after sending the given req - assignedRequest uint64 // when != 0, only the request with the given ID can be sent to this peer - assignToken chan struct{} // send to this channel before assigning, read from it after deassigning - lock sync.RWMutex + bufEstimate uint64 + lastTime mclock.AbsTime + params *ServerParams + sumCost uint64 // sum of req costs sent to this server + pending map[uint64]uint64 // value = sumCost after sending the given req + lock sync.RWMutex } func NewServerNode(params *ServerParams) *ServerNode { @@ -110,7 +108,6 @@ func NewServerNode(params *ServerParams) *ServerNode { lastTime: mclock.Now(), params: params, pending: make(map[uint64]uint64), - assignToken: make(chan struct{}, 1), } } @@ -127,94 +124,37 @@ func (peer *ServerNode) recalcBLE(time mclock.AbsTime) { } // safetyMargin is added to the flow control waiting time when estimated buffer value is low -const safetyMargin = time.Millisecond * 200 +const safetyMargin = time.Millisecond -func (peer *ServerNode) canSend(maxCost uint64) time.Duration { +func (peer *ServerNode) canSend(maxCost uint64) (time.Duration, float64) { + peer.recalcBLE(mclock.Now()) maxCost += uint64(safetyMargin) * peer.params.MinRecharge / uint64(fcTimeConst) if maxCost > peer.params.BufLimit { maxCost = peer.params.BufLimit } if peer.bufEstimate >= maxCost { - return 0 + return 0, float64(peer.bufEstimate-maxCost) / float64(peer.params.BufLimit) } - return time.Duration((maxCost - peer.bufEstimate) * uint64(fcTimeConst) / peer.params.MinRecharge) + return time.Duration((maxCost - peer.bufEstimate) * uint64(fcTimeConst) / peer.params.MinRecharge), 0 } // CanSend returns the minimum waiting time required before sending a request -// with the given maximum estimated cost -func (peer *ServerNode) CanSend(maxCost uint64) time.Duration { +// with the given maximum estimated cost. Second return value is the relative +// estimated buffer level after sending the request (divided by BufLimit). +func (peer *ServerNode) CanSend(maxCost uint64) (time.Duration, float64) { peer.lock.RLock() defer peer.lock.RUnlock() return peer.canSend(maxCost) } -// AssignRequest tries to assign the server node to the given request, guaranteeing -// that once it returns true, no request will be sent to the node before this one -func (peer *ServerNode) AssignRequest(reqID uint64) bool { - select { - case peer.assignToken <- struct{}{}: - default: - return false - } - peer.lock.Lock() - peer.assignedRequest = reqID - peer.lock.Unlock() - return true -} - -// MustAssignRequest waits until the node can be assigned to the given request. -// It is always guaranteed that assignments are released in a short amount of time. -func (peer *ServerNode) MustAssignRequest(reqID uint64) { - peer.assignToken <- struct{}{} - peer.lock.Lock() - peer.assignedRequest = reqID - peer.lock.Unlock() -} - -// DeassignRequest releases a request assignment in case the planned request -// is not being sent. -func (peer *ServerNode) DeassignRequest(reqID uint64) { - peer.lock.Lock() - if peer.assignedRequest == reqID { - peer.assignedRequest = 0 - <-peer.assignToken - } - peer.lock.Unlock() -} - -// IsAssigned returns true if the server node has already been assigned to a request -// (note that this function returning false does not guarantee that you can assign a request -// immediately afterwards, its only purpose is to help peer selection) -func (peer *ServerNode) IsAssigned() bool { - peer.lock.RLock() - locked := peer.assignedRequest != 0 - peer.lock.RUnlock() - return locked -} - -// blocks until request can be sent -func (peer *ServerNode) SendRequest(reqID, maxCost uint64) { +// QueueRequest should be called when the request has been assigned to the given +// server node, before putting it in the send queue. It is mandatory that requests +// are sent in the same order as the QueueRequest calls are made. +func (peer *ServerNode) QueueRequest(reqID, maxCost uint64) { peer.lock.Lock() defer peer.lock.Unlock() - if peer.assignedRequest != reqID { - peer.lock.Unlock() - peer.MustAssignRequest(reqID) - peer.lock.Lock() - } - - peer.recalcBLE(mclock.Now()) - wait := peer.canSend(maxCost) - for wait > 0 { - peer.lock.Unlock() - time.Sleep(wait) - peer.lock.Lock() - peer.recalcBLE(mclock.Now()) - wait = peer.canSend(maxCost) - } - peer.assignedRequest = 0 - <-peer.assignToken peer.bufEstimate -= maxCost peer.sumCost += maxCost if reqID >= 0 { @@ -222,6 +162,8 @@ func (peer *ServerNode) SendRequest(reqID, maxCost uint64) { } } +// GotReply adjusts estimated buffer value according to the value included in +// the latest request reply. func (peer *ServerNode) GotReply(reqID, bv uint64) { peer.lock.Lock() @@ -235,6 +177,10 @@ func (peer *ServerNode) GotReply(reqID, bv uint64) { return } delete(peer.pending, reqID) - peer.bufEstimate = bv - (peer.sumCost - sc) + cc := peer.sumCost - sc + peer.bufEstimate = 0 + if bv > cc { + peer.bufEstimate = bv - cc + } peer.lastTime = mclock.Now() } diff --git a/les/handler.go b/les/handler.go index 4271da8b8..ece2060ee 100644 --- a/les/handler.go +++ b/les/handler.go @@ -102,6 +102,7 @@ type ProtocolManager struct { odr *LesOdr server *LesServer serverPool *serverPool + reqDist *requestDistributor downloader *downloader.Downloader fetcher *lightFetcher @@ -203,8 +204,17 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, network blockchain.InsertHeaderChain, nil, nil, blockchain.Rollback, removePeer) } + manager.reqDist = newRequestDistributor(func() map[distPeer]struct{} { + m := make(map[distPeer]struct{}) + peers := manager.peers.AllPeers() + for _, peer := range peers { + m[peer] = struct{}{} + } + return m + }, manager.quitSync) if odr != nil { odr.removePeer = removePeer + odr.reqDist = manager.reqDist } /*validator := func(block *types.Block, parent *types.Block) error { @@ -334,17 +344,49 @@ func (pm *ProtocolManager) handle(p *peer) error { if pm.lightSync { requestHeadersByHash := func(origin common.Hash, amount int, skip int, reverse bool) error { reqID := getNextReqID() - cost := p.GetRequestCost(GetBlockHeadersMsg, amount) - p.fcServer.MustAssignRequest(reqID) - p.fcServer.SendRequest(reqID, cost) - return p.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse) + rq := &distReq{ + getCost: func(dp distPeer) uint64 { + peer := dp.(*peer) + return peer.GetRequestCost(GetBlockHeadersMsg, amount) + }, + canSend: func(dp distPeer) bool { + return dp.(*peer) == p + }, + request: func(dp distPeer) func() { + peer := dp.(*peer) + cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) + peer.fcServer.QueueRequest(reqID, cost) + return func() { peer.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse) } + }, + } + _, ok := <-pm.reqDist.queue(rq) + if !ok { + return ErrNoPeers + } + return nil } requestHeadersByNumber := func(origin uint64, amount int, skip int, reverse bool) error { reqID := getNextReqID() - cost := p.GetRequestCost(GetBlockHeadersMsg, amount) - p.fcServer.MustAssignRequest(reqID) - p.fcServer.SendRequest(reqID, cost) - return p.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse) + rq := &distReq{ + getCost: func(dp distPeer) uint64 { + peer := dp.(*peer) + return peer.GetRequestCost(GetBlockHeadersMsg, amount) + }, + canSend: func(dp distPeer) bool { + return dp.(*peer) == p + }, + request: func(dp distPeer) func() { + peer := dp.(*peer) + cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) + peer.fcServer.QueueRequest(reqID, cost) + return func() { peer.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse) } + }, + } + _, ok := <-pm.reqDist.queue(rq) + if !ok { + return ErrNoPeers + } + return nil } if err := pm.downloader.RegisterPeer(p.id, ethVersion, p.HeadAndTd, requestHeadersByHash, requestHeadersByNumber, nil, nil, nil); err != nil { @@ -884,7 +926,13 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } if deliverMsg != nil { - return pm.odr.Deliver(p, deliverMsg) + err := pm.odr.Deliver(p, deliverMsg) + if err != nil { + p.responseErrors++ + if p.responseErrors > maxResponseErrors { + return err + } + } } return nil } diff --git a/les/helper_test.go b/les/helper_test.go index f6293ad1a..3e8ce57b6 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -352,11 +352,15 @@ func (p *testServerPool) setPeer(peer *peer) { p.peer = peer } -func (p *testServerPool) selectPeerWait(uint64, func(*peer) (bool, time.Duration), <-chan struct{}) *peer { +func (p *testServerPool) getAllPeers() map[distPeer]struct{} { p.lock.RLock() defer p.lock.RUnlock() - return p.peer + m := make(map[distPeer]struct{}) + if p.peer != nil { + m[p.peer] = struct{}{} + } + return m } func (p *testServerPool) adjustResponseTime(*poolEntry, time.Duration, bool) { diff --git a/les/odr.go b/les/odr.go index afc894ab5..684f36c76 100644 --- a/les/odr.go +++ b/les/odr.go @@ -17,6 +17,7 @@ package les import ( + "context" "crypto/rand" "encoding/binary" "sync" @@ -26,20 +27,17 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" - "golang.org/x/net/context" ) var ( softRequestTimeout = time.Millisecond * 500 hardRequestTimeout = time.Second * 10 - retryPeers = time.Second * 1 ) // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) type odrPeerSelector interface { - selectPeerWait(uint64, func(*peer) (bool, time.Duration), <-chan struct{}) *peer adjustResponseTime(*poolEntry, time.Duration, bool) } @@ -51,6 +49,7 @@ type LesOdr struct { mlock, clock sync.Mutex sentReqs map[uint64]*sentReq serverPool odrPeerSelector + reqDist *requestDistributor } func NewLesOdr(db ethdb.Database) *LesOdr { @@ -165,69 +164,81 @@ func (self *LesOdr) requestPeer(req *sentReq, peer *peer, delivered, timeout cha func (self *LesOdr) networkRequest(ctx context.Context, lreq LesOdrRequest) error { answered := make(chan struct{}) req := &sentReq{ - valFunc: lreq.Valid, + valFunc: lreq.Validate, sentTo: make(map[*peer]chan struct{}), answered: answered, // reply delivered by any peer } - reqID := getNextReqID() - self.mlock.Lock() - self.sentReqs[reqID] = req - self.mlock.Unlock() + + exclude := make(map[*peer]struct{}) reqWg := new(sync.WaitGroup) reqWg.Add(1) defer reqWg.Done() - go func() { - reqWg.Wait() - self.mlock.Lock() - delete(self.sentReqs, reqID) - self.mlock.Unlock() - }() - exclude := make(map[*peer]struct{}) - for { - var p *peer - if self.serverPool != nil { - p = self.serverPool.selectPeerWait(reqID, func(p *peer) (bool, time.Duration) { - if _, ok := exclude[p]; ok || !lreq.CanSend(p) { - return false, 0 - } - return true, p.fcServer.CanSend(lreq.GetCost(p)) - }, ctx.Done()) - } - if p == nil { - select { - case <-ctx.Done(): - return ctx.Err() - case <-req.answered: - return nil - case <-time.After(retryPeers): - } - } else { + var timeout chan struct{} + reqID := getNextReqID() + rq := &distReq{ + getCost: func(dp distPeer) uint64 { + return lreq.GetCost(dp.(*peer)) + }, + canSend: func(dp distPeer) bool { + p := dp.(*peer) + _, ok := exclude[p] + return !ok && lreq.CanSend(p) + }, + request: func(dp distPeer) func() { + p := dp.(*peer) exclude[p] = struct{}{} delivered := make(chan struct{}) - timeout := make(chan struct{}) + timeout = make(chan struct{}) req.lock.Lock() req.sentTo[p] = delivered req.lock.Unlock() reqWg.Add(1) cost := lreq.GetCost(p) - p.fcServer.SendRequest(reqID, cost) + p.fcServer.QueueRequest(reqID, cost) go self.requestPeer(req, p, delivered, timeout, reqWg) - lreq.Request(reqID, p) - - select { - case <-ctx.Done(): - return ctx.Err() - case <-answered: - return nil - case <-timeout: + return func() { lreq.Request(reqID, p) } + }, + } + + self.mlock.Lock() + self.sentReqs[reqID] = req + self.mlock.Unlock() + + go func() { + reqWg.Wait() + self.mlock.Lock() + delete(self.sentReqs, reqID) + self.mlock.Unlock() + }() + + for { + peerChn := self.reqDist.queue(rq) + select { + case <-ctx.Done(): + self.reqDist.cancel(rq) + return ctx.Err() + case <-answered: + self.reqDist.cancel(rq) + return nil + case _, ok := <-peerChn: + if !ok { + return ErrNoPeers } } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-answered: + return nil + case <-timeout: + } } } -// Retrieve tries to fetch an object from the local db, then from the LES network. +// Retrieve tries to fetch an object from the LES network. // If the network retrieval was successful, it stores the object in local db. func (self *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err error) { lreq := LesRequest(req) diff --git a/les/odr_requests.go b/les/odr_requests.go index 53aced93c..1f853b341 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -49,7 +49,7 @@ type LesOdrRequest interface { GetCost(*peer) uint64 CanSend(*peer) bool Request(uint64, *peer) error - Valid(ethdb.Database, *Msg) error // if true, keeps the retrieved object + Validate(ethdb.Database, *Msg) error } func LesRequest(req light.OdrRequest) LesOdrRequest { @@ -92,7 +92,7 @@ func (r *BlockRequest) Request(reqID uint64, peer *peer) error { // Valid processes an ODR request reply message from the LES network // returns true and stores results in memory if the message was a valid reply // to the request (implementation of LesOdrRequest) -func (r *BlockRequest) Valid(db ethdb.Database, msg *Msg) error { +func (r *BlockRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating block body", "hash", r.Hash) // Ensure we have a correct message with a single block body @@ -148,7 +148,7 @@ func (r *ReceiptsRequest) Request(reqID uint64, peer *peer) error { // Valid processes an ODR request reply message from the LES network // returns true and stores results in memory if the message was a valid reply // to the request (implementation of LesOdrRequest) -func (r *ReceiptsRequest) Valid(db ethdb.Database, msg *Msg) error { +func (r *ReceiptsRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating block receipts", "hash", r.Hash) // Ensure we have a correct message with a single block receipt @@ -208,7 +208,7 @@ func (r *TrieRequest) Request(reqID uint64, peer *peer) error { // Valid processes an ODR request reply message from the LES network // returns true and stores results in memory if the message was a valid reply // to the request (implementation of LesOdrRequest) -func (r *TrieRequest) Valid(db ethdb.Database, msg *Msg) error { +func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating trie proof", "root", r.Id.Root, "key", r.Key) // Ensure we have a correct message with a single proof @@ -259,7 +259,7 @@ func (r *CodeRequest) Request(reqID uint64, peer *peer) error { // Valid processes an ODR request reply message from the LES network // returns true and stores results in memory if the message was a valid reply // to the request (implementation of LesOdrRequest) -func (r *CodeRequest) Valid(db ethdb.Database, msg *Msg) error { +func (r *CodeRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating code data", "hash", r.Hash) // Ensure we have a correct message with a single code element @@ -319,7 +319,7 @@ func (r *ChtRequest) Request(reqID uint64, peer *peer) error { // Valid processes an ODR request reply message from the LES network // returns true and stores results in memory if the message was a valid reply // to the request (implementation of LesOdrRequest) -func (r *ChtRequest) Valid(db ethdb.Database, msg *Msg) error { +func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating CHT", "cht", r.ChtNum, "block", r.BlockNum) // Ensure we have a correct message with a single proof element diff --git a/les/odr_test.go b/les/odr_test.go index 4f1fccb24..6b074f1a2 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -18,6 +18,7 @@ package les import ( "bytes" + "context" "math/big" "testing" "time" @@ -32,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/net/context" ) type odrTestFn func(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte @@ -162,8 +162,11 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { lpm, ldb, odr := newTestProtocolManagerMust(t, true, 0, nil) _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) pool := &testServerPool{} + lpm.reqDist = newRequestDistributor(pool.getAllPeers, lpm.quitSync) + odr.reqDist = lpm.reqDist pool.setPeer(lpeer) odr.serverPool = pool + lpeer.hasBlock = func(common.Hash, uint64) bool { return true } select { case <-time.After(time.Millisecond * 100): case err := <-err1: @@ -178,8 +181,11 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { bhash := core.GetCanonicalHash(db, i) b1 := fn(light.NoOdr, db, pm.chainConfig, pm.blockchain.(*core.BlockChain), nil, bhash) - ctx, _ := context.WithTimeout(context.Background(), 200*time.Millisecond) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() b2 := fn(ctx, ldb, lpm.chainConfig, nil, lpm.blockchain.(*light.LightChain), bhash) + eq := bytes.Equal(b1, b2) exp := i < expFail if exp && !eq { diff --git a/les/peer.go b/les/peer.go index ef5f8a6ce..4793da296 100644 --- a/les/peer.go +++ b/les/peer.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -37,7 +38,10 @@ var ( errNotRegistered = errors.New("peer is not registered") ) -const maxHeadInfoLen = 20 +const ( + maxHeadInfoLen = 20 + maxResponseErrors = 50 // number of invalid responses tolerated (makes the protocol less brittle but still avoids spam) +) type peer struct { *p2p.Peer @@ -53,9 +57,11 @@ type peer struct { lock sync.RWMutex announceChn chan announceData + sendQueue *execQueue - poolEntry *poolEntry - hasBlock func(common.Hash, uint64) bool + poolEntry *poolEntry + hasBlock func(common.Hash, uint64) bool + responseErrors int fcClient *flowcontrol.ClientNode // nil if the peer is server only fcServer *flowcontrol.ServerNode // nil if the peer is client only @@ -76,6 +82,14 @@ func newPeer(version, network int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { } } +func (p *peer) canQueue() bool { + return p.sendQueue.canQueue() +} + +func (p *peer) queueSend(f func()) { + p.sendQueue.queue(f) +} + // Info gathers and returns a collection of metadata known about a peer. func (p *peer) Info() *eth.PeerInfo { return ð.PeerInfo{ @@ -117,6 +131,11 @@ func (p *peer) Td() *big.Int { return new(big.Int).Set(p.headInfo.Td) } +// waitBefore implements distPeer interface +func (p *peer) waitBefore(maxCost uint64) (time.Duration, float64) { + return p.fcServer.CanSend(maxCost) +} + func sendRequest(w p2p.MsgWriter, msgcode, reqID, cost uint64, data interface{}) error { type req struct { ReqID uint64 @@ -237,11 +256,8 @@ func (p *peer) RequestHeaderProofs(reqID, cost uint64, reqs []*ChtReq) error { return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqs) } -func (p *peer) SendTxs(cost uint64, txs types.Transactions) error { +func (p *peer) SendTxs(reqID, cost uint64, txs types.Transactions) error { p.Log().Debug("Fetching batch of transactions", "count", len(txs)) - reqID := getNextReqID() - p.fcServer.MustAssignRequest(reqID) - p.fcServer.SendRequest(reqID, cost) return p2p.Send(p.rw, SendTxMsg, txs) } @@ -444,6 +460,7 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p + p.sendQueue = newExecQueue(100) return nil } @@ -453,8 +470,10 @@ func (ps *peerSet) Unregister(id string) error { ps.lock.Lock() defer ps.lock.Unlock() - if _, ok := ps.peers[id]; !ok { + if p, ok := ps.peers[id]; !ok { return errNotRegistered + } else { + p.sendQueue.quit() } delete(ps.peers, id) return nil diff --git a/les/request_test.go b/les/request_test.go index 10e9edf8b..ba1fc15bd 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -17,6 +17,7 @@ package les import ( + "context" "testing" "time" @@ -25,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" - "golang.org/x/net/context" ) var testBankSecureTrieKey = secAddr(testBankAddress) @@ -72,8 +72,11 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { lpm, ldb, odr := newTestProtocolManagerMust(t, true, 0, nil) _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) pool := &testServerPool{} + lpm.reqDist = newRequestDistributor(pool.getAllPeers, lpm.quitSync) + odr.reqDist = lpm.reqDist pool.setPeer(lpeer) odr.serverPool = pool + lpeer.hasBlock = func(common.Hash, uint64) bool { return true } select { case <-time.After(time.Millisecond * 100): case err := <-err1: @@ -88,7 +91,9 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { bhash := core.GetCanonicalHash(db, i) if req := fn(ldb, bhash, i); req != nil { - ctx, _ := context.WithTimeout(context.Background(), 200*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + err := odr.Retrieve(ctx, req) got := err == nil exp := i < expFail diff --git a/les/serverpool.go b/les/serverpool.go index 55d481dbf..64fe991c6 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -268,82 +268,6 @@ func (pool *serverPool) adjustResponseTime(entry *poolEntry, time time.Duration, } } -type selectPeerItem struct { - peer *peer - weight int64 - wait time.Duration -} - -func (sp selectPeerItem) Weight() int64 { - return sp.weight -} - -// selectPeer selects a suitable peer for a request, also returning a necessary waiting time to perform the request -// and a "locked" flag meaning that the request has been assigned to the given peer and its execution is guaranteed -// after the given waiting time. If locked flag is false, selectPeer should be called again after the waiting time. -func (pool *serverPool) selectPeer(reqID uint64, canSend func(*peer) (bool, time.Duration)) (*peer, time.Duration, bool) { - pool.lock.Lock() - type selectPeer struct { - peer *peer - rstat, tstat float64 - } - var list []selectPeer - sel := newWeightedRandomSelect() - for _, entry := range pool.entries { - if entry.state == psRegistered { - if !entry.peer.fcServer.IsAssigned() { - list = append(list, selectPeer{entry.peer, entry.responseStats.recentAvg(), entry.timeoutStats.recentAvg()}) - } - } - } - pool.lock.Unlock() - - for _, sp := range list { - ok, wait := canSend(sp.peer) - if ok { - w := int64(1000000000 * (peerSelectMinWeight + math.Exp(-(sp.rstat+float64(wait))/float64(responseScoreTC))*math.Pow((1-sp.tstat), timeoutPow))) - sel.update(selectPeerItem{peer: sp.peer, weight: w, wait: wait}) - } - } - choice := sel.choose() - if choice == nil { - return nil, 0, false - } - peer, wait := choice.(selectPeerItem).peer, choice.(selectPeerItem).wait - locked := false - if wait < time.Millisecond*100 { - if peer.fcServer.AssignRequest(reqID) { - ok, w := canSend(peer) - wait = time.Duration(w) - if ok && wait < time.Millisecond*100 { - locked = true - } else { - peer.fcServer.DeassignRequest(reqID) - wait = time.Millisecond * 100 - } - } - } else { - wait = time.Millisecond * 100 - } - return peer, wait, locked -} - -// selectPeer selects a suitable peer for a request, waiting until an assignment to -// the request is guaranteed or the process is aborted. -func (pool *serverPool) selectPeerWait(reqID uint64, canSend func(*peer) (bool, time.Duration), abort <-chan struct{}) *peer { - for { - peer, wait, locked := pool.selectPeer(reqID, canSend) - if locked { - return peer - } - select { - case <-abort: - return nil - case <-time.After(wait): - } - } -} - // eventLoop handles pool events and mutex locking for all internal functions func (pool *serverPool) eventLoop() { lookupCnt := 0 diff --git a/les/sync.go b/les/sync.go index c143cb145..c0e17f97d 100644 --- a/les/sync.go +++ b/les/sync.go @@ -17,12 +17,12 @@ package les import ( + "context" "time" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/light" - "golang.org/x/net/context" ) const ( @@ -77,8 +77,8 @@ func (pm *ProtocolManager) synchronise(peer *peer) { return } - ctx, _ := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() pm.blockchain.(*light.LightChain).SyncCht(ctx) - pm.downloader.Synchronise(peer.id, peer.Head(), peer.Td(), downloader.LightSync) } diff --git a/les/txrelay.go b/les/txrelay.go index 76d416c57..1ca3467e4 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -35,13 +35,14 @@ type LesTxRelay struct { peerList []*peer peerStartPos int lock sync.RWMutex + + reqDist *requestDistributor } func NewLesTxRelay() *LesTxRelay { return &LesTxRelay{ txSent: make(map[common.Hash]*ltrInfo), txPending: make(map[common.Hash]struct{}), - ps: newPeerSet(), } } @@ -108,10 +109,26 @@ func (self *LesTxRelay) send(txs types.Transactions, count int) { } for p, list := range sendTo { - cost := p.GetRequestCost(SendTxMsg, len(list)) - go func(p *peer, list types.Transactions, cost uint64) { - p.SendTxs(cost, list) - }(p, list, cost) + pp := p + ll := list + + reqID := getNextReqID() + rq := &distReq{ + getCost: func(dp distPeer) uint64 { + peer := dp.(*peer) + return peer.GetRequestCost(SendTxMsg, len(ll)) + }, + canSend: func(dp distPeer) bool { + return dp.(*peer) == pp + }, + request: func(dp distPeer) func() { + peer := dp.(*peer) + cost := peer.GetRequestCost(SendTxMsg, len(ll)) + peer.fcServer.QueueRequest(reqID, cost) + return func() { peer.SendTxs(reqID, cost, ll) } + }, + } + self.reqDist.queue(rq) } } diff --git a/light/lightchain.go b/light/lightchain.go index 4370dc0fc..82b7a5866 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -17,9 +17,11 @@ package light import ( + "context" "math/big" "sync" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -31,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/pow" "github.com/ethereum/go-ethereum/rlp" "github.com/hashicorp/golang-lru" - "golang.org/x/net/context" ) var ( @@ -369,9 +370,17 @@ func (self *LightChain) postChainEvents(events []interface{}) { // In the case of a light chain, InsertHeaderChain also creates and posts light // chain events when necessary. func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { + start := time.Now() + if i, err := self.hc.ValidateHeaderChain(chain, checkFreq); err != nil { + return i, err + } + // Make sure only one thread manipulates the chain at once self.chainmu.Lock() - defer self.chainmu.Unlock() + defer func() { + self.chainmu.Unlock() + time.Sleep(time.Millisecond * 10) // ugly hack; do not hog chain lock in case syncing is CPU-limited by validation + }() self.wg.Add(1) defer self.wg.Done() @@ -397,7 +406,7 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) } return err } - i, err := self.hc.InsertHeaderChain(chain, checkFreq, whFunc) + i, err := self.hc.InsertHeaderChain(chain, whFunc, start) go self.postChainEvents(events) return i, err } diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 8a99c69f1..7460fd1a3 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -17,6 +17,7 @@ package light import ( + "context" "fmt" "math/big" "runtime" @@ -30,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/pow" "github.com/hashicorp/golang-lru" - "golang.org/x/net/context" ) // So we can deterministically seed different blockchains diff --git a/light/odr.go b/light/odr.go index 4f6ef6b9e..ca6364f28 100644 --- a/light/odr.go +++ b/light/odr.go @@ -19,6 +19,7 @@ package light import ( + "context" "math/big" "github.com/ethereum/go-ethereum/common" @@ -27,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/net/context" ) // NoOdr is the default context passed to an ODR capable function when the ODR diff --git a/light/odr_test.go b/light/odr_test.go index e2eced346..ba82ec04f 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -18,6 +18,7 @@ package light import ( "bytes" + "context" "errors" "math/big" "testing" @@ -36,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/pow" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "golang.org/x/net/context" ) var ( @@ -277,8 +277,11 @@ func testChainOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { for i := uint64(0); i <= blockchain.CurrentHeader().Number.Uint64(); i++ { bhash := core.GetCanonicalHash(sdb, i) b1 := fn(NoOdr, sdb, blockchain, nil, bhash) - ctx, _ := context.WithTimeout(context.Background(), 200*time.Millisecond) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() b2 := fn(ctx, ldb, nil, lightchain, bhash) + eq := bytes.Equal(b1, b2) exp := i < expFail if exp && !eq { diff --git a/light/odr_util.go b/light/odr_util.go index 17e9aadcb..d7f8458f1 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -18,6 +18,7 @@ package light import ( "bytes" + "context" "errors" "math/big" @@ -27,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/net/context" ) var sha3_nil = crypto.Keccak256Hash(nil) diff --git a/light/state.go b/light/state.go index d3e047ef4..b184dc3a5 100644 --- a/light/state.go +++ b/light/state.go @@ -17,11 +17,11 @@ package light import ( + "context" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "golang.org/x/net/context" ) // LightState is a memory representation of a state. diff --git a/light/state_object.go b/light/state_object.go index f33ba217e..a54ea1d9f 100644 --- a/light/state_object.go +++ b/light/state_object.go @@ -18,13 +18,13 @@ package light import ( "bytes" + "context" "fmt" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/net/context" ) var emptyCodeHash = crypto.Keccak256(nil) diff --git a/light/state_test.go b/light/state_test.go index d594ab9ff..e776efec8 100644 --- a/light/state_test.go +++ b/light/state_test.go @@ -18,6 +18,7 @@ package light import ( "bytes" + "context" "math/big" "testing" @@ -26,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "golang.org/x/net/context" ) func makeTestState() (common.Hash, ethdb.Database) { diff --git a/light/trie.go b/light/trie.go index c5525358a..1440f2fbf 100644 --- a/light/trie.go +++ b/light/trie.go @@ -17,9 +17,10 @@ package light import ( + "context" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" - "golang.org/x/net/context" ) // LightTrie is an ODR-capable wrapper around trie.SecureTrie diff --git a/light/txpool.go b/light/txpool.go index 28c8d8ca5..446195806 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -17,6 +17,7 @@ package light import ( + "context" "fmt" "sync" "time" @@ -29,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/net/context" ) // txPermanent is the number of mined blocks after a mined transaction is @@ -230,13 +230,13 @@ func (pool *TxPool) rollbackTxs(hash common.Hash, txc txStateChanges) { } } -// setNewHead sets a new head header, processing (and rolling back if necessary) +// reorgOnNewHead sets a new head header, processing (and rolling back if necessary) // the blocks since the last known head and returns a txStateChanges map containing // the recently mined and rolled back transaction hashes. If an error (context // timeout) occurs during checking new blocks, it leaves the locally known head // at the latest checked block and still returns a valid txStateChanges, making it // possible to continue checking the missing blocks at the next chain head event -func (pool *TxPool) setNewHead(ctx context.Context, newHeader *types.Header) (txStateChanges, error) { +func (pool *TxPool) reorgOnNewHead(ctx context.Context, newHeader *types.Header) (txStateChanges, error) { txc := make(txStateChanges) oldh := pool.chain.GetHeaderByHash(pool.head) newh := newHeader @@ -276,15 +276,17 @@ func (pool *TxPool) setNewHead(ctx context.Context, newHeader *types.Header) (tx // clear old mined tx entries of old blocks if idx := newHeader.Number.Uint64(); idx > pool.clearIdx+txPermanent { idx2 := idx - txPermanent - for i := pool.clearIdx; i < idx2; i++ { - hash := core.GetCanonicalHash(pool.chainDb, i) - if list, ok := pool.mined[hash]; ok { - hashes := make([]common.Hash, len(list)) - for i, tx := range list { - hashes[i] = tx.Hash() + if len(pool.mined) > 0 { + for i := pool.clearIdx; i < idx2; i++ { + hash := core.GetCanonicalHash(pool.chainDb, i) + if list, ok := pool.mined[hash]; ok { + hashes := make([]common.Hash, len(list)) + for i, tx := range list { + hashes[i] = tx.Hash() + } + pool.relay.Discard(hashes) + delete(pool.mined, hash) } - pool.relay.Discard(hashes) - delete(pool.mined, hash) } } pool.clearIdx = idx2 @@ -303,19 +305,28 @@ func (pool *TxPool) eventLoop() { for ev := range pool.events.Chan() { switch ev.Data.(type) { case core.ChainHeadEvent: - pool.mu.Lock() - ctx, _ := context.WithTimeout(context.Background(), blockCheckTimeout) - head := pool.chain.CurrentHeader() - txc, _ := pool.setNewHead(ctx, head) - m, r := txc.getLists() - pool.relay.NewHead(pool.head, m, r) - pool.homestead = pool.config.IsHomestead(head.Number) - pool.signer = types.MakeSigner(pool.config, head.Number) - pool.mu.Unlock() + pool.setNewHead(ev.Data.(core.ChainHeadEvent).Block.Header()) + // hack in order to avoid hogging the lock; this part will + // be replaced by a subsequent PR. + time.Sleep(time.Millisecond) } } } +func (pool *TxPool) setNewHead(head *types.Header) { + pool.mu.Lock() + defer pool.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), blockCheckTimeout) + defer cancel() + + txc, _ := pool.reorgOnNewHead(ctx, head) + m, r := txc.getLists() + pool.relay.NewHead(pool.head, m, r) + pool.homestead = pool.config.IsHomestead(head.Number) + pool.signer = types.MakeSigner(pool.config, head.Number) +} + // Stop stops the light transaction pool func (pool *TxPool) Stop() { close(pool.quit) diff --git a/light/txpool_test.go b/light/txpool_test.go index 980c7c898..e93955511 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -17,6 +17,7 @@ package light import ( + "context" "math" "math/big" "testing" @@ -30,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/pow" - "golang.org/x/net/context" ) type testTxRelay struct { @@ -107,10 +107,11 @@ func TestTxPool(t *testing.T) { lightchain.SetValidator(bproc{}) txPermanent = 50 pool := NewTxPool(testChainConfig(), evmux, lightchain, relay) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() for ii, block := range gchain { i := ii + 1 - ctx, _ := context.WithTimeout(context.Background(), 200*time.Millisecond) s := sentTx(i - 1) e := sentTx(i) for i := s; i < e; i++ { diff --git a/light/vm_env.go b/light/vm_env.go index ebd229de8..54aa12875 100644 --- a/light/vm_env.go +++ b/light/vm_env.go @@ -17,12 +17,12 @@ package light import ( + "context" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "golang.org/x/net/context" ) // VMState is a wrapper for the light state that holds the actual context and diff --git a/mobile/big.go b/mobile/big.go index 9a55836c1..525717caa 100644 --- a/mobile/big.go +++ b/mobile/big.go @@ -93,3 +93,8 @@ func (bi *BigInts) Set(index int, bigint *BigInt) error { bi.bigints[index] = bigint.bigint return nil } + +// GetString returns the value of x as a formatted string in some number base. +func (bi *BigInt) GetString(base int) string { + return bi.bigint.Text(base) +} diff --git a/mobile/big_go1.7.go b/mobile/big_go1.7.go deleted file mode 100644 index 0447e1f66..000000000 --- a/mobile/big_go1.7.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// Contains the wrappers from the math/big package that require Go 1.7 and above. - -// +build go1.7 - -package geth - -// GetString returns the value of x as a formatted string in some number base. -func (bi *BigInt) GetString(base int) string { - return bi.bigint.Text(base) -} diff --git a/mobile/context.go b/mobile/context.go index 9df94b689..f1fff9011 100644 --- a/mobile/context.go +++ b/mobile/context.go @@ -20,9 +20,8 @@ package geth import ( + "context" "time" - - "golang.org/x/net/context" ) // Context carries a deadline, a cancelation signal, and other values across API diff --git a/rlp/decode.go b/rlp/decode.go index c4e5869cc..ee0b7dbcd 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -63,12 +63,16 @@ type Decoder interface { // must contain an element for each decoded field. Decode returns an // error if there are too few or too many elements. // -// The decoding of struct fields honours two struct tags, "tail" and -// "nil". For an explanation of "tail", see the example. -// The "nil" tag applies to pointer-typed fields and changes the -// decoding rules for the field such that input values of size zero -// decode as a nil pointer. This tag can be useful when decoding -// recursive types. +// The decoding of struct fields honours certain struct tags, "tail", +// "nil" and "-". +// +// The "-" tag ignores fields. +// +// For an explanation of "tail", see the example. +// +// The "nil" tag applies to pointer-typed fields and changes the decoding +// rules for the field such that input values of size zero decode as a nil +// pointer. This tag can be useful when decoding recursive types. // // type StructWithEmptyOK struct { // Foo *[20]byte `rlp:"nil"` diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 2d465b74d..d762e195d 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -339,6 +339,12 @@ var ( ) ) +type hasIgnoredField struct { + A uint + B uint `rlp:"-"` + C uint +} + var decodeTests = []decodeTest{ // booleans {input: "01", ptr: new(bool), value: true}, @@ -490,6 +496,13 @@ var decodeTests = []decodeTest{ value: tailRaw{A: 1, Tail: []RawValue{}}, }, + // struct tag "-" + { + input: "C20102", + ptr: new(hasIgnoredField), + value: hasIgnoredField{A: 1, C: 2}, + }, + // RawValue {input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))}, {input: "82FFFF", ptr: new(RawValue), value: RawValue(unhex("82FFFF"))}, diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 6f38294e4..827960f7c 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -218,6 +218,7 @@ var encTests = []encTest{ {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, output: "C20102"}, {val: &tailRaw{A: 1, Tail: []RawValue{}}, output: "C101"}, {val: &tailRaw{A: 1, Tail: nil}, output: "C101"}, + {val: &hasIgnoredField{A: 1, B: 2, C: 3}, output: "C20103"}, // nil {val: (*uint)(nil), output: "80"}, diff --git a/rlp/typecache.go b/rlp/typecache.go index a2f217c66..3df799e1e 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -37,11 +37,12 @@ type typeinfo struct { type tags struct { // rlp:"nil" controls whether empty input results in a nil pointer. nilOK bool - // rlp:"tail" controls whether this field swallows additional list // elements. It can only be set for the last field, which must be // of slice type. tail bool + // rlp:"-" ignores fields. + ignored bool } type typekey struct { @@ -101,6 +102,9 @@ func structFields(typ reflect.Type) (fields []field, err error) { if err != nil { return nil, err } + if tags.ignored { + continue + } info, err := cachedTypeInfo1(f.Type, tags) if err != nil { return nil, err @@ -117,6 +121,8 @@ func parseStructTag(typ reflect.Type, fi int) (tags, error) { for _, t := range strings.Split(f.Tag.Get("rlp"), ",") { switch t = strings.TrimSpace(t); t { case "": + case "-": + ts.ignored = true case "nil": ts.nilOK = true case "tail": diff --git a/rpc/client.go b/rpc/client.go index 78a6fe789..2c35ba54a 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -19,6 +19,7 @@ package rpc import ( "bytes" "container/list" + "context" "encoding/json" "errors" "fmt" @@ -31,7 +32,6 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "golang.org/x/net/context" ) var ( diff --git a/rpc/client_context_go1.4.go b/rpc/client_context_go1.4.go deleted file mode 100644 index ac956a17d..000000000 --- a/rpc/client_context_go1.4.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build !go1.5 - -package rpc - -import ( - "net" - "net/http" - "time" - - "golang.org/x/net/context" -) - -// In older versions of Go (below 1.5), dials cannot be canceled -// via a channel or context. The context deadline can still applied. - -// contextDialer returns a dialer that applies the deadline value from the given context. -func contextDialer(ctx context.Context) *net.Dialer { - dialer := &net.Dialer{KeepAlive: tcpKeepAliveInterval} - if deadline, ok := ctx.Deadline(); ok { - dialer.Deadline = deadline - } else { - dialer.Deadline = time.Now().Add(defaultDialTimeout) - } - return dialer -} - -// dialContext connects to the given address, aborting the dial if ctx is canceled. -func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { - return contextDialer(ctx).Dial(network, addr) -} - -// requestWithContext copies req, adding the cancelation channel and deadline from ctx. -func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { - // Set Timeout on the client if the context has a deadline. - // Note that there is no default timeout (unlike in contextDialer) because - // the timeout applies to the entire request, including reads from body. - if deadline, ok := ctx.Deadline(); ok { - c2 := *c - c2.Timeout = deadline.Sub(time.Now()) - c = &c2 - } - req2 := *req - return c, &req2 -} diff --git a/rpc/client_context_go1.5.go b/rpc/client_context_go1.5.go deleted file mode 100644 index 4a007d9f8..000000000 --- a/rpc/client_context_go1.5.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build go1.5,!go1.6 - -package rpc - -import ( - "net" - "net/http" - "time" - - "golang.org/x/net/context" -) - -// In Go 1.5, dials cannot be canceled via a channel or context. The context deadline can -// still be applied. Go 1.5 adds the ability to cancel HTTP requests via a channel. - -// contextDialer returns a dialer that applies the deadline value from the given context. -func contextDialer(ctx context.Context) *net.Dialer { - dialer := &net.Dialer{KeepAlive: tcpKeepAliveInterval} - if deadline, ok := ctx.Deadline(); ok { - dialer.Deadline = deadline - } else { - dialer.Deadline = time.Now().Add(defaultDialTimeout) - } - return dialer -} - -// dialContext connects to the given address, aborting the dial if ctx is canceled. -func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { - return contextDialer(ctx).Dial(network, addr) -} - -// requestWithContext copies req, adding the cancelation channel and deadline from ctx. -func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { - // Set Timeout on the client if the context has a deadline. - // Note that there is no default timeout (unlike in contextDialer) because - // the timeout applies to the entire request, including reads from body. - if deadline, ok := ctx.Deadline(); ok { - c2 := *c - c2.Timeout = deadline.Sub(time.Now()) - c = &c2 - } - req2 := *req - req2.Cancel = ctx.Done() - return c, &req2 -} diff --git a/rpc/client_context_go1.6.go b/rpc/client_context_go1.6.go deleted file mode 100644 index 67777ddc6..000000000 --- a/rpc/client_context_go1.6.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build go1.6,!go1.7 - -package rpc - -import ( - "net" - "net/http" - "time" - - "golang.org/x/net/context" -) - -// In Go 1.6, net.Dialer gained the ability to cancel via a channel. - -// contextDialer returns a dialer that applies the deadline value from the given context. -func contextDialer(ctx context.Context) *net.Dialer { - dialer := &net.Dialer{Cancel: ctx.Done(), KeepAlive: tcpKeepAliveInterval} - if deadline, ok := ctx.Deadline(); ok { - dialer.Deadline = deadline - } else { - dialer.Deadline = time.Now().Add(defaultDialTimeout) - } - return dialer -} - -// dialContext connects to the given address, aborting the dial if ctx is canceled. -func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { - return contextDialer(ctx).Dial(network, addr) -} - -// requestWithContext copies req, adding the cancelation channel and deadline from ctx. -func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { - // We set Timeout on the client for Go <= 1.5. There - // is no need to do that here because the dial will be canceled - // by package http. - req2 := *req - req2.Cancel = ctx.Done() - return c, &req2 -} diff --git a/rpc/client_context_go1.7.go b/rpc/client_context_go1.7.go deleted file mode 100644 index 56ce12ab8..000000000 --- a/rpc/client_context_go1.7.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build go1.7 - -package rpc - -import ( - "context" - "net" - "net/http" - "time" -) - -// In Go 1.7, context moved into the standard library and support -// for cancelation via context was added to net.Dialer and http.Request. - -// contextDialer returns a dialer that applies the deadline value from the given context. -func contextDialer(ctx context.Context) *net.Dialer { - dialer := &net.Dialer{Cancel: ctx.Done(), KeepAlive: tcpKeepAliveInterval} - if deadline, ok := ctx.Deadline(); ok { - dialer.Deadline = deadline - } else { - dialer.Deadline = time.Now().Add(defaultDialTimeout) - } - return dialer -} - -// dialContext connects to the given address, aborting the dial if ctx is canceled. -func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { - d := &net.Dialer{KeepAlive: tcpKeepAliveInterval} - return d.DialContext(ctx, network, addr) -} - -// requestWithContext copies req, adding the cancelation channel and deadline from ctx. -func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { - return c, req.WithContext(ctx) -} diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go index 3462b3685..8276a9ead 100644 --- a/rpc/client_example_test.go +++ b/rpc/client_example_test.go @@ -17,12 +17,12 @@ package rpc_test import ( + "context" "fmt" "math/big" "time" "github.com/ethereum/go-ethereum/rpc" - "golang.org/x/net/context" ) // In this example, our client whishes to track the latest 'block number' diff --git a/rpc/client_test.go b/rpc/client_test.go index 407ed9c06..41471dcea 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -17,6 +17,7 @@ package rpc import ( + "context" "fmt" "math/rand" "net" @@ -31,7 +32,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/log" - "golang.org/x/net/context" ) func TestClientRequest(t *testing.T) { diff --git a/rpc/http.go b/rpc/http.go index 7d4fe5d47..89175b149 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -18,6 +18,7 @@ package rpc import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -29,7 +30,6 @@ import ( "time" "github.com/rs/cors" - "golang.org/x/net/context" ) const ( @@ -115,11 +115,11 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if err != nil { return nil, err } - client, req := requestWithContext(hc.client, hc.req, ctx) + req := hc.req.WithContext(ctx) req.Body = ioutil.NopCloser(bytes.NewReader(body)) req.ContentLength = int64(len(body)) - resp, err := client.Do(req) + resp, err := hc.client.Do(req) if err != nil { return nil, err } @@ -171,6 +171,7 @@ func newCorsHandler(srv *Server, corsString string) http.Handler { AllowedOrigins: allowedOrigins, AllowedMethods: []string{"POST", "GET"}, MaxAge: 600, + AllowedHeaders: []string{"*"}, }) return c.Handler(srv) } diff --git a/rpc/inproc.go b/rpc/inproc.go index f72b97497..595a7ca65 100644 --- a/rpc/inproc.go +++ b/rpc/inproc.go @@ -17,9 +17,8 @@ package rpc import ( + "context" "net" - - "golang.org/x/net/context" ) // NewInProcClient attaches an in-process connection to the given RPC server. diff --git a/rpc/ipc.go b/rpc/ipc.go index 3c86d711c..8de18a56f 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -17,12 +17,11 @@ package rpc import ( + "context" "fmt" "net" "github.com/ethereum/go-ethereum/log" - - "golang.org/x/net/context" ) // CreateIPCListener creates an listener, on Unix platforms this is a unix socket, on diff --git a/rpc/ipc_unix.go b/rpc/ipc_unix.go index a25b21627..0851ea61e 100644 --- a/rpc/ipc_unix.go +++ b/rpc/ipc_unix.go @@ -19,11 +19,10 @@ package rpc import ( + "context" "net" "os" "path/filepath" - - "golang.org/x/net/context" ) // ipcListen will create a Unix socket on the given endpoint. diff --git a/rpc/ipc_windows.go b/rpc/ipc_windows.go index 68234d215..ca56a3ce4 100644 --- a/rpc/ipc_windows.go +++ b/rpc/ipc_windows.go @@ -19,10 +19,10 @@ package rpc import ( + "context" "net" "time" - "golang.org/x/net/context" "gopkg.in/natefinch/npipe.v2" ) diff --git a/rpc/server.go b/rpc/server.go index 4f9ce541e..ca7e3c01a 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -17,14 +17,13 @@ package rpc import ( + "context" "fmt" "reflect" "runtime" "sync/atomic" "github.com/ethereum/go-ethereum/log" - - "golang.org/x/net/context" "gopkg.in/fatih/set.v0" ) diff --git a/rpc/server_test.go b/rpc/server_test.go index c3c88fab7..90d62f26d 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -17,13 +17,12 @@ package rpc import ( + "context" "encoding/json" "net" "reflect" "testing" "time" - - "golang.org/x/net/context" ) type Service struct{} diff --git a/rpc/subscription.go b/rpc/subscription.go index bcdc3cdfc..9ab6af9e1 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -17,10 +17,9 @@ package rpc import ( + "context" "errors" "sync" - - "golang.org/x/net/context" ) var ( diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index 00c4e0e35..345b4e5f2 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -17,13 +17,12 @@ package rpc import ( + "context" "encoding/json" "net" "sync" "testing" "time" - - "golang.org/x/net/context" ) type NotificationTestService struct { diff --git a/rpc/utils.go b/rpc/utils.go index c249e9b4a..2506c4833 100644 --- a/rpc/utils.go +++ b/rpc/utils.go @@ -18,6 +18,7 @@ package rpc import ( "bufio" + "context" crand "crypto/rand" "encoding/binary" "encoding/hex" @@ -29,8 +30,6 @@ import ( "time" "unicode" "unicode/utf8" - - "golang.org/x/net/context" ) var ( diff --git a/rpc/websocket.go b/rpc/websocket.go index f4271fda8..587010820 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -17,6 +17,7 @@ package rpc import ( + "context" "crypto/tls" "fmt" "net" @@ -24,10 +25,9 @@ import ( "net/url" "os" "strings" + "time" "github.com/ethereum/go-ethereum/log" - - "golang.org/x/net/context" "golang.org/x/net/websocket" "gopkg.in/fatih/set.v0" ) @@ -150,3 +150,18 @@ func wsDialAddress(location *url.URL) string { } return location.Host } + +func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { + d := &net.Dialer{KeepAlive: tcpKeepAliveInterval} + return d.DialContext(ctx, network, addr) +} + +func contextDialer(ctx context.Context) *net.Dialer { + dialer := &net.Dialer{Cancel: ctx.Done(), KeepAlive: tcpKeepAliveInterval} + if deadline, ok := ctx.Deadline(); ok { + dialer.Deadline = deadline + } else { + dialer.Deadline = time.Now().Add(defaultDialTimeout) + } + return dialer +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index a61696678..ae113750f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -82,6 +82,7 @@ func StartHttpServer(api *api.Api, server *Server) { AllowedOrigins: allowedOrigins, AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"}, MaxAge: 600, + AllowedHeaders: []string{"*"}, }) hdlr := c.Handler(serveMux) diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 8d731c038..bf976a3e1 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -116,7 +116,7 @@ func (self *Kademlia) DBCount() int { // On is the entry point called when a new nodes is added // unsafe in that node is not checked to be already active node (to be called once) func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) { - log.Warn(fmt.Sprintf("%v", self)) + log.Debug(fmt.Sprintf("%v", self)) defer self.lock.Unlock() self.lock.Lock() diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index eb21a598d..093892e8d 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -17,6 +17,7 @@ package swap import ( + "context" "crypto/ecdsa" "fmt" "math/big" @@ -33,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/services/swap/swap" - "golang.org/x/net/context" ) // SwAP Swarm Accounting Protocol with diff --git a/swarm/swarm.go b/swarm/swarm.go index 44564a71d..bd256edaa 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -18,6 +18,7 @@ package swarm import ( "bytes" + "context" "crypto/ecdsa" "fmt" @@ -35,7 +36,6 @@ import ( httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" - "golang.org/x/net/context" ) // the swarm stack diff --git a/trie/trie_test.go b/trie/trie_test.go index 60307dba8..01ae3a4e7 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -377,7 +377,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { if len(allKeys) < 2 || r.Intn(100) < 10 { // new key key := make([]byte, r.Intn(50)) - randRead(r, key) + r.Read(key) allKeys = append(allKeys, key) return key } @@ -401,22 +401,6 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { return reflect.ValueOf(steps) } -// rand.Rand provides a Read method in Go 1.7 and later, but -// we can't use it yet. -func randRead(r *rand.Rand, b []byte) { - pos := 0 - val := 0 - for n := 0; n < len(b); n++ { - if pos == 0 { - val = r.Int() - pos = 7 - } - b[n] = byte(val) - val >>= 8 - pos-- - } -} - func runRandTest(rt randTest) bool { db, _ := ethdb.NewMemDatabase() tr, _ := New(common.Hash{}, db) diff --git a/build/_vendor/src/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go index 134654cf7..f143ed6a1 100644 --- a/build/_vendor/src/golang.org/x/net/context/context.go +++ b/vendor/golang.org/x/net/context/context.go @@ -7,7 +7,7 @@ // and between processes. // // Incoming requests to a server should create a Context, and outgoing calls to -// servers should accept a Context. The chain of function calls between must +// servers should accept a Context. The chain of function calls between must // propagate the Context, optionally replacing it with a modified copy created // using WithDeadline, WithTimeout, WithCancel, or WithValue. // @@ -16,14 +16,14 @@ // propagation: // // Do not store Contexts inside a struct type; instead, pass a Context -// explicitly to each function that needs it. The Context should be the first +// explicitly to each function that needs it. The Context should be the first // parameter, typically named ctx: // // func DoSomething(ctx context.Context, arg Arg) error { // // ... use ctx ... // } // -// Do not pass a nil Context, even if a function permits it. Pass context.TODO +// Do not pass a nil Context, even if a function permits it. Pass context.TODO // if you are unsure about which Context to use. // // Use context Values only for request-scoped data that transits processes and @@ -44,13 +44,13 @@ import "time" // Context's methods may be called by multiple goroutines simultaneously. type Context interface { // Deadline returns the time when work done on behalf of this context - // should be canceled. Deadline returns ok==false when no deadline is - // set. Successive calls to Deadline return the same results. + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. Deadline() (deadline time.Time, ok bool) // Done returns a channel that's closed when work done on behalf of this - // context should be canceled. Done may return nil if this context can - // never be canceled. Successive calls to Done return the same value. + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. // // WithCancel arranges for Done to be closed when cancel is called; // WithDeadline arranges for Done to be closed when the deadline @@ -79,24 +79,24 @@ type Context interface { // a Done channel for cancelation. Done() <-chan struct{} - // Err returns a non-nil error value after Done is closed. Err returns + // Err returns a non-nil error value after Done is closed. Err returns // Canceled if the context was canceled or DeadlineExceeded if the - // context's deadline passed. No other values for Err are defined. + // context's deadline passed. No other values for Err are defined. // After Done is closed, successive calls to Err return the same value. Err() error // Value returns the value associated with this context for key, or nil - // if no value is associated with key. Successive calls to Value with + // if no value is associated with key. Successive calls to Value with // the same key returns the same result. // // Use context values only for request-scoped data that transits // processes and API boundaries, not for passing optional parameters to // functions. // - // A key identifies a specific value in a Context. Functions that wish + // A key identifies a specific value in a Context. Functions that wish // to store values in Context typically allocate a key in a global // variable then use that key as the argument to context.WithValue and - // Context.Value. A key can be any type that supports equality; + // Context.Value. A key can be any type that supports equality; // packages should define keys as an unexported type to avoid // collisions. // @@ -115,7 +115,7 @@ type Context interface { // // This prevents collisions with keys defined in other packages. // type key int // - // // userKey is the key for user.User values in Contexts. It is + // // userKey is the key for user.User values in Contexts. It is // // unexported; clients use user.NewContext and user.FromContext // // instead of using this key directly. // var userKey key = 0 @@ -134,14 +134,14 @@ type Context interface { } // Background returns a non-nil, empty Context. It is never canceled, has no -// values, and has no deadline. It is typically used by the main function, +// values, and has no deadline. It is typically used by the main function, // initialization, and tests, and as the top-level Context for incoming // requests. func Background() Context { return background } -// TODO returns a non-nil, empty Context. Code should use context.TODO when +// TODO returns a non-nil, empty Context. Code should use context.TODO when // it's unclear which Context to use or it is not yet available (because the // surrounding function has not yet been extended to accept a Context // parameter). TODO is recognized by static analysis tools that determine diff --git a/build/_vendor/src/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go index f8cda19ad..d20f52b7d 100644 --- a/build/_vendor/src/golang.org/x/net/context/go17.go +++ b/vendor/golang.org/x/net/context/go17.go @@ -35,8 +35,8 @@ func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { } // WithDeadline returns a copy of the parent context with the deadline adjusted -// to be no later than d. If the parent's deadline is already earlier than d, -// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned // context's Done channel is closed when the deadline expires, when the returned // cancel function is called, or when the parent context's Done channel is // closed, whichever happens first. diff --git a/build/_vendor/src/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go index 5a30acabd..0f35592df 100644 --- a/build/_vendor/src/golang.org/x/net/context/pre_go17.go +++ b/vendor/golang.org/x/net/context/pre_go17.go @@ -13,7 +13,7 @@ import ( "time" ) -// An emptyCtx is never canceled, has no values, and has no deadline. It is not +// An emptyCtx is never canceled, has no values, and has no deadline. It is not // struct{}, since vars of this type must have distinct addresses. type emptyCtx int @@ -104,7 +104,7 @@ func propagateCancel(parent Context, child canceler) { } // parentCancelCtx follows a chain of parent references until it finds a -// *cancelCtx. This function understands how each of the concrete types in this +// *cancelCtx. This function understands how each of the concrete types in this // package represents its parent. func parentCancelCtx(parent Context) (*cancelCtx, bool) { for { @@ -134,14 +134,14 @@ func removeChild(parent Context, child canceler) { p.mu.Unlock() } -// A canceler is a context type that can be canceled directly. The +// A canceler is a context type that can be canceled directly. The // implementations are *cancelCtx and *timerCtx. type canceler interface { cancel(removeFromParent bool, err error) Done() <-chan struct{} } -// A cancelCtx can be canceled. When canceled, it also cancels any children +// A cancelCtx can be canceled. When canceled, it also cancels any children // that implement canceler. type cancelCtx struct { Context @@ -193,8 +193,8 @@ func (c *cancelCtx) cancel(removeFromParent bool, err error) { } // WithDeadline returns a copy of the parent context with the deadline adjusted -// to be no later than d. If the parent's deadline is already earlier than d, -// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned // context's Done channel is closed when the deadline expires, when the returned // cancel function is called, or when the parent context's Done channel is // closed, whichever happens first. @@ -226,8 +226,8 @@ func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { return c, func() { c.cancel(true, Canceled) } } -// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to -// implement Done and Err. It implements cancel by stopping its timer then +// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to +// implement Done and Err. It implements cancel by stopping its timer then // delegating to cancelCtx.cancel. type timerCtx struct { *cancelCtx @@ -281,7 +281,7 @@ func WithValue(parent Context, key interface{}, val interface{}) Context { return &valueCtx{parent, key, val} } -// A valueCtx carries a key-value pair. It implements Value for that key and +// A valueCtx carries a key-value pair. It implements Value for that key and // delegates all other calls to the embedded Context. type valueCtx struct { Context diff --git a/vendor/vendor.json b/vendor/vendor.json index 58bbd82ff..eaaf0290d 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1,6 +1,6 @@ { "comment": "", - "ignore": "test golang.org/x/net/context", + "ignore": "test", "package": [ { "checksumSHA1": "M30X+Wqn7AnUr1numUOkQRI7ET0=", @@ -384,6 +384,12 @@ "revisionTime": "2017-02-08T20:51:15Z" }, { + "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=", + "path": "golang.org/x/net/context", + "revision": "a6577fac2d73be281a500b310739095313165611", + "revisionTime": "2017-03-08T20:54:49Z" + }, + { "checksumSHA1": "vqc3a+oTUGX8PmD0TS+qQ7gmN8I=", "path": "golang.org/x/net/html", "revision": "b4690f45fa1cafc47b1c280c2e75116efe40cc13", diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index d69fb40db..1cf85b8d7 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -18,7 +18,7 @@ package whisperv5 import ( "math/big" - "math/rand" + mrand "math/rand" "testing" "time" @@ -33,12 +33,12 @@ var seed int64 // reproduciblity independent of their sequence. func InitSingleTest() { seed = time.Now().Unix() - rand.Seed(seed) + mrand.Seed(seed) } func InitDebugTest(i int64) { seed = i - rand.Seed(seed) + mrand.Seed(seed) } type FilterTestCase struct { @@ -55,7 +55,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { const topicNum = 8 f.Topics = make([]TopicType, topicNum) for i := 0; i < topicNum; i++ { - randomize(f.Topics[i][:]) + mrand.Read(f.Topics[i][:]) f.Topics[i][0] = 0x01 } @@ -68,7 +68,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { if symmetric { f.KeySym = make([]byte, 12) - randomize(f.KeySym) + mrand.Read(f.KeySym) f.SymKeyHash = crypto.Keccak256Hash(f.KeySym) } else { f.KeyAsym, err = crypto.GenerateKey() @@ -87,7 +87,7 @@ func generateTestCases(t *testing.T, SizeTestFilters int) []FilterTestCase { for i := 0; i < SizeTestFilters; i++ { f, _ := generateFilter(t, true) cases[i].f = f - cases[i].alive = (rand.Int()&int(1) == 0) + cases[i].alive = (mrand.Int()&int(1) == 0) } return cases } @@ -147,7 +147,7 @@ func TestComparePubKey(t *testing.T) { } // generate key3 == key1 - rand.Seed(seed) + mrand.Seed(seed) key3, err := crypto.GenerateKey() if err != nil { t.Fatalf("failed to generate third key with seed %d: %s.", seed, err) @@ -193,7 +193,7 @@ func TestMatchEnvelope(t *testing.T) { } // encrypt symmetrically - i := rand.Int() % 4 + i := mrand.Int() % 4 fsym.Topics[i] = params.Topic fasym.Topics[i] = params.Topic msg = NewSentMessage(params) @@ -544,7 +544,7 @@ func TestWatchers(t *testing.T) { var envelopes [NumMessages]*Envelope for i = 0; i < NumMessages; i++ { - j = rand.Uint32() % NumFilters + j = mrand.Uint32() % NumFilters e = generateCompatibeEnvelope(t, tst[j].f) envelopes[i] = e tst[j].msgCnt++ @@ -597,7 +597,7 @@ func TestWatchers(t *testing.T) { envelopes[0] = e tst[0].msgCnt++ for i = 1; i < NumMessages; i++ { - j = rand.Uint32() % NumFilters + j = mrand.Uint32() % NumFilters e = generateCompatibeEnvelope(t, tst[j].f) envelopes[i] = e tst[j].msgCnt++ diff --git a/whisper/whisperv5/message.go b/whisper/whisperv5/message.go index 9677f278e..5f964b072 100644 --- a/whisper/whisperv5/message.go +++ b/whisper/whisperv5/message.go @@ -128,7 +128,7 @@ func (msg *SentMessage) appendPadding(params *MessageParams) { panic("please fix the padding algorithm before releasing new version") } buf := make([]byte, padSize) - randomize(buf[1:]) + mrand.Read(buf[1:]) buf[0] = byte(padSize) if params.Padding != nil { copy(buf[1:], params.Padding) @@ -365,19 +365,3 @@ func (msg *ReceivedMessage) hash() []byte { } return crypto.Keccak256(msg.Raw) } - -// rand.Rand provides a Read method in Go 1.7 and later, -// but we can't use it yet. -func randomize(b []byte) { - cnt := 0 - val := mrand.Int63() - for n := 0; n < len(b); n++ { - b[n] = byte(val) - val >>= 8 - cnt++ - if cnt >= 7 { - cnt = 0 - val = mrand.Int63() - } - } -} diff --git a/whisper/whisperv5/message_test.go b/whisper/whisperv5/message_test.go index c6f1ca2ca..1ed7250d3 100644 --- a/whisper/whisperv5/message_test.go +++ b/whisper/whisperv5/message_test.go @@ -18,7 +18,7 @@ package whisperv5 import ( "bytes" - "math/rand" + mrand "math/rand" "testing" "github.com/ethereum/go-ethereum/crypto" @@ -34,13 +34,13 @@ func generateMessageParams() (*MessageParams, error) { // set all the parameters except p.Dst buf := make([]byte, 1024) - randomize(buf) - sz := rand.Intn(400) + mrand.Read(buf) + sz := mrand.Intn(400) var p MessageParams p.PoW = 0.01 p.WorkTime = 1 - p.TTL = uint32(rand.Intn(1024)) + p.TTL = uint32(mrand.Intn(1024)) p.Payload = make([]byte, sz) p.Padding = make([]byte, padSizeLimitUpper) p.KeySym = make([]byte, aesKeyLength) @@ -132,7 +132,7 @@ func TestMessageEncryption(t *testing.T) { func TestMessageWrap(t *testing.T) { seed = int64(1777444222) - rand.Seed(seed) + mrand.Seed(seed) target := 128.0 params, err := generateMessageParams() @@ -168,7 +168,7 @@ func TestMessageWrap(t *testing.T) { func TestMessageSeal(t *testing.T) { // this test depends on deterministic choice of seed (1976726903) seed = int64(1976726903) - rand.Seed(seed) + mrand.Seed(seed) params, err := generateMessageParams() if err != nil { @@ -179,8 +179,8 @@ func TestMessageSeal(t *testing.T) { params.TTL = 1 aesnonce := make([]byte, 12) salt := make([]byte, 12) - randomize(aesnonce) - randomize(salt) + mrand.Read(aesnonce) + mrand.Read(salt) env := NewEnvelope(params.TTL, params.Topic, salt, aesnonce, msg) if err != nil { diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index 312dacfc4..8d63d443c 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -18,6 +18,7 @@ package whisperv5 import ( "bytes" + mrand "math/rand" "testing" "time" @@ -49,7 +50,7 @@ func TestWhisperBasic(t *testing.T) { } peerID := make([]byte, 64) - randomize(peerID) + mrand.Read(peerID) peer, _ := w.getPeer(peerID) if peer != nil { t.Fatal("found peer for random key.") @@ -212,7 +213,7 @@ func TestWhisperSymKeyManagement(t *testing.T) { // add existing id, nothing should change randomKey := make([]byte, 16) - randomize(randomKey) + mrand.Read(randomKey) err = w.AddSymKey(id1, randomKey) if err == nil { t.Fatalf("failed AddSymKey with seed %d.", seed) |