diff options
Diffstat (limited to 'eth')
| -rw-r--r-- | eth/api.go | 1549 | ||||
| -rw-r--r-- | eth/api_backend.go | 201 | ||||
| -rw-r--r-- | eth/backend.go | 266 | ||||
| -rw-r--r-- | eth/backend_test.go | 2 | ||||
| -rw-r--r-- | eth/bind.go | 58 | ||||
| -rw-r--r-- | eth/db_upgrade.go | 13 | ||||
| -rw-r--r-- | eth/downloader/downloader.go | 678 | ||||
| -rw-r--r-- | eth/downloader/downloader_test.go | 124 | ||||
| -rw-r--r-- | eth/downloader/metrics.go | 10 | ||||
| -rw-r--r-- | eth/downloader/peer.go | 50 | ||||
| -rw-r--r-- | eth/downloader/queue.go | 138 | ||||
| -rw-r--r-- | eth/downloader/types.go | 20 | ||||
| -rw-r--r-- | eth/fetcher/fetcher.go | 98 | ||||
| -rw-r--r-- | eth/fetcher/fetcher_test.go | 137 | ||||
| -rw-r--r-- | eth/fetcher/metrics.go | 3 | ||||
| -rw-r--r-- | eth/filters/api.go | 63 | ||||
| -rw-r--r-- | eth/filters/filter_system.go | 19 | ||||
| -rw-r--r-- | eth/filters/filter_test.go | 4 | ||||
| -rw-r--r-- | eth/gasprice/gasprice.go (renamed from eth/gasprice.go) | 57 | ||||
| -rw-r--r-- | eth/handler.go | 196 | ||||
| -rw-r--r-- | eth/handler_test.go | 228 | ||||
| -rw-r--r-- | eth/helper_test.go | 2 | ||||
| -rw-r--r-- | eth/metrics.go | 26 | ||||
| -rw-r--r-- | eth/peer.go | 50 | ||||
| -rw-r--r-- | eth/protocol.go | 50 | ||||
| -rw-r--r-- | eth/protocol_test.go | 3 |
26 files changed, 793 insertions, 3252 deletions
diff --git a/eth/api.go b/eth/api.go index 9f5f2e677..3b7abb69a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -18,8 +18,6 @@ package eth import ( "bytes" - "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -27,111 +25,30 @@ import ( "math/big" "os" "runtime" - "strings" - "sync" - "time" "github.com/ethereum/ethash" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/compiler" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/miner" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/syndtr/goleveldb/leveldb" - "golang.org/x/net/context" ) -const defaultGas = uint64(90000) - -// blockByNumber is a commonly used helper function which retrieves and returns -// the block for the given block number, capable of handling two special blocks: -// rpc.LatestBlockNumber and rpc.PendingBlockNumber. It returns nil when no block -// could be found. -func blockByNumber(m *miner.Miner, bc *core.BlockChain, blockNr rpc.BlockNumber) *types.Block { - // Pending block is only known by the miner - if blockNr == rpc.PendingBlockNumber { - block, _ := m.Pending() - return block - } - // Otherwise resolve and return the block - if blockNr == rpc.LatestBlockNumber { - return bc.CurrentBlock() - } - return bc.GetBlockByNumber(uint64(blockNr)) -} - -// stateAndBlockByNumber is a commonly used helper function which retrieves and -// returns the state and containing block for the given block number, capable of -// handling two special states: rpc.LatestBlockNumber and rpc.PendingBlockNumber. -// It returns nil when no block or state could be found. -func stateAndBlockByNumber(m *miner.Miner, bc *core.BlockChain, blockNr rpc.BlockNumber, chainDb ethdb.Database) (*state.StateDB, *types.Block, error) { - // Pending state is only known by the miner - if blockNr == rpc.PendingBlockNumber { - block, state := m.Pending() - return state, block, nil - } - // Otherwise resolve the block number and return its state - block := blockByNumber(m, bc, blockNr) - if block == nil { - return nil, nil, nil - } - stateDb, err := state.New(block.Root(), chainDb) - return stateDb, block, err -} - -// PublicEthereumAPI provides an API to access Ethereum related information. -// It offers only methods that operate on public data that is freely available to anyone. +// PublicEthereumAPI provides an API to access Ethereum full node-related +// information. type PublicEthereumAPI struct { - e *Ethereum - gpo *GasPriceOracle + e *Ethereum } -// NewPublicEthereumAPI creates a new Ethereum protocol API. +// NewPublicEthereumAPI creates a new Etheruem protocol API for full nodes. func NewPublicEthereumAPI(e *Ethereum) *PublicEthereumAPI { - return &PublicEthereumAPI{ - e: e, - gpo: e.gpo, - } -} - -// GasPrice returns a suggestion for a gas price. -func (s *PublicEthereumAPI) GasPrice() *big.Int { - return s.gpo.SuggestPrice() -} - -// GetCompilers returns the collection of available smart contract compilers -func (s *PublicEthereumAPI) GetCompilers() ([]string, error) { - solc, err := s.e.Solc() - if err == nil && solc != nil { - return []string{"Solidity"}, nil - } - - return []string{}, nil -} - -// CompileSolidity compiles the given solidity source -func (s *PublicEthereumAPI) CompileSolidity(source string) (map[string]*compiler.Contract, error) { - solc, err := s.e.Solc() - if err != nil { - return nil, err - } - - if solc == nil { - return nil, errors.New("solc (solidity compiler) not found") - } - - return solc.Compile(source) + return &PublicEthereumAPI{e} } // Etherbase is the address that mining rewards will be send to @@ -144,40 +61,11 @@ func (s *PublicEthereumAPI) Coinbase() (common.Address, error) { return s.Etherbase() } -// ProtocolVersion returns the current Ethereum protocol version this node supports -func (s *PublicEthereumAPI) ProtocolVersion() *rpc.HexNumber { - return rpc.NewHexNumber(s.e.EthVersion()) -} - // Hashrate returns the POW hashrate func (s *PublicEthereumAPI) Hashrate() *rpc.HexNumber { return rpc.NewHexNumber(s.e.Miner().HashRate()) } -// Syncing returns false in case the node is currently not syncing with the network. It can be up to date or has not -// yet received the latest block headers from its pears. In case it is synchronizing: -// - startingBlock: block number this node started to synchronise from -// - currentBlock: block number this node is currently importing -// - highestBlock: block number of the highest block header this node has received from peers -// - pulledStates: number of state entries processed until now -// - knownStates: number of known state entries that still need to be pulled -func (s *PublicEthereumAPI) Syncing() (interface{}, error) { - origin, current, height, pulled, known := s.e.Downloader().Progress() - - // Return not syncing if the synchronisation already completed - if current >= height { - return false, nil - } - // Otherwise gather the block sync stats - return map[string]interface{}{ - "startingBlock": rpc.NewHexNumber(origin), - "currentBlock": rpc.NewHexNumber(current), - "highestBlock": rpc.NewHexNumber(height), - "pulledStates": rpc.NewHexNumber(pulled), - "knownStates": rpc.NewHexNumber(known), - }, nil -} - // PublicMinerAPI provides an API to control the miner. // It offers only methods that operate on data that pose no security risk when it is publicly accessible. type PublicMinerAPI struct { @@ -303,1197 +191,18 @@ func (s *PrivateMinerAPI) MakeDAG(blockNr rpc.BlockNumber) (bool, error) { return true, nil } -// PublicTxPoolAPI offers and API for the transaction pool. It only operates on data that is non confidential. -type PublicTxPoolAPI struct { - e *Ethereum -} - -// NewPublicTxPoolAPI creates a new tx pool service that gives information about the transaction pool. -func NewPublicTxPoolAPI(e *Ethereum) *PublicTxPoolAPI { - return &PublicTxPoolAPI{e} -} - -// Content returns the transactions contained within the transaction pool. -func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string][]*RPCTransaction { - content := map[string]map[string]map[string][]*RPCTransaction{ - "pending": make(map[string]map[string][]*RPCTransaction), - "queued": make(map[string]map[string][]*RPCTransaction), - } - pending, queue := s.e.TxPool().Content() - - // Flatten the pending transactions - for account, batches := range pending { - dump := make(map[string][]*RPCTransaction) - for nonce, txs := range batches { - nonce := fmt.Sprintf("%d", nonce) - for _, tx := range txs { - dump[nonce] = append(dump[nonce], newRPCPendingTransaction(tx)) - } - } - content["pending"][account.Hex()] = dump - } - // Flatten the queued transactions - for account, batches := range queue { - dump := make(map[string][]*RPCTransaction) - for nonce, txs := range batches { - nonce := fmt.Sprintf("%d", nonce) - for _, tx := range txs { - dump[nonce] = append(dump[nonce], newRPCPendingTransaction(tx)) - } - } - content["queued"][account.Hex()] = dump - } - return content -} - -// Status returns the number of pending and queued transaction in the pool. -func (s *PublicTxPoolAPI) Status() map[string]*rpc.HexNumber { - pending, queue := s.e.TxPool().Stats() - return map[string]*rpc.HexNumber{ - "pending": rpc.NewHexNumber(pending), - "queued": rpc.NewHexNumber(queue), - } -} - -// Inspect retrieves the content of the transaction pool and flattens it into an -// easily inspectable list. -func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string][]string { - content := map[string]map[string]map[string][]string{ - "pending": make(map[string]map[string][]string), - "queued": make(map[string]map[string][]string), - } - pending, queue := s.e.TxPool().Content() - - // Define a formatter to flatten a transaction into a string - var format = func(tx *types.Transaction) string { - if to := tx.To(); to != nil { - return fmt.Sprintf("%s: %v wei + %v × %v gas", tx.To().Hex(), tx.Value(), tx.Gas(), tx.GasPrice()) - } - return fmt.Sprintf("contract creation: %v wei + %v × %v gas", tx.Value(), tx.Gas(), tx.GasPrice()) - } - // Flatten the pending transactions - for account, batches := range pending { - dump := make(map[string][]string) - for nonce, txs := range batches { - nonce := fmt.Sprintf("%d", nonce) - for _, tx := range txs { - dump[nonce] = append(dump[nonce], format(tx)) - } - } - content["pending"][account.Hex()] = dump - } - // Flatten the queued transactions - for account, batches := range queue { - dump := make(map[string][]string) - for nonce, txs := range batches { - nonce := fmt.Sprintf("%d", nonce) - for _, tx := range txs { - dump[nonce] = append(dump[nonce], format(tx)) - } - } - content["queued"][account.Hex()] = dump - } - return content -} - -// PublicAccountAPI provides an API to access accounts managed by this node. -// It offers only methods that can retrieve accounts. -type PublicAccountAPI struct { - am *accounts.Manager -} - -// NewPublicAccountAPI creates a new PublicAccountAPI. -func NewPublicAccountAPI(am *accounts.Manager) *PublicAccountAPI { - return &PublicAccountAPI{am: am} -} - -// Accounts returns the collection of accounts this node manages -func (s *PublicAccountAPI) Accounts() []accounts.Account { - return s.am.Accounts() -} - -// PrivateAccountAPI provides an API to access accounts managed by this node. -// It offers methods to create, (un)lock en list accounts. Some methods accept -// passwords and are therefore considered private by default. -type PrivateAccountAPI struct { - am *accounts.Manager - txPool *core.TxPool - txMu *sync.Mutex - gpo *GasPriceOracle -} - -// NewPrivateAccountAPI create a new PrivateAccountAPI. -func NewPrivateAccountAPI(e *Ethereum) *PrivateAccountAPI { - return &PrivateAccountAPI{ - am: e.accountManager, - txPool: e.txPool, - txMu: &e.txMu, - gpo: e.gpo, - } -} - -// ListAccounts will return a list of addresses for accounts this node manages. -func (s *PrivateAccountAPI) ListAccounts() []common.Address { - accounts := s.am.Accounts() - addresses := make([]common.Address, len(accounts)) - for i, acc := range accounts { - addresses[i] = acc.Address - } - return addresses -} - -// NewAccount will create a new account and returns the address for the new account. -func (s *PrivateAccountAPI) NewAccount(password string) (common.Address, error) { - acc, err := s.am.NewAccount(password) - if err == nil { - return acc.Address, nil - } - return common.Address{}, err -} - -// ImportRawKey stores the given hex encoded ECDSA key into the key directory, -// encrypting it with the passphrase. -func (s *PrivateAccountAPI) ImportRawKey(privkey string, password string) (common.Address, error) { - hexkey, err := hex.DecodeString(privkey) - if err != nil { - return common.Address{}, err - } - - acc, err := s.am.ImportECDSA(crypto.ToECDSA(hexkey), password) - return acc.Address, err -} - -// UnlockAccount will unlock the account associated with the given address with -// the given password for duration seconds. If duration is nil it will use a -// default of 300 seconds. It returns an indication if the account was unlocked. -func (s *PrivateAccountAPI) UnlockAccount(addr common.Address, password string, duration *rpc.HexNumber) (bool, error) { - if duration == nil { - duration = rpc.NewHexNumber(300) - } - a := accounts.Account{Address: addr} - d := time.Duration(duration.Int64()) * time.Second - if err := s.am.TimedUnlock(a, password, d); err != nil { - return false, err - } - return true, nil -} - -// LockAccount will lock the account associated with the given address when it's unlocked. -func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { - return s.am.Lock(addr) == nil -} - -// SignAndSendTransaction will create a transaction from the given arguments and -// tries to sign it with the key associated with args.To. If the given passwd isn't -// able to decrypt the key it fails. -func (s *PrivateAccountAPI) SignAndSendTransaction(args SendTxArgs, passwd string) (common.Hash, error) { - args = prepareSendTxArgs(args, s.gpo) - - s.txMu.Lock() - defer s.txMu.Unlock() - - if args.Nonce == nil { - args.Nonce = rpc.NewHexNumber(s.txPool.State().GetNonce(args.From)) - } - - var tx *types.Transaction - if args.To == nil { - tx = types.NewContractCreation(args.Nonce.Uint64(), args.Value.BigInt(), args.Gas.BigInt(), args.GasPrice.BigInt(), common.FromHex(args.Data)) - } else { - tx = types.NewTransaction(args.Nonce.Uint64(), *args.To, args.Value.BigInt(), args.Gas.BigInt(), args.GasPrice.BigInt(), common.FromHex(args.Data)) - } - - signature, err := s.am.SignWithPassphrase(args.From, passwd, tx.SigHash().Bytes()) - if err != nil { - return common.Hash{}, err - } - - return submitTransaction(s.txPool, tx, signature) -} - -// PublicBlockChainAPI provides an API to access the Ethereum blockchain. -// It offers only methods that operate on public data that is freely available to anyone. -type PublicBlockChainAPI struct { - config *core.ChainConfig - bc *core.BlockChain - chainDb ethdb.Database - eventMux *event.TypeMux - muNewBlockSubscriptions sync.Mutex // protects newBlocksSubscriptions - newBlockSubscriptions map[string]func(core.ChainEvent) error // callbacks for new block subscriptions - am *accounts.Manager - miner *miner.Miner - gpo *GasPriceOracle -} - -// NewPublicBlockChainAPI creates a new Etheruem blockchain API. -func NewPublicBlockChainAPI(config *core.ChainConfig, bc *core.BlockChain, m *miner.Miner, chainDb ethdb.Database, gpo *GasPriceOracle, eventMux *event.TypeMux, am *accounts.Manager) *PublicBlockChainAPI { - api := &PublicBlockChainAPI{ - config: config, - bc: bc, - miner: m, - chainDb: chainDb, - eventMux: eventMux, - am: am, - newBlockSubscriptions: make(map[string]func(core.ChainEvent) error), - gpo: gpo, - } - - go api.subscriptionLoop() - - return api -} - -// subscriptionLoop reads events from the global event mux and creates notifications for the matched subscriptions. -func (s *PublicBlockChainAPI) subscriptionLoop() { - sub := s.eventMux.Subscribe(core.ChainEvent{}) - for event := range sub.Chan() { - if chainEvent, ok := event.Data.(core.ChainEvent); ok { - s.muNewBlockSubscriptions.Lock() - for id, notifyOf := range s.newBlockSubscriptions { - if notifyOf(chainEvent) == rpc.ErrNotificationNotFound { - delete(s.newBlockSubscriptions, id) - } - } - s.muNewBlockSubscriptions.Unlock() - } - } -} - -// BlockNumber returns the block number of the chain head. -func (s *PublicBlockChainAPI) BlockNumber() *big.Int { - return s.bc.CurrentHeader().Number -} - -// GetBalance returns the amount of wei for the given address in the state of the -// given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta -// block numbers are also allowed. -func (s *PublicBlockChainAPI) GetBalance(address common.Address, blockNr rpc.BlockNumber) (*big.Int, error) { - state, _, err := stateAndBlockByNumber(s.miner, s.bc, blockNr, s.chainDb) - if state == nil || err != nil { - return nil, err - } - return state.GetBalance(address), nil -} - -// GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all -// transactions in the block are returned in full detail, otherwise only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetBlockByNumber(blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { - if block := blockByNumber(s.miner, s.bc, blockNr); block != nil { - response, err := s.rpcOutputBlock(block, true, fullTx) - if err == nil && blockNr == rpc.PendingBlockNumber { - // Pending blocks need to nil out a few fields - for _, field := range []string{"hash", "nonce", "logsBloom", "miner"} { - response[field] = nil - } - } - return response, err - } - return nil, nil -} - -// GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full -// detail, otherwise only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetBlockByHash(blockHash common.Hash, fullTx bool) (map[string]interface{}, error) { - if block := s.bc.GetBlockByHash(blockHash); block != nil { - return s.rpcOutputBlock(block, true, fullTx) - } - return nil, nil -} - -// GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. When fullTx is true -// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(blockNr rpc.BlockNumber, index rpc.HexNumber) (map[string]interface{}, error) { - if block := blockByNumber(s.miner, s.bc, blockNr); block != nil { - uncles := block.Uncles() - if index.Int() < 0 || index.Int() >= len(uncles) { - glog.V(logger.Debug).Infof("uncle block on index %d not found for block #%d", index.Int(), blockNr) - return nil, nil - } - block = types.NewBlockWithHeader(uncles[index.Int()]) - return s.rpcOutputBlock(block, false, false) - } - return nil, nil -} - -// GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. When fullTx is true -// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(blockHash common.Hash, index rpc.HexNumber) (map[string]interface{}, error) { - if block := s.bc.GetBlockByHash(blockHash); block != nil { - uncles := block.Uncles() - if index.Int() < 0 || index.Int() >= len(uncles) { - glog.V(logger.Debug).Infof("uncle block on index %d not found for block %s", index.Int(), blockHash.Hex()) - return nil, nil - } - block = types.NewBlockWithHeader(uncles[index.Int()]) - return s.rpcOutputBlock(block, false, false) - } - return nil, nil -} - -// GetUncleCountByBlockNumber returns number of uncles in the block for the given block number -func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(blockNr rpc.BlockNumber) *rpc.HexNumber { - if block := blockByNumber(s.miner, s.bc, blockNr); block != nil { - return rpc.NewHexNumber(len(block.Uncles())) - } - return nil -} - -// GetUncleCountByBlockHash returns number of uncles in the block for the given block hash -func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(blockHash common.Hash) *rpc.HexNumber { - if block := s.bc.GetBlockByHash(blockHash); block != nil { - return rpc.NewHexNumber(len(block.Uncles())) - } - return nil -} - -// NewBlocksArgs allows the user to specify if the returned block should include transactions and in which format. -type NewBlocksArgs struct { - IncludeTransactions bool `json:"includeTransactions"` - TransactionDetails bool `json:"transactionDetails"` -} - -// NewBlocks triggers a new block event each time a block is appended to the chain. It accepts an argument which allows -// the caller to specify whether the output should contain transactions and in what format. -func (s *PublicBlockChainAPI) NewBlocks(ctx context.Context, args NewBlocksArgs) (rpc.Subscription, error) { - notifier, supported := rpc.NotifierFromContext(ctx) - if !supported { - return nil, rpc.ErrNotificationsUnsupported - } - - // create a subscription that will remove itself when unsubscribed/cancelled - subscription, err := notifier.NewSubscription(func(subId string) { - s.muNewBlockSubscriptions.Lock() - delete(s.newBlockSubscriptions, subId) - s.muNewBlockSubscriptions.Unlock() - }) - - if err != nil { - return nil, err - } - - // add a callback that is called on chain events which will format the block and notify the client - s.muNewBlockSubscriptions.Lock() - s.newBlockSubscriptions[subscription.ID()] = func(e core.ChainEvent) error { - notification, err := s.rpcOutputBlock(e.Block, args.IncludeTransactions, args.TransactionDetails) - if err == nil { - return subscription.Notify(notification) - } - glog.V(logger.Warn).Info("unable to format block %v\n", err) - return nil - } - s.muNewBlockSubscriptions.Unlock() - return subscription, nil -} - -// GetCode returns the code stored at the given address in the state for the given block number. -func (s *PublicBlockChainAPI) GetCode(address common.Address, blockNr rpc.BlockNumber) (string, error) { - state, _, err := stateAndBlockByNumber(s.miner, s.bc, blockNr, s.chainDb) - if state == nil || err != nil { - return "", err - } - res := state.GetCode(address) - if len(res) == 0 { // backwards compatibility - return "0x", nil - } - return common.ToHex(res), nil -} - -// GetStorageAt returns the storage from the state at the given address, key and -// block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block -// numbers are also allowed. -func (s *PublicBlockChainAPI) GetStorageAt(address common.Address, key string, blockNr rpc.BlockNumber) (string, error) { - state, _, err := stateAndBlockByNumber(s.miner, s.bc, blockNr, s.chainDb) - if state == nil || err != nil { - return "0x", err - } - return state.GetState(address, common.HexToHash(key)).Hex(), nil -} - -// callmsg is the message type used for call transactions. -type callmsg struct { - from *state.StateObject - to *common.Address - gas, gasPrice *big.Int - value *big.Int - data []byte -} - -// accessor boilerplate to implement core.Message -func (m callmsg) From() (common.Address, error) { return m.from.Address(), nil } -func (m callmsg) FromFrontier() (common.Address, error) { return m.from.Address(), nil } -func (m callmsg) Nonce() uint64 { return m.from.Nonce() } -func (m callmsg) To() *common.Address { return m.to } -func (m callmsg) GasPrice() *big.Int { return m.gasPrice } -func (m callmsg) Gas() *big.Int { return m.gas } -func (m callmsg) Value() *big.Int { return m.value } -func (m callmsg) Data() []byte { return m.data } - -// CallArgs represents the arguments for a call. -type CallArgs struct { - From common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *rpc.HexNumber `json:"gas"` - GasPrice *rpc.HexNumber `json:"gasPrice"` - Value rpc.HexNumber `json:"value"` - Data string `json:"data"` -} - -func (s *PublicBlockChainAPI) doCall(args CallArgs, blockNr rpc.BlockNumber) (string, *big.Int, error) { - // Fetch the state associated with the block number - stateDb, block, err := stateAndBlockByNumber(s.miner, s.bc, blockNr, s.chainDb) - if stateDb == nil || err != nil { - return "0x", nil, err - } - stateDb = stateDb.Copy() - - // Retrieve the account state object to interact with - var from *state.StateObject - if args.From == (common.Address{}) { - accounts := s.am.Accounts() - if len(accounts) == 0 { - from = stateDb.GetOrNewStateObject(common.Address{}) - } else { - from = stateDb.GetOrNewStateObject(accounts[0].Address) - } - } else { - from = stateDb.GetOrNewStateObject(args.From) - } - from.SetBalance(common.MaxBig) - - // Assemble the CALL invocation - msg := callmsg{ - from: from, - to: args.To, - gas: args.Gas.BigInt(), - gasPrice: args.GasPrice.BigInt(), - value: args.Value.BigInt(), - data: common.FromHex(args.Data), - } - if msg.gas == nil { - msg.gas = big.NewInt(50000000) - } - if msg.gasPrice == nil { - msg.gasPrice = s.gpo.SuggestPrice() - } - - // Execute the call and return - vmenv := core.NewEnv(stateDb, s.config, s.bc, msg, block.Header(), s.config.VmConfig) - gp := new(core.GasPool).AddGas(common.MaxBig) - - res, requiredGas, _, err := core.NewStateTransition(vmenv, msg, gp).TransitionDb() - if len(res) == 0 { // backwards compatibility - return "0x", requiredGas, err - } - return common.ToHex(res), requiredGas, err -} - -// Call executes the given transaction on the state for the given block number. -// It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(args CallArgs, blockNr rpc.BlockNumber) (string, error) { - result, _, err := s.doCall(args, blockNr) - return result, err -} - -// EstimateGas returns an estimate of the amount of gas needed to execute the given transaction. -func (s *PublicBlockChainAPI) EstimateGas(args CallArgs) (*rpc.HexNumber, error) { - _, gas, err := s.doCall(args, rpc.PendingBlockNumber) - return rpc.NewHexNumber(gas), err -} - -// rpcOutputBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are -// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain -// transaction hashes. -func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields := map[string]interface{}{ - "number": rpc.NewHexNumber(b.Number()), - "hash": b.Hash(), - "parentHash": b.ParentHash(), - "nonce": b.Header().Nonce, - "sha3Uncles": b.UncleHash(), - "logsBloom": b.Bloom(), - "stateRoot": b.Root(), - "miner": b.Coinbase(), - "difficulty": rpc.NewHexNumber(b.Difficulty()), - "totalDifficulty": rpc.NewHexNumber(s.bc.GetTd(b.Hash(), b.NumberU64())), - "extraData": fmt.Sprintf("0x%x", b.Extra()), - "size": rpc.NewHexNumber(b.Size().Int64()), - "gasLimit": rpc.NewHexNumber(b.GasLimit()), - "gasUsed": rpc.NewHexNumber(b.GasUsed()), - "timestamp": rpc.NewHexNumber(b.Time()), - "transactionsRoot": b.TxHash(), - "receiptRoot": b.ReceiptHash(), - } - - if inclTx { - formatTx := func(tx *types.Transaction) (interface{}, error) { - return tx.Hash(), nil - } - - if fullTx { - formatTx = func(tx *types.Transaction) (interface{}, error) { - return newRPCTransaction(b, tx.Hash()) - } - } - - txs := b.Transactions() - transactions := make([]interface{}, len(txs)) - var err error - for i, tx := range b.Transactions() { - if transactions[i], err = formatTx(tx); err != nil { - return nil, err - } - } - fields["transactions"] = transactions - } - - uncles := b.Uncles() - uncleHashes := make([]common.Hash, len(uncles)) - for i, uncle := range uncles { - uncleHashes[i] = uncle.Hash() - } - fields["uncles"] = uncleHashes - - return fields, nil -} - -// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction -type RPCTransaction struct { - BlockHash common.Hash `json:"blockHash"` - BlockNumber *rpc.HexNumber `json:"blockNumber"` - From common.Address `json:"from"` - Gas *rpc.HexNumber `json:"gas"` - GasPrice *rpc.HexNumber `json:"gasPrice"` - Hash common.Hash `json:"hash"` - Input string `json:"input"` - Nonce *rpc.HexNumber `json:"nonce"` - To *common.Address `json:"to"` - TransactionIndex *rpc.HexNumber `json:"transactionIndex"` - Value *rpc.HexNumber `json:"value"` -} - -// newRPCPendingTransaction returns a pending transaction that will serialize to the RPC representation -func newRPCPendingTransaction(tx *types.Transaction) *RPCTransaction { - from, _ := tx.FromFrontier() - - return &RPCTransaction{ - From: from, - Gas: rpc.NewHexNumber(tx.Gas()), - GasPrice: rpc.NewHexNumber(tx.GasPrice()), - Hash: tx.Hash(), - Input: fmt.Sprintf("0x%x", tx.Data()), - Nonce: rpc.NewHexNumber(tx.Nonce()), - To: tx.To(), - Value: rpc.NewHexNumber(tx.Value()), - } -} - -// newRPCTransaction returns a transaction that will serialize to the RPC representation. -func newRPCTransactionFromBlockIndex(b *types.Block, txIndex int) (*RPCTransaction, error) { - if txIndex >= 0 && txIndex < len(b.Transactions()) { - tx := b.Transactions()[txIndex] - from, err := tx.FromFrontier() - if err != nil { - return nil, err - } - - return &RPCTransaction{ - BlockHash: b.Hash(), - BlockNumber: rpc.NewHexNumber(b.Number()), - From: from, - Gas: rpc.NewHexNumber(tx.Gas()), - GasPrice: rpc.NewHexNumber(tx.GasPrice()), - Hash: tx.Hash(), - Input: fmt.Sprintf("0x%x", tx.Data()), - Nonce: rpc.NewHexNumber(tx.Nonce()), - To: tx.To(), - TransactionIndex: rpc.NewHexNumber(txIndex), - Value: rpc.NewHexNumber(tx.Value()), - }, nil - } - - return nil, nil -} - -// newRPCTransaction returns a transaction that will serialize to the RPC representation. -func newRPCTransaction(b *types.Block, txHash common.Hash) (*RPCTransaction, error) { - for idx, tx := range b.Transactions() { - if tx.Hash() == txHash { - return newRPCTransactionFromBlockIndex(b, idx) - } - } - - return nil, nil -} - -// PublicTransactionPoolAPI exposes methods for the RPC interface -type PublicTransactionPoolAPI struct { - eventMux *event.TypeMux - chainDb ethdb.Database - gpo *GasPriceOracle - bc *core.BlockChain - miner *miner.Miner - am *accounts.Manager - txPool *core.TxPool - txMu *sync.Mutex - muPendingTxSubs sync.Mutex - pendingTxSubs map[string]rpc.Subscription -} - -// NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. -func NewPublicTransactionPoolAPI(e *Ethereum) *PublicTransactionPoolAPI { - api := &PublicTransactionPoolAPI{ - eventMux: e.eventMux, - gpo: e.gpo, - chainDb: e.chainDb, - bc: e.blockchain, - am: e.accountManager, - txPool: e.txPool, - txMu: &e.txMu, - miner: e.miner, - pendingTxSubs: make(map[string]rpc.Subscription), - } - go api.subscriptionLoop() - - return api -} - -// subscriptionLoop listens for events on the global event mux and creates notifications for subscriptions. -func (s *PublicTransactionPoolAPI) subscriptionLoop() { - sub := s.eventMux.Subscribe(core.TxPreEvent{}) - for event := range sub.Chan() { - tx := event.Data.(core.TxPreEvent) - if from, err := tx.Tx.FromFrontier(); err == nil { - if s.am.HasAddress(from) { - s.muPendingTxSubs.Lock() - for id, sub := range s.pendingTxSubs { - if sub.Notify(tx.Tx.Hash()) == rpc.ErrNotificationNotFound { - delete(s.pendingTxSubs, id) - } - } - s.muPendingTxSubs.Unlock() - } - } - } -} - -func getTransaction(chainDb ethdb.Database, txPool *core.TxPool, txHash common.Hash) (*types.Transaction, bool, error) { - txData, err := chainDb.Get(txHash.Bytes()) - isPending := false - tx := new(types.Transaction) - - if err == nil && len(txData) > 0 { - if err := rlp.DecodeBytes(txData, tx); err != nil { - return nil, isPending, err - } - } else { - // pending transaction? - tx = txPool.GetTransaction(txHash) - isPending = true - } - - return tx, isPending, nil -} - -// GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. -func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(blockNr rpc.BlockNumber) *rpc.HexNumber { - if block := blockByNumber(s.miner, s.bc, blockNr); block != nil { - return rpc.NewHexNumber(len(block.Transactions())) - } - return nil -} - -// GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. -func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(blockHash common.Hash) *rpc.HexNumber { - if block := s.bc.GetBlockByHash(blockHash); block != nil { - return rpc.NewHexNumber(len(block.Transactions())) - } - return nil -} - -// GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. -func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(blockNr rpc.BlockNumber, index rpc.HexNumber) (*RPCTransaction, error) { - if block := blockByNumber(s.miner, s.bc, blockNr); block != nil { - return newRPCTransactionFromBlockIndex(block, index.Int()) - } - return nil, nil -} - -// GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. -func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(blockHash common.Hash, index rpc.HexNumber) (*RPCTransaction, error) { - if block := s.bc.GetBlockByHash(blockHash); block != nil { - return newRPCTransactionFromBlockIndex(block, index.Int()) - } - return nil, nil -} - -// GetTransactionCount returns the number of transactions the given address has sent for the given block number -func (s *PublicTransactionPoolAPI) GetTransactionCount(address common.Address, blockNr rpc.BlockNumber) (*rpc.HexNumber, error) { - state, _, err := stateAndBlockByNumber(s.miner, s.bc, blockNr, s.chainDb) - if state == nil || err != nil { - return nil, err - } - return rpc.NewHexNumber(state.GetNonce(address)), nil -} - -// getTransactionBlockData fetches the meta data for the given transaction from the chain database. This is useful to -// retrieve block information for a hash. It returns the block hash, block index and transaction index. -func getTransactionBlockData(chainDb ethdb.Database, txHash common.Hash) (common.Hash, uint64, uint64, error) { - var txBlock struct { - BlockHash common.Hash - BlockIndex uint64 - Index uint64 - } - - blockData, err := chainDb.Get(append(txHash.Bytes(), 0x0001)) - if err != nil { - return common.Hash{}, uint64(0), uint64(0), err - } - - reader := bytes.NewReader(blockData) - if err = rlp.Decode(reader, &txBlock); err != nil { - return common.Hash{}, uint64(0), uint64(0), err - } - - return txBlock.BlockHash, txBlock.BlockIndex, txBlock.Index, nil -} - -// GetTransactionByHash returns the transaction for the given hash -func (s *PublicTransactionPoolAPI) GetTransactionByHash(txHash common.Hash) (*RPCTransaction, error) { - var tx *types.Transaction - var isPending bool - var err error - - if tx, isPending, err = getTransaction(s.chainDb, s.txPool, txHash); err != nil { - glog.V(logger.Debug).Infof("%v\n", err) - return nil, nil - } else if tx == nil { - return nil, nil - } - - if isPending { - return newRPCPendingTransaction(tx), nil - } - - blockHash, _, _, err := getTransactionBlockData(s.chainDb, txHash) - if err != nil { - glog.V(logger.Debug).Infof("%v\n", err) - return nil, nil - } - - if block := s.bc.GetBlockByHash(blockHash); block != nil { - return newRPCTransaction(block, txHash) - } - - return nil, nil -} - -// GetTransactionReceipt returns the transaction receipt for the given transaction hash. -func (s *PublicTransactionPoolAPI) GetTransactionReceipt(txHash common.Hash) (map[string]interface{}, error) { - receipt := core.GetReceipt(s.chainDb, txHash) - if receipt == nil { - glog.V(logger.Debug).Infof("receipt not found for transaction %s", txHash.Hex()) - return nil, nil - } - - tx, _, err := getTransaction(s.chainDb, s.txPool, txHash) - if err != nil { - glog.V(logger.Debug).Infof("%v\n", err) - return nil, nil - } - - txBlock, blockIndex, index, err := getTransactionBlockData(s.chainDb, txHash) - if err != nil { - glog.V(logger.Debug).Infof("%v\n", err) - return nil, nil - } - - from, err := tx.FromFrontier() - if err != nil { - glog.V(logger.Debug).Infof("%v\n", err) - return nil, nil - } - - fields := map[string]interface{}{ - "root": common.Bytes2Hex(receipt.PostState), - "blockHash": txBlock, - "blockNumber": rpc.NewHexNumber(blockIndex), - "transactionHash": txHash, - "transactionIndex": rpc.NewHexNumber(index), - "from": from, - "to": tx.To(), - "gasUsed": rpc.NewHexNumber(receipt.GasUsed), - "cumulativeGasUsed": rpc.NewHexNumber(receipt.CumulativeGasUsed), - "contractAddress": nil, - "logs": receipt.Logs, - } - - if receipt.Logs == nil { - fields["logs"] = []vm.Logs{} - } - - // If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation - if bytes.Compare(receipt.ContractAddress.Bytes(), bytes.Repeat([]byte{0}, 20)) != 0 { - fields["contractAddress"] = receipt.ContractAddress - } - - return fields, nil -} - -// sign is a helper function that signs a transaction with the private key of the given address. -func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transaction) (*types.Transaction, error) { - signature, err := s.am.Sign(addr, tx.SigHash().Bytes()) - if err != nil { - return nil, err - } - return tx.WithSignature(signature) -} - -// SendTxArgs represents the arguments to sumbit a new transaction into the transaction pool. -type SendTxArgs struct { - From common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *rpc.HexNumber `json:"gas"` - GasPrice *rpc.HexNumber `json:"gasPrice"` - Value *rpc.HexNumber `json:"value"` - Data string `json:"data"` - Nonce *rpc.HexNumber `json:"nonce"` -} - -// prepareSendTxArgs is a helper function that fills in default values for unspecified tx fields. -func prepareSendTxArgs(args SendTxArgs, gpo *GasPriceOracle) SendTxArgs { - if args.Gas == nil { - args.Gas = rpc.NewHexNumber(defaultGas) - } - if args.GasPrice == nil { - args.GasPrice = rpc.NewHexNumber(gpo.SuggestPrice()) - } - if args.Value == nil { - args.Value = rpc.NewHexNumber(0) - } - return args -} - -// submitTransaction is a helper function that submits tx to txPool and creates a log entry. -func submitTransaction(txPool *core.TxPool, tx *types.Transaction, signature []byte) (common.Hash, error) { - signedTx, err := tx.WithSignature(signature) - if err != nil { - return common.Hash{}, err - } - - txPool.SetLocal(signedTx) - if err := txPool.Add(signedTx); err != nil { - return common.Hash{}, err - } - - if signedTx.To() == nil { - from, _ := signedTx.From() - addr := crypto.CreateAddress(from, signedTx.Nonce()) - glog.V(logger.Info).Infof("Tx(%s) created: %s\n", signedTx.Hash().Hex(), addr.Hex()) - } else { - glog.V(logger.Info).Infof("Tx(%s) to: %s\n", signedTx.Hash().Hex(), tx.To().Hex()) - } - - return signedTx.Hash(), nil -} - -// SendTransaction creates a transaction for the given argument, sign it and submit it to the -// transaction pool. -func (s *PublicTransactionPoolAPI) SendTransaction(args SendTxArgs) (common.Hash, error) { - args = prepareSendTxArgs(args, s.gpo) - - s.txMu.Lock() - defer s.txMu.Unlock() - - if args.Nonce == nil { - args.Nonce = rpc.NewHexNumber(s.txPool.State().GetNonce(args.From)) - } - - var tx *types.Transaction - if args.To == nil { - tx = types.NewContractCreation(args.Nonce.Uint64(), args.Value.BigInt(), args.Gas.BigInt(), args.GasPrice.BigInt(), common.FromHex(args.Data)) - } else { - tx = types.NewTransaction(args.Nonce.Uint64(), *args.To, args.Value.BigInt(), args.Gas.BigInt(), args.GasPrice.BigInt(), common.FromHex(args.Data)) - } - - signature, err := s.am.Sign(args.From, tx.SigHash().Bytes()) - if err != nil { - return common.Hash{}, err - } - - return submitTransaction(s.txPool, tx, signature) -} - -// SendRawTransaction will add the signed transaction to the transaction pool. -// The sender is responsible for signing the transaction and using the correct nonce. -func (s *PublicTransactionPoolAPI) SendRawTransaction(encodedTx string) (string, error) { - tx := new(types.Transaction) - if err := rlp.DecodeBytes(common.FromHex(encodedTx), tx); err != nil { - return "", err - } - - s.txPool.SetLocal(tx) - if err := s.txPool.Add(tx); err != nil { - return "", err - } - - if tx.To() == nil { - from, err := tx.FromFrontier() - if err != nil { - return "", err - } - addr := crypto.CreateAddress(from, tx.Nonce()) - glog.V(logger.Info).Infof("Tx(%x) created: %x\n", tx.Hash(), addr) - } else { - glog.V(logger.Info).Infof("Tx(%x) to: %x\n", tx.Hash(), tx.To()) - } - - return tx.Hash().Hex(), nil -} - -// Sign signs the given hash using the key that matches the address. The key must be -// unlocked in order to sign the hash. -func (s *PublicTransactionPoolAPI) Sign(addr common.Address, hash common.Hash) (string, error) { - signature, error := s.am.Sign(addr, hash[:]) - return common.ToHex(signature), error -} - -// SignTransactionArgs represents the arguments to sign a transaction. -type SignTransactionArgs struct { - From common.Address - To *common.Address - Nonce *rpc.HexNumber - Value *rpc.HexNumber - Gas *rpc.HexNumber - GasPrice *rpc.HexNumber - Data string - - BlockNumber int64 -} - -// Tx is a helper object for argument and return values -type Tx struct { - tx *types.Transaction - - To *common.Address `json:"to"` - From common.Address `json:"from"` - Nonce *rpc.HexNumber `json:"nonce"` - Value *rpc.HexNumber `json:"value"` - Data string `json:"data"` - GasLimit *rpc.HexNumber `json:"gas"` - GasPrice *rpc.HexNumber `json:"gasPrice"` - Hash common.Hash `json:"hash"` -} - -// UnmarshalJSON parses JSON data into tx. -func (tx *Tx) UnmarshalJSON(b []byte) (err error) { - req := struct { - To *common.Address `json:"to"` - From common.Address `json:"from"` - Nonce *rpc.HexNumber `json:"nonce"` - Value *rpc.HexNumber `json:"value"` - Data string `json:"data"` - GasLimit *rpc.HexNumber `json:"gas"` - GasPrice *rpc.HexNumber `json:"gasPrice"` - Hash common.Hash `json:"hash"` - }{} - - if err := json.Unmarshal(b, &req); err != nil { - return err - } - - tx.To = req.To - tx.From = req.From - tx.Nonce = req.Nonce - tx.Value = req.Value - tx.Data = req.Data - tx.GasLimit = req.GasLimit - tx.GasPrice = req.GasPrice - tx.Hash = req.Hash - - data := common.Hex2Bytes(tx.Data) - - if tx.Nonce == nil { - return fmt.Errorf("need nonce") - } - if tx.Value == nil { - tx.Value = rpc.NewHexNumber(0) - } - if tx.GasLimit == nil { - tx.GasLimit = rpc.NewHexNumber(0) - } - if tx.GasPrice == nil { - tx.GasPrice = rpc.NewHexNumber(int64(50000000000)) - } - - if req.To == nil { - tx.tx = types.NewContractCreation(tx.Nonce.Uint64(), tx.Value.BigInt(), tx.GasLimit.BigInt(), tx.GasPrice.BigInt(), data) - } else { - tx.tx = types.NewTransaction(tx.Nonce.Uint64(), *tx.To, tx.Value.BigInt(), tx.GasLimit.BigInt(), tx.GasPrice.BigInt(), data) - } - - return nil -} - -// SignTransactionResult represents a RLP encoded signed transaction. -type SignTransactionResult struct { - Raw string `json:"raw"` - Tx *Tx `json:"tx"` -} - -func newTx(t *types.Transaction) *Tx { - from, _ := t.FromFrontier() - return &Tx{ - tx: t, - To: t.To(), - From: from, - Value: rpc.NewHexNumber(t.Value()), - Nonce: rpc.NewHexNumber(t.Nonce()), - Data: "0x" + common.Bytes2Hex(t.Data()), - GasLimit: rpc.NewHexNumber(t.Gas()), - GasPrice: rpc.NewHexNumber(t.GasPrice()), - Hash: t.Hash(), - } -} - -// SignTransaction will sign the given transaction with the from account. -// The node needs to have the private key of the account corresponding with -// the given from address and it needs to be unlocked. -func (s *PublicTransactionPoolAPI) SignTransaction(args SignTransactionArgs) (*SignTransactionResult, error) { - if args.Gas == nil { - args.Gas = rpc.NewHexNumber(defaultGas) - } - if args.GasPrice == nil { - args.GasPrice = rpc.NewHexNumber(s.gpo.SuggestPrice()) - } - if args.Value == nil { - args.Value = rpc.NewHexNumber(0) - } - - s.txMu.Lock() - defer s.txMu.Unlock() - - if args.Nonce == nil { - args.Nonce = rpc.NewHexNumber(s.txPool.State().GetNonce(args.From)) - } - - var tx *types.Transaction - if args.To == nil { - tx = types.NewContractCreation(args.Nonce.Uint64(), args.Value.BigInt(), args.Gas.BigInt(), args.GasPrice.BigInt(), common.FromHex(args.Data)) - } else { - tx = types.NewTransaction(args.Nonce.Uint64(), *args.To, args.Value.BigInt(), args.Gas.BigInt(), args.GasPrice.BigInt(), common.FromHex(args.Data)) - } - - signedTx, err := s.sign(args.From, tx) - if err != nil { - return nil, err - } - - data, err := rlp.EncodeToBytes(signedTx) - if err != nil { - return nil, err - } - - return &SignTransactionResult{"0x" + common.Bytes2Hex(data), newTx(signedTx)}, nil -} - -// PendingTransactions returns the transactions that are in the transaction pool and have a from address that is one of -// the accounts this node manages. -func (s *PublicTransactionPoolAPI) PendingTransactions() []*RPCTransaction { - pending := s.txPool.GetTransactions() - transactions := make([]*RPCTransaction, 0, len(pending)) - for _, tx := range pending { - from, _ := tx.FromFrontier() - if s.am.HasAddress(from) { - transactions = append(transactions, newRPCPendingTransaction(tx)) - } - } - return transactions -} - -// NewPendingTransactions creates a subscription that is triggered each time a transaction enters the transaction pool -// and is send from one of the transactions this nodes manages. -func (s *PublicTransactionPoolAPI) NewPendingTransactions(ctx context.Context) (rpc.Subscription, error) { - notifier, supported := rpc.NotifierFromContext(ctx) - if !supported { - return nil, rpc.ErrNotificationsUnsupported - } - - subscription, err := notifier.NewSubscription(func(id string) { - s.muPendingTxSubs.Lock() - delete(s.pendingTxSubs, id) - s.muPendingTxSubs.Unlock() - }) - - if err != nil { - return nil, err - } - - s.muPendingTxSubs.Lock() - s.pendingTxSubs[subscription.ID()] = subscription - s.muPendingTxSubs.Unlock() - - return subscription, nil -} - -// Resend accepts an existing transaction and a new gas price and limit. It will remove the given transaction from the -// pool and reinsert it with the new gas price and limit. -func (s *PublicTransactionPoolAPI) Resend(tx Tx, gasPrice, gasLimit *rpc.HexNumber) (common.Hash, error) { - - pending := s.txPool.GetTransactions() - for _, p := range pending { - if pFrom, err := p.FromFrontier(); err == nil && pFrom == tx.From && p.SigHash() == tx.tx.SigHash() { - if gasPrice == nil { - gasPrice = rpc.NewHexNumber(tx.tx.GasPrice()) - } - if gasLimit == nil { - gasLimit = rpc.NewHexNumber(tx.tx.Gas()) - } - - var newTx *types.Transaction - if tx.tx.To() == nil { - newTx = types.NewContractCreation(tx.tx.Nonce(), tx.tx.Value(), gasPrice.BigInt(), gasLimit.BigInt(), tx.tx.Data()) - } else { - newTx = types.NewTransaction(tx.tx.Nonce(), *tx.tx.To(), tx.tx.Value(), gasPrice.BigInt(), gasLimit.BigInt(), tx.tx.Data()) - } - - signedTx, err := s.sign(tx.From, newTx) - if err != nil { - return common.Hash{}, err - } - - s.txPool.RemoveTx(tx.Hash) - if err = s.txPool.Add(signedTx); err != nil { - return common.Hash{}, err - } - - return signedTx.Hash(), nil - } - } - - return common.Hash{}, fmt.Errorf("Transaction %#x not found", tx.Hash) -} - -// PrivateAdminAPI is the collection of Etheruem APIs exposed over the private -// admin endpoint. +// PrivateAdminAPI is the collection of Etheruem full node-related APIs +// exposed over the private admin endpoint. type PrivateAdminAPI struct { eth *Ethereum } -// NewPrivateAdminAPI creates a new API definition for the private admin methods -// of the Ethereum service. +// NewPrivateAdminAPI creates a new API definition for the full node private +// admin methods of the Ethereum service. func NewPrivateAdminAPI(eth *Ethereum) *PrivateAdminAPI { return &PrivateAdminAPI{eth: eth} } -// SetSolc sets the Solidity compiler path to be used by the node. -func (api *PrivateAdminAPI) SetSolc(path string) (string, error) { - solc, err := api.eth.SetSolc(path) - if err != nil { - return "", err - } - return solc.Info(), nil -} - // ExportChain exports the current blockchain into a local file. func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { // Make sure we can create the file to export into @@ -1562,14 +271,14 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { return true, nil } -// PublicDebugAPI is the collection of Etheruem APIs exposed over the public -// debugging endpoint. +// PublicDebugAPI is the collection of Etheruem full node APIs exposed +// over the public debugging endpoint. type PublicDebugAPI struct { eth *Ethereum } -// NewPublicDebugAPI creates a new API definition for the public debug methods -// of the Ethereum service. +// NewPublicDebugAPI creates a new API definition for the full node- +// related public debug methods of the Ethereum service. func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { return &PublicDebugAPI{eth: eth} } @@ -1587,76 +296,25 @@ func (api *PublicDebugAPI) DumpBlock(number uint64) (state.World, error) { return stateDb.RawDump(), nil } -// GetBlockRlp retrieves the RLP encoded for of a single block. -func (api *PublicDebugAPI) GetBlockRlp(number uint64) (string, error) { - block := api.eth.BlockChain().GetBlockByNumber(number) - if block == nil { - return "", fmt.Errorf("block #%d not found", number) - } - encoded, err := rlp.EncodeToBytes(block) - if err != nil { - return "", err - } - return fmt.Sprintf("%x", encoded), nil -} - -// PrintBlock retrieves a block and returns its pretty printed form. -func (api *PublicDebugAPI) PrintBlock(number uint64) (string, error) { - block := api.eth.BlockChain().GetBlockByNumber(number) - if block == nil { - return "", fmt.Errorf("block #%d not found", number) - } - return fmt.Sprintf("%s", block), nil -} - -// SeedHash retrieves the seed hash of a block. -func (api *PublicDebugAPI) SeedHash(number uint64) (string, error) { - block := api.eth.BlockChain().GetBlockByNumber(number) - if block == nil { - return "", fmt.Errorf("block #%d not found", number) - } - hash, err := ethash.GetSeedHash(number) - if err != nil { - return "", err - } - return fmt.Sprintf("0x%x", hash), nil -} - -// PrivateDebugAPI is the collection of Etheruem APIs exposed over the private -// debugging endpoint. +// PrivateDebugAPI is the collection of Etheruem full node APIs exposed over +// the private debugging endpoint. type PrivateDebugAPI struct { config *core.ChainConfig eth *Ethereum } -// NewPrivateDebugAPI creates a new API definition for the private debug methods -// of the Ethereum service. +// NewPrivateDebugAPI creates a new API definition for the full node-related +// private debug methods of the Ethereum service. func NewPrivateDebugAPI(config *core.ChainConfig, eth *Ethereum) *PrivateDebugAPI { return &PrivateDebugAPI{config: config, eth: eth} } -// ChaindbProperty returns leveldb properties of the chain database. -func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) { - ldb, ok := api.eth.chainDb.(interface { - LDB() *leveldb.DB - }) - if !ok { - return "", fmt.Errorf("chaindbProperty does not work for memory databases") - } - if property == "" { - property = "leveldb.stats" - } else if !strings.HasPrefix(property, "leveldb.") { - property = "leveldb." + property - } - return ldb.LDB().GetProperty(property) -} - // BlockTraceResult is the returned value when replaying a block to check for // consensus results and full VM trace logs for all included transactions. type BlockTraceResult struct { - Validated bool `json:"validated"` - StructLogs []structLogRes `json:"structLogs"` - Error string `json:"error"` + Validated bool `json:"validated"` + StructLogs []ethapi.StructLogRes `json:"structLogs"` + Error string `json:"error"` } // TraceBlock processes the given block's RLP but does not import the block in to @@ -1671,7 +329,7 @@ func (api *PrivateDebugAPI) TraceBlock(blockRlp []byte, config *vm.Config) Block validated, logs, err := api.traceBlock(&block, config) return BlockTraceResult{ Validated: validated, - StructLogs: formatLogs(logs), + StructLogs: ethapi.FormatLogs(logs), Error: formatError(err), } } @@ -1697,7 +355,7 @@ func (api *PrivateDebugAPI) TraceBlockByNumber(number uint64, config *vm.Config) validated, logs, err := api.traceBlock(block, config) return BlockTraceResult{ Validated: validated, - StructLogs: formatLogs(logs), + StructLogs: ethapi.FormatLogs(logs), Error: formatError(err), } } @@ -1713,7 +371,7 @@ func (api *PrivateDebugAPI) TraceBlockByHash(hash common.Hash, config *vm.Config validated, logs, err := api.traceBlock(block, config) return BlockTraceResult{ Validated: validated, - StructLogs: formatLogs(logs), + StructLogs: ethapi.FormatLogs(logs), Error: formatError(err), } } @@ -1763,63 +421,25 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, config *vm.Config) (b return true, collector.traces, nil } -// SetHead rewinds the head of the blockchain to a previous block. -func (api *PrivateDebugAPI) SetHead(number uint64) { - api.eth.BlockChain().SetHead(number) -} - -// ExecutionResult groups all structured logs emitted by the EVM -// while replaying a transaction in debug mode as well as the amount of -// gas used and the return value -type ExecutionResult struct { - Gas *big.Int `json:"gas"` - ReturnValue string `json:"returnValue"` - StructLogs []structLogRes `json:"structLogs"` -} - -// structLogRes stores a structured log emitted by the EVM while replaying a -// transaction in debug mode -type structLogRes struct { - Pc uint64 `json:"pc"` - Op string `json:"op"` - Gas *big.Int `json:"gas"` - GasCost *big.Int `json:"gasCost"` - Depth int `json:"depth"` - Error string `json:"error"` - Stack []string `json:"stack"` - Memory []string `json:"memory"` - Storage map[string]string `json:"storage"` +// callmsg is the message type used for call transations. +type callmsg struct { + addr common.Address + to *common.Address + gas, gasPrice *big.Int + value *big.Int + data []byte } -// formatLogs formats EVM returned structured logs for json output -func formatLogs(structLogs []vm.StructLog) []structLogRes { - formattedStructLogs := make([]structLogRes, len(structLogs)) - for index, trace := range structLogs { - formattedStructLogs[index] = structLogRes{ - Pc: trace.Pc, - Op: trace.Op.String(), - Gas: trace.Gas, - GasCost: trace.GasCost, - Depth: trace.Depth, - Error: formatError(trace.Err), - Stack: make([]string, len(trace.Stack)), - Storage: make(map[string]string), - } - - for i, stackValue := range trace.Stack { - formattedStructLogs[index].Stack[i] = fmt.Sprintf("%x", common.LeftPadBytes(stackValue.Bytes(), 32)) - } - - for i := 0; i+32 <= len(trace.Memory); i += 32 { - formattedStructLogs[index].Memory = append(formattedStructLogs[index].Memory, fmt.Sprintf("%x", trace.Memory[i:i+32])) - } - - for i, storageValue := range trace.Storage { - formattedStructLogs[index].Storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) - } - } - return formattedStructLogs -} +// accessor boilerplate to implement core.Message +func (m callmsg) From() (common.Address, error) { return m.addr, nil } +func (m callmsg) FromFrontier() (common.Address, error) { return m.addr, nil } +func (m callmsg) Nonce() uint64 { return 0 } +func (m callmsg) CheckNonce() bool { return false } +func (m callmsg) To() *common.Address { return m.to } +func (m callmsg) GasPrice() *big.Int { return m.gasPrice } +func (m callmsg) Gas() *big.Int { return m.gas } +func (m callmsg) Value() *big.Int { return m.value } +func (m callmsg) Data() []byte { return m.data } // formatError formats a Go error into either an empty string or the data content // of the error itself. @@ -1832,7 +452,7 @@ func formatError(err error) string { // TraceTransaction returns the structured logs created during the execution of EVM // and returns them as a JSON object. -func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogConfig) (*ExecutionResult, error) { +func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogConfig) (*ethapi.ExecutionResult, error) { if logger == nil { logger = new(vm.LogConfig) } @@ -1862,7 +482,7 @@ func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogC return nil, fmt.Errorf("sender retrieval failed: %v", err) } msg := callmsg{ - from: stateDb.GetOrNewStateObject(from), + addr: from, to: tx.To(), gas: tx.Gas(), gasPrice: tx.GasPrice(), @@ -1885,90 +505,11 @@ func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogC if err != nil { return nil, fmt.Errorf("tracing failed: %v", err) } - return &ExecutionResult{ + return ðapi.ExecutionResult{ Gas: gas, ReturnValue: fmt.Sprintf("%x", ret), - StructLogs: formatLogs(vmenv.StructLogs()), + StructLogs: ethapi.FormatLogs(vmenv.StructLogs()), }, nil } return nil, errors.New("database inconsistency") } - -// TraceCall executes a call and returns the amount of gas, created logs and optionally returned values. -func (s *PublicBlockChainAPI) TraceCall(args CallArgs, blockNr rpc.BlockNumber) (*ExecutionResult, error) { - // Fetch the state associated with the block number - stateDb, block, err := stateAndBlockByNumber(s.miner, s.bc, blockNr, s.chainDb) - if stateDb == nil || err != nil { - return nil, err - } - stateDb = stateDb.Copy() - - // Retrieve the account state object to interact with - var from *state.StateObject - if args.From == (common.Address{}) { - accounts := s.am.Accounts() - if len(accounts) == 0 { - from = stateDb.GetOrNewStateObject(common.Address{}) - } else { - from = stateDb.GetOrNewStateObject(accounts[0].Address) - } - } else { - from = stateDb.GetOrNewStateObject(args.From) - } - from.SetBalance(common.MaxBig) - - // Assemble the CALL invocation - msg := callmsg{ - from: from, - to: args.To, - gas: args.Gas.BigInt(), - gasPrice: args.GasPrice.BigInt(), - value: args.Value.BigInt(), - data: common.FromHex(args.Data), - } - if msg.gas.Cmp(common.Big0) == 0 { - msg.gas = big.NewInt(50000000) - } - if msg.gasPrice.Cmp(common.Big0) == 0 { - msg.gasPrice = new(big.Int).Mul(big.NewInt(50), common.Shannon) - } - - // Execute the call and return - vmenv := core.NewEnv(stateDb, s.config, s.bc, msg, block.Header(), vm.Config{ - Debug: true, - }) - gp := new(core.GasPool).AddGas(common.MaxBig) - - ret, gas, err := core.ApplyMessage(vmenv, msg, gp) - return &ExecutionResult{ - Gas: gas, - ReturnValue: fmt.Sprintf("%x", ret), - StructLogs: formatLogs(vmenv.StructLogs()), - }, nil -} - -// PublicNetAPI offers network related RPC methods -type PublicNetAPI struct { - net *p2p.Server - networkVersion int -} - -// NewPublicNetAPI creates a new net API instance. -func NewPublicNetAPI(net *p2p.Server, networkVersion int) *PublicNetAPI { - return &PublicNetAPI{net, networkVersion} -} - -// Listening returns an indication if the node is listening for network connections. -func (s *PublicNetAPI) Listening() bool { - return true // always listening -} - -// PeerCount returns the number of connected peers -func (s *PublicNetAPI) PeerCount() *rpc.HexNumber { - return rpc.NewHexNumber(s.net.PeerCount()) -} - -// Version returns the current ethereum protocol version. -func (s *PublicNetAPI) Version() string { - return fmt.Sprintf("%d", s.networkVersion) -} diff --git a/eth/api_backend.go b/eth/api_backend.go new file mode 100644 index 000000000..efcdb3361 --- /dev/null +++ b/eth/api_backend.go @@ -0,0 +1,201 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package eth + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/ethapi" + rpc "github.com/ethereum/go-ethereum/rpc" + "golang.org/x/net/context" +) + +// EthApiBackend implements ethapi.Backend for full nodes +type EthApiBackend struct { + eth *Ethereum + gpo *gasprice.GasPriceOracle +} + +func (b *EthApiBackend) SetHead(number uint64) { + b.eth.blockchain.SetHead(number) +} + +func (b *EthApiBackend) HeaderByNumber(blockNr rpc.BlockNumber) *types.Header { + // Pending block is only known by the miner + if blockNr == rpc.PendingBlockNumber { + block, _ := b.eth.miner.Pending() + return block.Header() + } + // Otherwise resolve and return the block + if blockNr == rpc.LatestBlockNumber { + return b.eth.blockchain.CurrentBlock().Header() + } + return b.eth.blockchain.GetHeaderByNumber(uint64(blockNr)) +} + +func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { + // Pending block is only known by the miner + if blockNr == rpc.PendingBlockNumber { + block, _ := b.eth.miner.Pending() + return block, nil + } + // Otherwise resolve and return the block + if blockNr == rpc.LatestBlockNumber { + return b.eth.blockchain.CurrentBlock(), nil + } + return b.eth.blockchain.GetBlockByNumber(uint64(blockNr)), nil +} + +func (b *EthApiBackend) StateAndHeaderByNumber(blockNr rpc.BlockNumber) (ethapi.State, *types.Header, error) { + // Pending state is only known by the miner + if blockNr == rpc.PendingBlockNumber { + block, state := b.eth.miner.Pending() + return EthApiState{state}, block.Header(), nil + } + // Otherwise resolve the block number and return its state + header := b.HeaderByNumber(blockNr) + if header == nil { + return nil, nil, nil + } + stateDb, err := state.New(header.Root, b.eth.chainDb) + return EthApiState{stateDb}, header, err +} + +func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { + return b.eth.blockchain.GetBlockByHash(blockHash), nil +} + +func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { + return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil +} + +func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { + return b.eth.blockchain.GetTdByHash(blockHash) +} + +func (b *EthApiBackend) GetVMEnv(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header) (vm.Environment, func() error, error) { + stateDb := state.(EthApiState).state.Copy() + addr, _ := msg.From() + from := stateDb.GetOrNewStateObject(addr) + from.SetBalance(common.MaxBig) + vmError := func() error { return nil } + return core.NewEnv(stateDb, b.eth.chainConfig, b.eth.blockchain, msg, header, b.eth.chainConfig.VmConfig), vmError, nil +} + +func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + b.eth.txPool.SetLocal(signedTx) + return b.eth.txPool.Add(signedTx) +} + +func (b *EthApiBackend) RemoveTx(txHash common.Hash) { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + b.eth.txPool.RemoveTx(txHash) +} + +func (b *EthApiBackend) GetPoolTransactions() types.Transactions { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + return b.eth.txPool.GetTransactions() +} + +func (b *EthApiBackend) GetPoolTransaction(txHash common.Hash) *types.Transaction { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + return b.eth.txPool.GetTransaction(txHash) +} + +func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + return b.eth.txPool.State().GetNonce(addr), nil +} + +func (b *EthApiBackend) Stats() (pending int, queued int) { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + return b.eth.txPool.Stats() +} + +func (b *EthApiBackend) TxPoolContent() (map[common.Address]map[uint64][]*types.Transaction, map[common.Address]map[uint64][]*types.Transaction) { + b.eth.txMu.Lock() + defer b.eth.txMu.Unlock() + + return b.eth.TxPool().Content() +} + +func (b *EthApiBackend) Downloader() *downloader.Downloader { + return b.eth.Downloader() +} + +func (b *EthApiBackend) ProtocolVersion() int { + return b.eth.EthVersion() +} + +func (b *EthApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { + return b.gpo.SuggestPrice(), nil +} + +func (b *EthApiBackend) ChainDb() ethdb.Database { + return b.eth.ChainDb() +} + +func (b *EthApiBackend) EventMux() *event.TypeMux { + return b.eth.EventMux() +} + +func (b *EthApiBackend) AccountManager() *accounts.Manager { + return b.eth.AccountManager() +} + +type EthApiState struct { + state *state.StateDB +} + +func (s EthApiState) GetBalance(ctx context.Context, addr common.Address) (*big.Int, error) { + return s.state.GetBalance(addr), nil +} + +func (s EthApiState) GetCode(ctx context.Context, addr common.Address) ([]byte, error) { + return s.state.GetCode(addr), nil +} + +func (s EthApiState) GetState(ctx context.Context, a common.Address, b common.Hash) (common.Hash, error) { + return s.state.GetState(a, b), nil +} + +func (s EthApiState) GetNonce(ctx context.Context, addr common.Address) (uint64, error) { + return s.state.GetNonce(addr), nil +} diff --git a/eth/backend.go b/eth/backend.go index 006523484..c8a9af6ee 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -39,8 +39,10 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/miner" @@ -101,95 +103,86 @@ type Config struct { TestGenesisState ethdb.Database // Genesis state to seed the database with (testing only!) } +// Ethereum implements the Ethereum full node service. type Ethereum struct { - chainConfig *core.ChainConfig + chainConfig *core.ChainConfig + // Channel for shutting down the service shutdownChan chan bool // Channel for shutting down the ethereum stopDbUpgrade func() // stop chain db sequential key upgrade - - // DB interfaces - chainDb ethdb.Database // Block chain database - dappDb ethdb.Database // Dapp database - // Handlers txPool *core.TxPool txMu sync.Mutex blockchain *core.BlockChain - accountManager *accounts.Manager - pow *ethash.Ethash protocolManager *ProtocolManager - SolcPath string - solc *compiler.Solidity - gpo *GasPriceOracle + // DB interfaces + chainDb ethdb.Database // Block chain database + dappDb ethdb.Database // Dapp database - GpoMinGasPrice *big.Int - GpoMaxGasPrice *big.Int - GpoFullBlockRatio int - GpobaseStepDown int - GpobaseStepUp int - GpobaseCorrectionFactor int + eventMux *event.TypeMux + pow *ethash.Ethash + httpclient *httpclient.HTTPClient + accountManager *accounts.Manager - httpclient *httpclient.HTTPClient + apiBackend *EthApiBackend - eventMux *event.TypeMux - miner *miner.Miner + miner *miner.Miner + Mining bool + MinerThreads int + AutoDAG bool + autodagquit chan bool + etherbase common.Address + solcPath string + solc *compiler.Solidity - Mining bool - MinerThreads int NatSpec bool - AutoDAG bool PowTest bool - autodagquit chan bool - etherbase common.Address netVersionId int - netRPCService *PublicNetAPI + netRPCService *ethapi.PublicNetAPI } +// New creates a new Ethereum object (including the +// initialisation of the common Ethereum object) func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { - // Open the chain database and perform any upgrades needed - chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles) + chainDb, dappDb, err := CreateDBs(ctx, config) if err != nil { return nil, err } - if db, ok := chainDb.(*ethdb.LDBDatabase); ok { - db.Meter("eth/db/chaindata/") - } - if err := upgradeChainDatabase(chainDb); err != nil { - return nil, err - } - if err := addMipmapBloomBins(chainDb); err != nil { + stopDbUpgrade := upgradeSequentialKeys(chainDb) + if err := SetupGenesisBlock(&chainDb, config); err != nil { return nil, err } - stopDbUpgrade := upgradeSequentialKeys(chainDb) - - dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache, config.DatabaseHandles) + pow, err := CreatePoW(config) if err != nil { return nil, err } - if db, ok := dappDb.(*ethdb.LDBDatabase); ok { - db.Meter("eth/db/dapp/") - } - glog.V(logger.Info).Infof("Protocol Versions: %v, Network Id: %v", ProtocolVersions, config.NetworkId) - // Load up any custom genesis block if requested - if len(config.Genesis) > 0 { - block, err := core.WriteGenesisBlock(chainDb, strings.NewReader(config.Genesis)) - if err != nil { - return nil, err - } - glog.V(logger.Info).Infof("Successfully wrote custom genesis block: %x", block.Hash()) + eth := &Ethereum{ + chainDb: chainDb, + dappDb: dappDb, + eventMux: ctx.EventMux, + accountManager: config.AccountManager, + pow: pow, + shutdownChan: make(chan bool), + stopDbUpgrade: stopDbUpgrade, + httpclient: httpclient.New(config.DocRoot), + netVersionId: config.NetworkId, + NatSpec: config.NatSpec, + PowTest: config.PowTest, + etherbase: config.Etherbase, + MinerThreads: config.MinerThreads, + AutoDAG: config.AutoDAG, + solcPath: config.SolcPath, } - // Load up a test setup if directly injected - if config.TestGenesisState != nil { - chainDb = config.TestGenesisState + if err := upgradeChainDatabase(chainDb); err != nil { + return nil, err } - if config.TestGenesisBlock != nil { - core.WriteTd(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64(), config.TestGenesisBlock.Difficulty()) - core.WriteBlock(chainDb, config.TestGenesisBlock) - core.WriteCanonicalHash(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64()) - core.WriteHeadBlockHash(chainDb, config.TestGenesisBlock.Hash()) + if err := addMipmapBloomBins(chainDb); err != nil { + return nil, err } + glog.V(logger.Info).Infof("Protocol Versions: %v, Network Id: %v", ProtocolVersions, config.NetworkId) + if !config.SkipBcVersionCheck { bcVersion := core.GetBlockChainVersion(chainDb) if bcVersion != config.BlockChainVersion && bcVersion != 0 { @@ -197,44 +190,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } core.WriteBlockChainVersion(chainDb, config.BlockChainVersion) } - glog.V(logger.Info).Infof("Blockchain DB Version: %d", config.BlockChainVersion) - - eth := &Ethereum{ - shutdownChan: make(chan bool), - stopDbUpgrade: stopDbUpgrade, - chainDb: chainDb, - dappDb: dappDb, - eventMux: ctx.EventMux, - accountManager: config.AccountManager, - etherbase: config.Etherbase, - netVersionId: config.NetworkId, - NatSpec: config.NatSpec, - MinerThreads: config.MinerThreads, - SolcPath: config.SolcPath, - AutoDAG: config.AutoDAG, - PowTest: config.PowTest, - GpoMinGasPrice: config.GpoMinGasPrice, - GpoMaxGasPrice: config.GpoMaxGasPrice, - GpoFullBlockRatio: config.GpoFullBlockRatio, - GpobaseStepDown: config.GpobaseStepDown, - GpobaseStepUp: config.GpobaseStepUp, - GpobaseCorrectionFactor: config.GpobaseCorrectionFactor, - httpclient: httpclient.New(config.DocRoot), - } - switch { - case config.PowTest: - glog.V(logger.Info).Infof("ethash used in test mode") - eth.pow, err = ethash.NewForTesting() - if err != nil { - return nil, err - } - case config.PowShared: - glog.V(logger.Info).Infof("ethash used in shared mode") - eth.pow = ethash.NewShared() - - default: - eth.pow = ethash.New() - } // load the genesis block or write a new one if no genesis // block is prenent in the database. @@ -250,6 +205,8 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if config.ChainConfig == nil { return nil, errors.New("missing chain config") } + core.WriteChainConfig(chainDb, genesis.Hash(), config.ChainConfig) + eth.chainConfig = config.ChainConfig eth.chainConfig.VmConfig = vm.Config{ EnableJit: config.EnableJit, @@ -263,8 +220,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } return nil, err } - eth.gpo = NewGasPriceOracle(eth) - newPool := core.NewTxPool(eth.chainConfig, eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit) eth.txPool = newPool @@ -275,13 +230,83 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.miner.SetGasPrice(config.GasPrice) eth.miner.SetExtra(config.ExtraData) + gpoParams := &gasprice.GpoParams{ + GpoMinGasPrice: config.GpoMinGasPrice, + GpoMaxGasPrice: config.GpoMaxGasPrice, + GpoFullBlockRatio: config.GpoFullBlockRatio, + GpobaseStepDown: config.GpobaseStepDown, + GpobaseStepUp: config.GpobaseStepUp, + GpobaseCorrectionFactor: config.GpobaseCorrectionFactor, + } + gpo := gasprice.NewGasPriceOracle(eth.blockchain, chainDb, eth.eventMux, gpoParams) + eth.apiBackend = &EthApiBackend{eth, gpo} + return eth, nil } +// CreateDBs creates the chain and dapp databases for an Ethereum service +func CreateDBs(ctx *node.ServiceContext, config *Config) (chainDb, dappDb ethdb.Database, err error) { + // Open the chain database and perform any upgrades needed + chainDb, err = ctx.OpenDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles) + if err != nil { + return nil, nil, err + } + if db, ok := chainDb.(*ethdb.LDBDatabase); ok { + db.Meter("eth/db/chaindata/") + } + + dappDb, err = ctx.OpenDatabase("dapp", config.DatabaseCache, config.DatabaseHandles) + if err != nil { + return nil, nil, err + } + if db, ok := dappDb.(*ethdb.LDBDatabase); ok { + db.Meter("eth/db/dapp/") + } + return +} + +// SetupGenesisBlock initializes the genesis block for an Ethereum service +func SetupGenesisBlock(chainDb *ethdb.Database, config *Config) error { + // Load up any custom genesis block if requested + if len(config.Genesis) > 0 { + block, err := core.WriteGenesisBlock(*chainDb, strings.NewReader(config.Genesis)) + if err != nil { + return err + } + glog.V(logger.Info).Infof("Successfully wrote custom genesis block: %x", block.Hash()) + } + // Load up a test setup if directly injected + if config.TestGenesisState != nil { + *chainDb = config.TestGenesisState + } + if config.TestGenesisBlock != nil { + core.WriteTd(*chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64(), config.TestGenesisBlock.Difficulty()) + core.WriteBlock(*chainDb, config.TestGenesisBlock) + core.WriteCanonicalHash(*chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64()) + core.WriteHeadBlockHash(*chainDb, config.TestGenesisBlock.Hash()) + } + return nil +} + +// CreatePoW creates the required type of PoW instance for an Ethereum service +func CreatePoW(config *Config) (*ethash.Ethash, error) { + switch { + case config.PowTest: + glog.V(logger.Info).Infof("ethash used in test mode") + return ethash.NewForTesting() + case config.PowShared: + glog.V(logger.Info).Infof("ethash used in shared mode") + return ethash.NewShared(), nil + + default: + return ethash.New(), nil + } +} + // APIs returns the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *Ethereum) APIs() []rpc.API { - return []rpc.API{ + return append(ethapi.GetAPIs(s.apiBackend, &s.solcPath, &s.solc), []rpc.API{ { Namespace: "eth", Version: "1.0", @@ -290,26 +315,6 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: NewPublicAccountAPI(s.accountManager), - Public: true, - }, { - Namespace: "personal", - Version: "1.0", - Service: NewPrivateAccountAPI(s), - Public: false, - }, { - Namespace: "eth", - Version: "1.0", - Service: NewPublicBlockChainAPI(s.chainConfig, s.blockchain, s.miner, s.chainDb, s.gpo, s.eventMux, s.accountManager), - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: NewPublicTransactionPoolAPI(s), - Public: true, - }, { - Namespace: "eth", - Version: "1.0", Service: NewPublicMinerAPI(s), Public: true, }, { @@ -323,11 +328,6 @@ func (s *Ethereum) APIs() []rpc.API { Service: NewPrivateMinerAPI(s), Public: false, }, { - Namespace: "txpool", - Version: "1.0", - Service: NewPublicTxPoolAPI(s), - Public: true, - }, { Namespace: "eth", Version: "1.0", Service: filters.NewPublicFilterAPI(s.chainDb, s.eventMux), @@ -355,7 +355,7 @@ func (s *Ethereum) APIs() []rpc.API { Version: "1.0", Service: ethreg.NewPrivateRegistarAPI(s.chainConfig, s.blockchain, s.chainDb, s.txPool, s.accountManager), }, - } + }...) } func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) { @@ -388,6 +388,7 @@ func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } func (s *Ethereum) TxPool() *core.TxPool { return s.txPool } func (s *Ethereum) EventMux() *event.TypeMux { return s.eventMux } +func (s *Ethereum) Pow() *ethash.Ethash { return s.pow } func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) DappDb() ethdb.Database { return s.dappDb } func (s *Ethereum) IsListening() bool { return true } // Always listening @@ -404,11 +405,11 @@ func (s *Ethereum) Protocols() []p2p.Protocol { // Start implements node.Service, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *Ethereum) Start(srvr *p2p.Server) error { + s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) if s.AutoDAG { s.StartAutoDAG() } s.protocolManager.Start() - s.netRPCService = NewPublicNetAPI(srvr, s.NetVersion()) return nil } @@ -507,21 +508,6 @@ func (self *Ethereum) HTTPClient() *httpclient.HTTPClient { return self.httpclient } -func (self *Ethereum) Solc() (*compiler.Solidity, error) { - var err error - if self.solc == nil { - self.solc, err = compiler.New(self.SolcPath) - } - return self.solc, err -} - -// set in js console via admin interface or wrapper from cli flags -func (self *Ethereum) SetSolc(solcPath string) (*compiler.Solidity, error) { - self.SolcPath = solcPath - self.solc = nil - return self.Solc() -} - // dagFiles(epoch) returns the two alternative DAG filenames (not a path) // 1) <revision>-<hex(seedhash[8])> 2) full-R<revision>-<hex(seedhash[8])> func dagFiles(epoch uint64) (string, string) { diff --git a/eth/backend_test.go b/eth/backend_test.go index cb94adbf0..105d71080 100644 --- a/eth/backend_test.go +++ b/eth/backend_test.go @@ -32,7 +32,7 @@ func TestMipmapUpgrade(t *testing.T) { addr := common.BytesToAddress([]byte("jeff")) genesis := core.WriteGenesisBlockForTesting(db) - chain, receipts := core.GenerateChain(genesis, db, 10, func(i int, gen *core.BlockGen) { + chain, receipts := core.GenerateChain(nil, genesis, db, 10, func(i int, gen *core.BlockGen) { var receipts types.Receipts switch i { case 1: diff --git a/eth/bind.go b/eth/bind.go index fb7f29f60..c1366464f 100644 --- a/eth/bind.go +++ b/eth/bind.go @@ -21,8 +21,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + "golang.org/x/net/context" ) // ContractBackend implements bind.ContractBackend with direct calls to Ethereum @@ -33,38 +35,44 @@ import ( // object. These should be rewritten to internal Go method calls when the Go API // is refactored to support a clean library use. type ContractBackend struct { - eapi *PublicEthereumAPI // Wrapper around the Ethereum object to access metadata - bcapi *PublicBlockChainAPI // Wrapper around the blockchain to access chain data - txapi *PublicTransactionPoolAPI // Wrapper around the transaction pool to access transaction data + eapi *ethapi.PublicEthereumAPI // Wrapper around the Ethereum object to access metadata + bcapi *ethapi.PublicBlockChainAPI // Wrapper around the blockchain to access chain data + txapi *ethapi.PublicTransactionPoolAPI // Wrapper around the transaction pool to access transaction data } // NewContractBackend creates a new native contract backend using an existing // Etheruem object. func NewContractBackend(eth *Ethereum) *ContractBackend { return &ContractBackend{ - eapi: NewPublicEthereumAPI(eth), - bcapi: NewPublicBlockChainAPI(eth.chainConfig, eth.blockchain, eth.miner, eth.chainDb, eth.gpo, eth.eventMux, eth.accountManager), - txapi: NewPublicTransactionPoolAPI(eth), + eapi: ethapi.NewPublicEthereumAPI(eth.apiBackend, nil, nil), + bcapi: ethapi.NewPublicBlockChainAPI(eth.apiBackend), + txapi: ethapi.NewPublicTransactionPoolAPI(eth.apiBackend), } } // HasCode implements bind.ContractVerifier.HasCode by retrieving any code associated // with the contract from the local API, and checking its size. -func (b *ContractBackend) HasCode(contract common.Address, pending bool) (bool, error) { +func (b *ContractBackend) HasCode(ctx context.Context, contract common.Address, pending bool) (bool, error) { + if ctx == nil { + ctx = context.Background() + } block := rpc.LatestBlockNumber if pending { block = rpc.PendingBlockNumber } - out, err := b.bcapi.GetCode(contract, block) + out, err := b.bcapi.GetCode(ctx, contract, block) return len(common.FromHex(out)) > 0, err } // ContractCall implements bind.ContractCaller executing an Ethereum contract // call with the specified data as the input. The pending flag requests execution // against the pending block, not the stable head of the chain. -func (b *ContractBackend) ContractCall(contract common.Address, data []byte, pending bool) ([]byte, error) { +func (b *ContractBackend) ContractCall(ctx context.Context, contract common.Address, data []byte, pending bool) ([]byte, error) { + if ctx == nil { + ctx = context.Background() + } // Convert the input args to the API spec - args := CallArgs{ + args := ethapi.CallArgs{ To: &contract, Data: common.ToHex(data), } @@ -73,21 +81,27 @@ func (b *ContractBackend) ContractCall(contract common.Address, data []byte, pen block = rpc.PendingBlockNumber } // Execute the call and convert the output back to Go types - out, err := b.bcapi.Call(args, block) + out, err := b.bcapi.Call(ctx, args, block) return common.FromHex(out), err } // PendingAccountNonce implements bind.ContractTransactor retrieving the current // pending nonce associated with an account. -func (b *ContractBackend) PendingAccountNonce(account common.Address) (uint64, error) { - out, err := b.txapi.GetTransactionCount(account, rpc.PendingBlockNumber) +func (b *ContractBackend) PendingAccountNonce(ctx context.Context, account common.Address) (uint64, error) { + if ctx == nil { + ctx = context.Background() + } + out, err := b.txapi.GetTransactionCount(ctx, account, rpc.PendingBlockNumber) return out.Uint64(), err } // SuggestGasPrice implements bind.ContractTransactor retrieving the currently // suggested gas price to allow a timely execution of a transaction. -func (b *ContractBackend) SuggestGasPrice() (*big.Int, error) { - return b.eapi.GasPrice(), nil +func (b *ContractBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { + if ctx == nil { + ctx = context.Background() + } + return b.eapi.GasPrice(ctx) } // EstimateGasLimit implements bind.ContractTransactor triing to estimate the gas @@ -95,8 +109,11 @@ func (b *ContractBackend) SuggestGasPrice() (*big.Int, error) { // the backend blockchain. There is no guarantee that this is the true gas limit // requirement as other transactions may be added or removed by miners, but it // should provide a basis for setting a reasonable default. -func (b *ContractBackend) EstimateGasLimit(sender common.Address, contract *common.Address, value *big.Int, data []byte) (*big.Int, error) { - out, err := b.bcapi.EstimateGas(CallArgs{ +func (b *ContractBackend) EstimateGasLimit(ctx context.Context, sender common.Address, contract *common.Address, value *big.Int, data []byte) (*big.Int, error) { + if ctx == nil { + ctx = context.Background() + } + out, err := b.bcapi.EstimateGas(ctx, ethapi.CallArgs{ From: sender, To: contract, Value: *rpc.NewHexNumber(value), @@ -107,8 +124,11 @@ func (b *ContractBackend) EstimateGasLimit(sender common.Address, contract *comm // SendTransaction implements bind.ContractTransactor injects the transaction // into the pending pool for execution. -func (b *ContractBackend) SendTransaction(tx *types.Transaction) error { +func (b *ContractBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error { + if ctx == nil { + ctx = context.Background() + } raw, _ := rlp.EncodeToBytes(tx) - _, err := b.txapi.SendRawTransaction(common.ToHex(raw)) + _, err := b.txapi.SendRawTransaction(ctx, common.ToHex(raw)) return err } diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go index 12de60fe7..172bb0954 100644 --- a/eth/db_upgrade.go +++ b/eth/db_upgrade.go @@ -93,6 +93,9 @@ func upgradeSequentialKeys(db ethdb.Database) (stopFn func()) { func upgradeSequentialCanonicalNumbers(db ethdb.Database, stopFn func() bool) (error, bool) { prefix := []byte("block-num-") it := db.(*ethdb.LDBDatabase).NewIterator() + defer func() { + it.Release() + }() it.Seek(prefix) cnt := 0 for bytes.HasPrefix(it.Key(), prefix) { @@ -100,6 +103,9 @@ func upgradeSequentialCanonicalNumbers(db ethdb.Database, stopFn func() bool) (e if len(keyPtr) < 20 { cnt++ if cnt%100000 == 0 { + it.Release() + it = db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(keyPtr) glog.V(logger.Info).Infof("converting %d canonical numbers...", cnt) } number := big.NewInt(0).SetBytes(keyPtr[10:]).Uint64() @@ -130,6 +136,9 @@ func upgradeSequentialCanonicalNumbers(db ethdb.Database, stopFn func() bool) (e func upgradeSequentialBlocks(db ethdb.Database, stopFn func() bool) (error, bool) { prefix := []byte("block-") it := db.(*ethdb.LDBDatabase).NewIterator() + defer func() { + it.Release() + }() it.Seek(prefix) cnt := 0 for bytes.HasPrefix(it.Key(), prefix) { @@ -137,6 +146,9 @@ func upgradeSequentialBlocks(db ethdb.Database, stopFn func() bool) (error, bool if len(keyPtr) >= 38 { cnt++ if cnt%10000 == 0 { + it.Release() + it = db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(keyPtr) glog.V(logger.Info).Infof("converting %d blocks...", cnt) } // convert header, body, td and block receipts @@ -175,6 +187,7 @@ func upgradeSequentialBlocks(db ethdb.Database, stopFn func() bool) (error, bool func upgradeSequentialOrphanedReceipts(db ethdb.Database, stopFn func() bool) (error, bool) { prefix := []byte("receipts-block-") it := db.(*ethdb.LDBDatabase).NewIterator() + defer it.Release() it.Seek(prefix) cnt := 0 for bytes.HasPrefix(it.Key(), prefix) { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 92124cfeb..aee21122a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -48,23 +48,17 @@ var ( MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request MaxStateFetch = 384 // Amount of node state values to allow fetching per request - MaxForkAncestry = 3 * params.EpochDuration.Uint64() // Maximum chain reorganisation - - hashTTL = 3 * time.Second // [eth/61] Time it takes for a hash request to time out - blockTargetRTT = 3 * time.Second / 2 // [eth/61] Target time for completing a block retrieval request - blockTTL = 3 * blockTargetRTT // [eth/61] Maximum time allowance before a block request is considered expired - - rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests - rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests - rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value - ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion - ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts + MaxForkAncestry = 3 * params.EpochDuration.Uint64() // Maximum chain reorganisation + rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests + rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests + rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value + ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion + ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts qosTuningPeers = 5 // Number of peers to tune based on (best peers) qosConfidenceCap = 10 // Number of peers above which not to modify RTT confidence qosTuningImpact = 0.25 // Impact that a new tuning target has on the previous value - maxQueuedHashes = 32 * 1024 // [eth/61] Maximum number of hashes to queue for import (DOS protection) maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) maxHeadersProcess = 2048 // Number of header download results to import at once into the chain maxResultsProcess = 2048 // Number of content download results to import at once into the chain @@ -84,16 +78,13 @@ var ( errStallingPeer = errors.New("peer is stalling") errNoPeers = errors.New("no peers to keep download active") errTimeout = errors.New("timeout") - errEmptyHashSet = errors.New("empty hash set by peer") errEmptyHeaderSet = errors.New("empty header set by peer") errPeersUnavailable = errors.New("no peers available or all tried for download") - errAlreadyInPool = errors.New("hash already in pool") errInvalidAncestor = errors.New("retrieved ancestor is invalid") errInvalidChain = errors.New("retrieved hash chain is invalid") errInvalidBlock = errors.New("retrieved block is invalid") errInvalidBody = errors.New("retrieved block body is invalid") errInvalidReceipt = errors.New("retrieved receipt is invalid") - errCancelHashFetch = errors.New("hash download canceled (requested)") errCancelBlockFetch = errors.New("block download canceled (requested)") errCancelHeaderFetch = errors.New("block header download canceled (requested)") errCancelBodyFetch = errors.New("block body download canceled (requested)") @@ -102,6 +93,7 @@ var ( errCancelHeaderProcessing = errors.New("header processing canceled (requested)") errCancelContentProcessing = errors.New("content processing canceled (requested)") errNoSyncActive = errors.New("no sync active") + errTooOld = errors.New("peer doesn't speak recent enough protocol version (need version >= 62)") ) type Downloader struct { @@ -146,13 +138,10 @@ type Downloader struct { // Channels newPeerCh chan *peer - hashCh chan dataPack // [eth/61] Channel receiving inbound hashes - blockCh chan dataPack // [eth/61] Channel receiving inbound blocks headerCh chan dataPack // [eth/62] Channel receiving inbound block headers bodyCh chan dataPack // [eth/62] Channel receiving inbound block bodies receiptCh chan dataPack // [eth/63] Channel receiving inbound receipts stateCh chan dataPack // [eth/63] Channel receiving inbound node state data - blockWakeCh chan bool // [eth/61] Channel to signal the block fetcher of new tasks bodyWakeCh chan bool // [eth/62] Channel to signal the block body fetcher of new tasks receiptWakeCh chan bool // [eth/63] Channel to signal the receipt fetcher of new tasks stateWakeCh chan bool // [eth/63] Channel to signal the state fetcher of new tasks @@ -199,13 +188,10 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, ha rollback: rollback, dropPeer: dropPeer, newPeerCh: make(chan *peer, 1), - hashCh: make(chan dataPack, 1), - blockCh: make(chan dataPack, 1), headerCh: make(chan dataPack, 1), bodyCh: make(chan dataPack, 1), receiptCh: make(chan dataPack, 1), stateCh: make(chan dataPack, 1), - blockWakeCh: make(chan bool, 1), bodyWakeCh: make(chan bool, 1), receiptWakeCh: make(chan bool, 1), stateWakeCh: make(chan bool, 1), @@ -251,12 +237,11 @@ func (d *Downloader) Synchronising() bool { // RegisterPeer injects a new download peer into the set of block source to be // used for fetching hashes and blocks from. func (d *Downloader) RegisterPeer(id string, version int, head common.Hash, - getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, getReceipts receiptFetcherFn, getNodeData stateFetcherFn) error { glog.V(logger.Detail).Infoln("Registering peer", id) - if err := d.peers.Register(newPeer(id, version, head, getRelHashes, getAbsHashes, getBlocks, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)); err != nil { + if err := d.peers.Register(newPeer(id, version, head, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)); err != nil { glog.V(logger.Error).Infoln("Register failed:", err) return err } @@ -291,7 +276,9 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode case errBusy: glog.V(logger.Detail).Infof("Synchronisation already in progress") - case errTimeout, errBadPeer, errStallingPeer, errEmptyHashSet, errEmptyHeaderSet, errPeersUnavailable, errInvalidAncestor, errInvalidChain: + case errTimeout, errBadPeer, errStallingPeer, + errEmptyHeaderSet, errPeersUnavailable, errTooOld, + errInvalidAncestor, errInvalidChain: glog.V(logger.Debug).Infof("Removing peer %v: %v", id, err) d.dropPeer(id) @@ -323,13 +310,13 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode d.queue.Reset() d.peers.Reset() - for _, ch := range []chan bool{d.blockWakeCh, d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} { + for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} { select { case <-ch: default: } } - for _, ch := range []chan dataPack{d.hashCh, d.blockCh, d.headerCh, d.bodyCh, d.receiptCh, d.stateCh} { + for _, ch := range []chan dataPack{d.headerCh, d.bodyCh, d.receiptCh, d.stateCh} { for empty := false; !empty; { select { case <-ch: @@ -377,105 +364,73 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e d.mux.Post(DoneEvent{}) } }() + if p.version < 62 { + return errTooOld + } glog.V(logger.Debug).Infof("Synchronising with the network using: %s [eth/%d]", p.id, p.version) defer func(start time.Time) { glog.V(logger.Debug).Infof("Synchronisation terminated after %v", time.Since(start)) }(time.Now()) - switch { - case p.version == 61: - // Look up the sync boundaries: the common ancestor and the target block - latest, err := d.fetchHeight61(p) - if err != nil { - return err - } - origin, err := d.findAncestor61(p, latest) - if err != nil { - return err - } - d.syncStatsLock.Lock() - if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { - d.syncStatsChainOrigin = origin - } - d.syncStatsChainHeight = latest - d.syncStatsLock.Unlock() + // Look up the sync boundaries: the common ancestor and the target block + latest, err := d.fetchHeight(p) + if err != nil { + return err + } + height := latest.Number.Uint64() - // Initiate the sync using a concurrent hash and block retrieval algorithm - d.queue.Prepare(origin+1, d.mode, 0, nil) - if d.syncInitHook != nil { - d.syncInitHook(origin, latest) - } - return d.spawnSync(origin+1, - func() error { return d.fetchHashes61(p, td, origin+1) }, - func() error { return d.fetchBlocks61(origin + 1) }, - ) - - case p.version >= 62: - // Look up the sync boundaries: the common ancestor and the target block - latest, err := d.fetchHeight(p) - if err != nil { - return err - } - height := latest.Number.Uint64() + origin, err := d.findAncestor(p, height) + if err != nil { + return err + } + d.syncStatsLock.Lock() + if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { + d.syncStatsChainOrigin = origin + } + d.syncStatsChainHeight = height + d.syncStatsLock.Unlock() - origin, err := d.findAncestor(p, height) - if err != nil { - return err - } - d.syncStatsLock.Lock() - if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { - d.syncStatsChainOrigin = origin - } - d.syncStatsChainHeight = height - d.syncStatsLock.Unlock() - - // Initiate the sync using a concurrent header and content retrieval algorithm - pivot := uint64(0) - switch d.mode { - case LightSync: - pivot = height - case FastSync: - // Calculate the new fast/slow sync pivot point - if d.fsPivotLock == nil { - pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval))) - if err != nil { - panic(fmt.Sprintf("Failed to access crypto random source: %v", err)) - } - if height > uint64(fsMinFullBlocks)+pivotOffset.Uint64() { - pivot = height - uint64(fsMinFullBlocks) - pivotOffset.Uint64() - } - } else { - // Pivot point locked in, use this and do not pick a new one! - pivot = d.fsPivotLock.Number.Uint64() + // Initiate the sync using a concurrent header and content retrieval algorithm + pivot := uint64(0) + switch d.mode { + case LightSync: + pivot = height + case FastSync: + // Calculate the new fast/slow sync pivot point + if d.fsPivotLock == nil { + pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval))) + if err != nil { + panic(fmt.Sprintf("Failed to access crypto random source: %v", err)) } - // If the point is below the origin, move origin back to ensure state download - if pivot < origin { - if pivot > 0 { - origin = pivot - 1 - } else { - origin = 0 - } + if height > uint64(fsMinFullBlocks)+pivotOffset.Uint64() { + pivot = height - uint64(fsMinFullBlocks) - pivotOffset.Uint64() } - glog.V(logger.Debug).Infof("Fast syncing until pivot block #%d", pivot) + } else { + // Pivot point locked in, use this and do not pick a new one! + pivot = d.fsPivotLock.Number.Uint64() } - d.queue.Prepare(origin+1, d.mode, pivot, latest) - if d.syncInitHook != nil { - d.syncInitHook(origin, height) + // If the point is below the origin, move origin back to ensure state download + if pivot < origin { + if pivot > 0 { + origin = pivot - 1 + } else { + origin = 0 + } } - return d.spawnSync(origin+1, - func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved - func() error { return d.processHeaders(origin+1, td) }, // Headers are always retrieved - func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync - func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync - func() error { return d.fetchNodeData() }, // Node state data is retrieved during fast sync - ) - - default: - // Something very wrong, stop right here - glog.V(logger.Error).Infof("Unsupported eth protocol: %d", p.version) - return errBadPeer + glog.V(logger.Debug).Infof("Fast syncing until pivot block #%d", pivot) + } + d.queue.Prepare(origin+1, d.mode, pivot, latest) + if d.syncInitHook != nil { + d.syncInitHook(origin, height) } + return d.spawnSync(origin+1, + func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved + func() error { return d.processHeaders(origin+1, td) }, // Headers are always retrieved + func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync + func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync + func() error { return d.fetchNodeData() }, // Node state data is retrieved during fast sync + ) } // spawnSync runs d.process and all given fetcher functions to completion in @@ -540,452 +495,6 @@ func (d *Downloader) Terminate() { d.cancel() } -// fetchHeight61 retrieves the head block of the remote peer to aid in estimating -// the total time a pending synchronisation would take. -func (d *Downloader) fetchHeight61(p *peer) (uint64, error) { - glog.V(logger.Debug).Infof("%v: retrieving remote chain height", p) - - // Request the advertised remote head block and wait for the response - go p.getBlocks([]common.Hash{p.head}) - - timeout := time.After(hashTTL) - for { - select { - case <-d.cancelCh: - return 0, errCancelBlockFetch - - case packet := <-d.blockCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received blocks from incorrect peer(%s)", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - blocks := packet.(*blockPack).blocks - if len(blocks) != 1 { - glog.V(logger.Debug).Infof("%v: invalid number of head blocks: %d != 1", p, len(blocks)) - return 0, errBadPeer - } - return blocks[0].NumberU64(), nil - - case <-timeout: - glog.V(logger.Debug).Infof("%v: head block timeout", p) - return 0, errTimeout - - case <-d.hashCh: - // Out of bounds hashes received, ignore them - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } -} - -// findAncestor61 tries to locate the common ancestor block of the local chain and -// a remote peers blockchain. In the general case when our node was in sync and -// on the correct chain, checking the top N blocks should already get us a match. -// In the rare scenario when we ended up on a long reorganisation (i.e. none of -// the head blocks match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor61(p *peer, height uint64) (uint64, error) { - glog.V(logger.Debug).Infof("%v: looking for common ancestor", p) - - // Figure out the valid ancestor range to prevent rewrite attacks - floor, ceil := int64(-1), d.headBlock().NumberU64() - if ceil >= MaxForkAncestry { - floor = int64(ceil - MaxForkAncestry) - } - // Request the topmost blocks to short circuit binary ancestor lookup - head := ceil - if head > height { - head = height - } - from := int64(head) - int64(MaxHashFetch) + 1 - if from < 0 { - from = 0 - } - go p.getAbsHashes(uint64(from), MaxHashFetch) - - // Wait for the remote response to the head fetch - number, hash := uint64(0), common.Hash{} - timeout := time.After(hashTTL) - - for finished := false; !finished; { - select { - case <-d.cancelCh: - return 0, errCancelHashFetch - - case packet := <-d.hashCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - hashes := packet.(*hashPack).hashes - if len(hashes) == 0 { - glog.V(logger.Debug).Infof("%v: empty head hash set", p) - return 0, errEmptyHashSet - } - // Check if a common ancestor was found - finished = true - for i := len(hashes) - 1; i >= 0; i-- { - // Skip any headers that underflow/overflow our requested set - header := d.getHeader(hashes[i]) - if header == nil || header.Number.Int64() < from || header.Number.Uint64() > head { - continue - } - // Otherwise check if we already know the header or not - if d.hasBlockAndState(hashes[i]) { - number, hash = header.Number.Uint64(), header.Hash() - break - } - } - - case <-timeout: - glog.V(logger.Debug).Infof("%v: head hash timeout", p) - return 0, errTimeout - - case <-d.blockCh: - // Out of bounds blocks received, ignore them - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } - // If the head fetch already found an ancestor, return - if !common.EmptyHash(hash) { - if int64(number) <= floor { - glog.V(logger.Warn).Infof("%v: potential rewrite attack: #%d [%x…] <= #%d limit", p, number, hash[:4], floor) - return 0, errInvalidAncestor - } - glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, number, hash[:4]) - return number, nil - } - // Ancestor not found, we need to binary search over our chain - start, end := uint64(0), head - if floor > 0 { - start = uint64(floor) - } - for start+1 < end { - // Split our chain interval in two, and request the hash to cross check - check := (start + end) / 2 - - timeout := time.After(hashTTL) - go p.getAbsHashes(uint64(check), 1) - - // Wait until a reply arrives to this request - for arrived := false; !arrived; { - select { - case <-d.cancelCh: - return 0, errCancelHashFetch - - case packet := <-d.hashCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - hashes := packet.(*hashPack).hashes - if len(hashes) != 1 { - glog.V(logger.Debug).Infof("%v: invalid search hash set (%d)", p, len(hashes)) - return 0, errBadPeer - } - arrived = true - - // Modify the search interval based on the response - if !d.hasBlockAndState(hashes[0]) { - end = check - break - } - block := d.getBlock(hashes[0]) // this doesn't check state, hence the above explicit check - if block.NumberU64() != check { - glog.V(logger.Debug).Infof("%v: non requested hash #%d [%x…], instead of #%d", p, block.NumberU64(), block.Hash().Bytes()[:4], check) - return 0, errBadPeer - } - start = check - - case <-timeout: - glog.V(logger.Debug).Infof("%v: search hash timeout", p) - return 0, errTimeout - - case <-d.blockCh: - // Out of bounds blocks received, ignore them - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } - } - // Ensure valid ancestry and return - if int64(start) <= floor { - glog.V(logger.Warn).Infof("%v: potential rewrite attack: #%d [%x…] <= #%d limit", p, start, hash[:4], floor) - return 0, errInvalidAncestor - } - glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, start, hash[:4]) - return start, nil -} - -// fetchHashes61 keeps retrieving hashes from the requested number, until no more -// are returned, potentially throttling on the way. -func (d *Downloader) fetchHashes61(p *peer, td *big.Int, from uint64) error { - glog.V(logger.Debug).Infof("%v: downloading hashes from #%d", p, from) - - // Create a timeout timer, and the associated hash fetcher - request := time.Now() // time of the last fetch request - timeout := time.NewTimer(0) // timer to dump a non-responsive active peer - <-timeout.C // timeout channel should be initially empty - defer timeout.Stop() - - getHashes := func(from uint64) { - glog.V(logger.Detail).Infof("%v: fetching %d hashes from #%d", p, MaxHashFetch, from) - - request = time.Now() - timeout.Reset(hashTTL) - go p.getAbsHashes(from, MaxHashFetch) - } - // Start pulling hashes, until all are exhausted - getHashes(from) - gotHashes := false - - for { - select { - case <-d.cancelCh: - return errCancelHashFetch - - case packet := <-d.hashCh: - // Make sure the active peer is giving us the hashes - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId()) - break - } - hashReqTimer.UpdateSince(request) - timeout.Stop() - - // If no more hashes are inbound, notify the block fetcher and return - if packet.Items() == 0 { - glog.V(logger.Debug).Infof("%v: no available hashes", p) - - select { - case d.blockWakeCh <- false: - case <-d.cancelCh: - } - // If no hashes were retrieved at all, the peer violated it's TD promise that it had a - // better chain compared to ours. The only exception is if it's promised blocks were - // already imported by other means (e.g. fetcher): - // - // R <remote peer>, L <local node>: Both at block 10 - // R: Mine block 11, and propagate it to L - // L: Queue block 11 for import - // L: Notice that R's head and TD increased compared to ours, start sync - // L: Import of block 11 finishes - // L: Sync begins, and finds common ancestor at 11 - // L: Request new hashes up from 11 (R's TD was higher, it must have something) - // R: Nothing to give - if !gotHashes && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 { - return errStallingPeer - } - return nil - } - gotHashes = true - hashes := packet.(*hashPack).hashes - - // Otherwise insert all the new hashes, aborting in case of junk - glog.V(logger.Detail).Infof("%v: scheduling %d hashes from #%d", p, len(hashes), from) - - inserts := d.queue.Schedule61(hashes, true) - if len(inserts) != len(hashes) { - glog.V(logger.Debug).Infof("%v: stale hashes", p) - return errBadPeer - } - // Notify the block fetcher of new hashes, but stop if queue is full - if d.queue.PendingBlocks() < maxQueuedHashes { - // We still have hashes to fetch, send continuation wake signal (potential) - select { - case d.blockWakeCh <- true: - default: - } - } else { - // Hash limit reached, send a termination wake signal (enforced) - select { - case d.blockWakeCh <- false: - case <-d.cancelCh: - } - return nil - } - // Queue not yet full, fetch the next batch - from += uint64(len(hashes)) - getHashes(from) - - case <-timeout.C: - glog.V(logger.Debug).Infof("%v: hash request timed out", p) - hashTimeoutMeter.Mark(1) - return errTimeout - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } -} - -// fetchBlocks61 iteratively downloads the scheduled hashes, taking any available -// peers, reserving a chunk of blocks for each, waiting for delivery and also -// periodically checking for timeouts. -func (d *Downloader) fetchBlocks61(from uint64) error { - glog.V(logger.Debug).Infof("Downloading blocks from #%d", from) - defer glog.V(logger.Debug).Infof("Block download terminated") - - // Create a timeout timer for scheduling expiration tasks - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - - update := make(chan struct{}, 1) - - // Fetch blocks until the hash fetcher's done - finished := false - for { - select { - case <-d.cancelCh: - return errCancelBlockFetch - - case packet := <-d.blockCh: - // If the peer was previously banned and failed to deliver it's pack - // in a reasonable time frame, ignore it's message. - if peer := d.peers.Peer(packet.PeerId()); peer != nil { - blocks := packet.(*blockPack).blocks - - // Deliver the received chunk of blocks and check chain validity - accepted, err := d.queue.DeliverBlocks(peer.id, blocks) - if err == errInvalidChain { - return err - } - // Unless a peer delivered something completely else than requested (usually - // caused by a timed out request which came through in the end), set it to - // idle. If the delivery's stale, the peer should have already been idled. - if err != errStaleDelivery { - peer.SetBlocksIdle(accepted) - } - // Issue a log to the user to see what's going on - switch { - case err == nil && len(blocks) == 0: - glog.V(logger.Detail).Infof("%s: no blocks delivered", peer) - case err == nil: - glog.V(logger.Detail).Infof("%s: delivered %d blocks", peer, len(blocks)) - default: - glog.V(logger.Detail).Infof("%s: delivery failed: %v", peer, err) - } - } - // Blocks arrived, try to update the progress - select { - case update <- struct{}{}: - default: - } - - case cont := <-d.blockWakeCh: - // The hash fetcher sent a continuation flag, check if it's done - if !cont { - finished = true - } - // Hashes arrive, try to update the progress - select { - case update <- struct{}{}: - default: - } - - case <-ticker.C: - // Sanity check update the progress - select { - case update <- struct{}{}: - default: - } - - case <-update: - // Short circuit if we lost all our peers - if d.peers.Len() == 0 { - return errNoPeers - } - // Check for block request timeouts and demote the responsible peers - for pid, fails := range d.queue.ExpireBlocks(blockTTL) { - if peer := d.peers.Peer(pid); peer != nil { - if fails > 1 { - glog.V(logger.Detail).Infof("%s: block delivery timeout", peer) - peer.SetBlocksIdle(0) - } else { - glog.V(logger.Debug).Infof("%s: stalling block delivery, dropping", peer) - d.dropPeer(pid) - } - } - } - // If there's nothing more to fetch, wait or terminate - if d.queue.PendingBlocks() == 0 { - if !d.queue.InFlightBlocks() && finished { - glog.V(logger.Debug).Infof("Block fetching completed") - return nil - } - break - } - // Send a download request to all idle peers, until throttled - throttled := false - idles, total := d.peers.BlockIdlePeers() - - for _, peer := range idles { - // Short circuit if throttling activated - if d.queue.ShouldThrottleBlocks() { - throttled = true - break - } - // Reserve a chunk of hashes for a peer. A nil can mean either that - // no more hashes are available, or that the peer is known not to - // have them. - request := d.queue.ReserveBlocks(peer, peer.BlockCapacity(blockTargetRTT)) - if request == nil { - continue - } - if glog.V(logger.Detail) { - glog.Infof("%s: requesting %d blocks", peer, len(request.Hashes)) - } - // Fetch the chunk and make sure any errors return the hashes to the queue - if err := peer.Fetch61(request); err != nil { - // Although we could try and make an attempt to fix this, this error really - // means that we've double allocated a fetch task to a peer. If that is the - // case, the internal state of the downloader and the queue is very wrong so - // better hard crash and note the error instead of silently accumulating into - // a much bigger issue. - panic(fmt.Sprintf("%v: fetch assignment failed", peer)) - } - } - // Make sure that we have peers available for fetching. If all peers have been tried - // and all failed throw an error - if !throttled && !d.queue.InFlightBlocks() && len(idles) == total { - return errPeersUnavailable - } - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } -} - // fetchHeight retrieves the head header of the remote peer to aid in estimating // the total time a pending synchronisation would take. func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { @@ -1022,11 +531,6 @@ func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { case <-d.stateCh: case <-d.receiptCh: // Out of bounds delivery, ignore - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1067,7 +571,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { for finished := false; !finished; { select { case <-d.cancelCh: - return 0, errCancelHashFetch + return 0, errCancelHeaderFetch case packet := <-d.headerCh: // Discard anything not from the origin peer @@ -1114,11 +618,6 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { case <-d.stateCh: case <-d.receiptCh: // Out of bounds delivery, ignore - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } // If the head fetch already found an ancestor, return @@ -1146,7 +645,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { for arrived := false; !arrived; { select { case <-d.cancelCh: - return 0, errCancelHashFetch + return 0, errCancelHeaderFetch case packer := <-d.headerCh: // Discard anything not from the origin peer @@ -1182,11 +681,6 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { case <-d.stateCh: case <-d.receiptCh: // Out of bounds delivery, ignore - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1305,11 +799,6 @@ func (d *Downloader) fetchHeaders(p *peer, from uint64) error { case <-d.cancelCh: } return errBadPeer - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1555,7 +1044,14 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv // Check for fetch request timeouts and demote the responsible peers for pid, fails := range expire() { if peer := d.peers.Peer(pid); peer != nil { - if fails > 1 { + // If a lot of retrieval elements expired, we might have overestimated the remote peer or perhaps + // ourselves. Only reset to minimal throughput but don't drop just yet. If even the minimal times + // out that sync wise we need to get rid of the peer. + // + // The reason the minimum threshold is 2 is because the downloader tries to estimate the bandwidth + // and latency of a peer separately, which requires pushing the measures capacity a bit and seeing + // how response times reacts, to it always requests one more than the minimum (i.e. min 2). + if fails > 2 { glog.V(logger.Detail).Infof("%s: %s delivery timeout", peer, strings.ToLower(kind)) setIdle(peer, 0) } else { @@ -1623,11 +1119,6 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv if !progressed && !throttled && !running && len(idles) == total && pending() > 0 { return errPeersUnavailable } - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1859,7 +1350,7 @@ func (d *Downloader) processContent() error { } if err != nil { glog.V(logger.Debug).Infof("Result #%d [%x…] processing failed: %v", results[index].Header.Number, results[index].Header.Hash().Bytes()[:4], err) - return err + return errInvalidChain } // Shift the results to the next batch results = results[items:] @@ -1867,19 +1358,6 @@ func (d *Downloader) processContent() error { } } -// DeliverHashes injects a new batch of hashes received from a remote node into -// the download schedule. This is usually invoked through the BlockHashesMsg by -// the protocol handler. -func (d *Downloader) DeliverHashes(id string, hashes []common.Hash) (err error) { - return d.deliver(id, d.hashCh, &hashPack{id, hashes}, hashInMeter, hashDropMeter) -} - -// DeliverBlocks injects a new batch of blocks received from a remote node. -// This is usually invoked through the BlocksMsg by the protocol handler. -func (d *Downloader) DeliverBlocks(id string, blocks []*types.Block) (err error) { - return d.deliver(id, d.blockCh, &blockPack{id, blocks}, blockInMeter, blockDropMeter) -} - // DeliverHeaders injects a new batch of block headers received from a remote // node into the download schedule. func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) (err error) { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index e9e051ded..4ca28091c 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -55,7 +55,7 @@ func init() { // reassembly. func makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts, heavy bool) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) { // Generate the block chain - blocks, receipts := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(nil, parent, testdb, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // If a heavy chain is requested, delay blocks to raise difficulty @@ -399,14 +399,12 @@ func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Ha var err error switch version { - case 61: - err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay), nil, nil, nil, nil, nil) case 62: - err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), nil, nil) + err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), nil, nil) case 63: - err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) + err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) case 64: - err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) + err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) } if err == nil { // Assign the owned hashes, headers and blocks to the peer (deep copy) @@ -465,86 +463,6 @@ func (dl *downloadTester) dropPeer(id string) { dl.downloader.UnregisterPeer(id) } -// peerGetRelHashesFn constructs a GetHashes function associated with a specific -// peer in the download tester. The returned function can be used to retrieve -// batches of hashes from the particularly requested peer. -func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) func(head common.Hash) error { - return func(head common.Hash) error { - time.Sleep(delay) - - dl.lock.RLock() - defer dl.lock.RUnlock() - - // Gather the next batch of hashes - hashes := dl.peerHashes[id] - result := make([]common.Hash, 0, MaxHashFetch) - for i, hash := range hashes { - if hash == head { - i++ - for len(result) < cap(result) && i < len(hashes) { - result = append(result, hashes[i]) - i++ - } - break - } - } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dl.downloader.DeliverHashes(id, result) - }() - return nil - } -} - -// peerGetAbsHashesFn constructs a GetHashesFromNumber function associated with -// a particular peer in the download tester. The returned function can be used to -// retrieve batches of hashes from the particularly requested peer. -func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) func(uint64, int) error { - return func(head uint64, count int) error { - time.Sleep(delay) - - dl.lock.RLock() - defer dl.lock.RUnlock() - - // Gather the next batch of hashes - hashes := dl.peerHashes[id] - result := make([]common.Hash, 0, count) - for i := 0; i < count && len(hashes)-int(head)-1-i >= 0; i++ { - result = append(result, hashes[len(hashes)-int(head)-1-i]) - } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dl.downloader.DeliverHashes(id, result) - }() - return nil - } -} - -// peerGetBlocksFn constructs a getBlocks function associated with a particular -// peer in the download tester. The returned function can be used to retrieve -// batches of blocks from the particularly requested peer. -func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([]common.Hash) error { - return func(hashes []common.Hash) error { - time.Sleep(delay) - - dl.lock.RLock() - defer dl.lock.RUnlock() - - blocks := dl.peerBlocks[id] - result := make([]*types.Block, 0, len(hashes)) - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - result = append(result, block) - } - } - go dl.downloader.DeliverBlocks(id, result) - - return nil - } -} - // peerGetRelHeadersFn constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. @@ -730,7 +648,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng // Tests that simple synchronization against a canonical chain works correctly. // In this test common ancestor lookup should be short circuited and not require // binary searching. -func TestCanonicalSynchronisation61(t *testing.T) { testCanonicalSynchronisation(t, 61, FullSync) } func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62, FullSync) } func TestCanonicalSynchronisation63Full(t *testing.T) { testCanonicalSynchronisation(t, 63, FullSync) } func TestCanonicalSynchronisation63Fast(t *testing.T) { testCanonicalSynchronisation(t, 63, FastSync) } @@ -759,7 +676,6 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling61(t *testing.T) { testThrottling(t, 61, FullSync) } func TestThrottling62(t *testing.T) { testThrottling(t, 62, FullSync) } func TestThrottling63Full(t *testing.T) { testThrottling(t, 63, FullSync) } func TestThrottling63Fast(t *testing.T) { testThrottling(t, 63, FastSync) } @@ -845,7 +761,6 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync61(t *testing.T) { testForkedSync(t, 61, FullSync) } func TestForkedSync62(t *testing.T) { testForkedSync(t, 62, FullSync) } func TestForkedSync63Full(t *testing.T) { testForkedSync(t, 63, FullSync) } func TestForkedSync63Fast(t *testing.T) { testForkedSync(t, 63, FastSync) } @@ -881,7 +796,6 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync61(t *testing.T) { testHeavyForkedSync(t, 61, FullSync) } func TestHeavyForkedSync62(t *testing.T) { testHeavyForkedSync(t, 62, FullSync) } func TestHeavyForkedSync63Full(t *testing.T) { testHeavyForkedSync(t, 63, FullSync) } func TestHeavyForkedSync63Fast(t *testing.T) { testHeavyForkedSync(t, 63, FastSync) } @@ -915,24 +829,9 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) } -// Tests that an inactive downloader will not accept incoming hashes and blocks. -func TestInactiveDownloader61(t *testing.T) { - t.Parallel() - tester := newTester() - - // Check that neither hashes nor blocks are accepted - if err := tester.downloader.DeliverHashes("bad peer", []common.Hash{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } - if err := tester.downloader.DeliverBlocks("bad peer", []*types.Block{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } -} - // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync61(t *testing.T) { testBoundedForkedSync(t, 61, FullSync) } func TestBoundedForkedSync62(t *testing.T) { testBoundedForkedSync(t, 62, FullSync) } func TestBoundedForkedSync63Full(t *testing.T) { testBoundedForkedSync(t, 63, FullSync) } func TestBoundedForkedSync63Fast(t *testing.T) { testBoundedForkedSync(t, 63, FastSync) } @@ -968,7 +867,6 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync61(t *testing.T) { testBoundedHeavyForkedSync(t, 61, FullSync) } func TestBoundedHeavyForkedSync62(t *testing.T) { testBoundedHeavyForkedSync(t, 62, FullSync) } func TestBoundedHeavyForkedSync63Full(t *testing.T) { testBoundedHeavyForkedSync(t, 63, FullSync) } func TestBoundedHeavyForkedSync63Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 63, FastSync) } @@ -1039,7 +937,6 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel61(t *testing.T) { testCancel(t, 61, FullSync) } func TestCancel62(t *testing.T) { testCancel(t, 62, FullSync) } func TestCancel63Full(t *testing.T) { testCancel(t, 63, FullSync) } func TestCancel63Fast(t *testing.T) { testCancel(t, 63, FastSync) } @@ -1081,7 +978,6 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation61(t *testing.T) { testMultiSynchronisation(t, 61, FullSync) } func TestMultiSynchronisation62(t *testing.T) { testMultiSynchronisation(t, 62, FullSync) } func TestMultiSynchronisation63Full(t *testing.T) { testMultiSynchronisation(t, 63, FullSync) } func TestMultiSynchronisation63Fast(t *testing.T) { testMultiSynchronisation(t, 63, FastSync) } @@ -1112,7 +1008,6 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation61(t *testing.T) { testMultiProtoSync(t, 61, FullSync) } func TestMultiProtoSynchronisation62(t *testing.T) { testMultiProtoSync(t, 62, FullSync) } func TestMultiProtoSynchronisation63Full(t *testing.T) { testMultiProtoSync(t, 63, FullSync) } func TestMultiProtoSynchronisation63Fast(t *testing.T) { testMultiProtoSync(t, 63, FastSync) } @@ -1131,7 +1026,6 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - tester.newPeer("peer 61", 61, hashes, nil, blocks, nil) tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) @@ -1143,7 +1037,7 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { assertOwnChain(t, tester, targetBlocks+1) // Check that no peers have been dropped off - for _, version := range []int{61, 62, 63, 64} { + for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peerHashes[peer]; !ok { t.Errorf("%s dropped", peer) @@ -1368,7 +1262,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Tests that a peer advertising an high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack61(t *testing.T) { testHighTDStarvationAttack(t, 61, FullSync) } func TestHighTDStarvationAttack62(t *testing.T) { testHighTDStarvationAttack(t, 62, FullSync) } func TestHighTDStarvationAttack63Full(t *testing.T) { testHighTDStarvationAttack(t, 63, FullSync) } func TestHighTDStarvationAttack63Fast(t *testing.T) { testHighTDStarvationAttack(t, 63, FastSync) } @@ -1391,7 +1284,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping61(t *testing.T) { testBlockHeaderAttackerDropping(t, 61) } func TestBlockHeaderAttackerDropping62(t *testing.T) { testBlockHeaderAttackerDropping(t, 62) } func TestBlockHeaderAttackerDropping63(t *testing.T) { testBlockHeaderAttackerDropping(t, 63) } func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } @@ -1409,7 +1301,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { {errStallingPeer, true}, // Peer was detected to be stalling, drop it {errNoPeers, false}, // No peers to download from, soft race, no issue {errTimeout, true}, // No hashes received in due time, drop the peer - {errEmptyHashSet, true}, // No hashes were returned as a response, drop as it's a dead end {errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end {errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser {errInvalidAncestor, true}, // Agreed upon ancestor is not acceptable, drop the chain rewriter @@ -1417,7 +1308,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { {errInvalidBlock, false}, // A bad peer was detected, but not the sync origin {errInvalidBody, false}, // A bad peer was detected, but not the sync origin {errInvalidReceipt, false}, // A bad peer was detected, but not the sync origin - {errCancelHashFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop {errCancelBlockFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop {errCancelHeaderFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop {errCancelBodyFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop @@ -1450,7 +1340,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress61(t *testing.T) { testSyncProgress(t, 61, FullSync) } func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) } func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) } func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) } @@ -1524,7 +1413,6 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress61(t *testing.T) { testForkedSyncProgress(t, 61, FullSync) } func TestForkedSyncProgress62(t *testing.T) { testForkedSyncProgress(t, 62, FullSync) } func TestForkedSyncProgress63Full(t *testing.T) { testForkedSyncProgress(t, 63, FullSync) } func TestForkedSyncProgress63Fast(t *testing.T) { testForkedSyncProgress(t, 63, FastSync) } @@ -1601,7 +1489,6 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress61(t *testing.T) { testFailedSyncProgress(t, 61, FullSync) } func TestFailedSyncProgress62(t *testing.T) { testFailedSyncProgress(t, 62, FullSync) } func TestFailedSyncProgress63Full(t *testing.T) { testFailedSyncProgress(t, 63, FullSync) } func TestFailedSyncProgress63Fast(t *testing.T) { testFailedSyncProgress(t, 63, FastSync) } @@ -1679,7 +1566,6 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress61(t *testing.T) { testFakedSyncProgress(t, 61, FullSync) } func TestFakedSyncProgress62(t *testing.T) { testFakedSyncProgress(t, 62, FullSync) } func TestFakedSyncProgress63Full(t *testing.T) { testFakedSyncProgress(t, 63, FullSync) } func TestFakedSyncProgress63Fast(t *testing.T) { testFakedSyncProgress(t, 63, FastSync) } diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go index d6fcfa25c..0d76c7dfd 100644 --- a/eth/downloader/metrics.go +++ b/eth/downloader/metrics.go @@ -23,16 +23,6 @@ import ( ) var ( - hashInMeter = metrics.NewMeter("eth/downloader/hashes/in") - hashReqTimer = metrics.NewTimer("eth/downloader/hashes/req") - hashDropMeter = metrics.NewMeter("eth/downloader/hashes/drop") - hashTimeoutMeter = metrics.NewMeter("eth/downloader/hashes/timeout") - - blockInMeter = metrics.NewMeter("eth/downloader/blocks/in") - blockReqTimer = metrics.NewTimer("eth/downloader/blocks/req") - blockDropMeter = metrics.NewMeter("eth/downloader/blocks/drop") - blockTimeoutMeter = metrics.NewMeter("eth/downloader/blocks/timeout") - headerInMeter = metrics.NewMeter("eth/downloader/headers/in") headerReqTimer = metrics.NewTimer("eth/downloader/headers/req") headerDropMeter = metrics.NewMeter("eth/downloader/headers/drop") diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 94d44fca4..c2b7a52d0 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -37,11 +37,6 @@ const ( measurementImpact = 0.1 // The impact a single measurement has on a peer's final throughput value. ) -// Hash and block fetchers belonging to eth/61 and below -type relativeHashFetcherFn func(common.Hash) error -type absoluteHashFetcherFn func(uint64, int) error -type blockFetcherFn func([]common.Hash) error - // Block header and body fetchers belonging to eth/62 and above type relativeHeaderFetcherFn func(common.Hash, int, int, bool) error type absoluteHeaderFetcherFn func(uint64, int, int, bool) error @@ -79,10 +74,6 @@ type peer struct { lacking map[common.Hash]struct{} // Set of hashes not to request (didn't have previously) - getRelHashes relativeHashFetcherFn // [eth/61] Method to retrieve a batch of hashes from an origin hash - getAbsHashes absoluteHashFetcherFn // [eth/61] Method to retrieve a batch of hashes from an absolute position - getBlocks blockFetcherFn // [eth/61] Method to retrieve a batch of blocks - getRelHeaders relativeHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an origin hash getAbsHeaders absoluteHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an absolute position getBlockBodies blockBodyFetcherFn // [eth/62] Method to retrieve a batch of block bodies @@ -97,7 +88,6 @@ type peer struct { // newPeer create a new downloader peer, with specific hash and block retrieval // mechanisms. func newPeer(id string, version int, head common.Hash, - getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, getReceipts receiptFetcherFn, getNodeData stateFetcherFn) *peer { return &peer{ @@ -105,10 +95,6 @@ func newPeer(id string, version int, head common.Hash, head: head, lacking: make(map[common.Hash]struct{}), - getRelHashes: getRelHashes, - getAbsHashes: getAbsHashes, - getBlocks: getBlocks, - getRelHeaders: getRelHeaders, getAbsHeaders: getAbsHeaders, getBlockBodies: getBlockBodies, @@ -138,28 +124,6 @@ func (p *peer) Reset() { p.lacking = make(map[common.Hash]struct{}) } -// Fetch61 sends a block retrieval request to the remote peer. -func (p *peer) Fetch61(request *fetchRequest) error { - // Sanity check the protocol version - if p.version != 61 { - panic(fmt.Sprintf("block fetch [eth/61] requested on eth/%d", p.version)) - } - // Short circuit if the peer is already fetching - if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) { - return errAlreadyFetching - } - p.blockStarted = time.Now() - - // Convert the hash set to a retrievable slice - hashes := make([]common.Hash, 0, len(request.Hashes)) - for hash, _ := range request.Hashes { - hashes = append(hashes, hash) - } - go p.getBlocks(hashes) - - return nil -} - // FetchHeaders sends a header retrieval request to the remote peer. func (p *peer) FetchHeaders(from uint64, count int) error { // Sanity check the protocol version @@ -481,20 +445,6 @@ func (ps *peerSet) AllPeers() []*peer { return list } -// BlockIdlePeers retrieves a flat list of all the currently idle peers within the -// active peer set, ordered by their reputation. -func (ps *peerSet) BlockIdlePeers() ([]*peer, int) { - idle := func(p *peer) bool { - return atomic.LoadInt32(&p.blockIdle) == 0 - } - throughput := func(p *peer) float64 { - p.lock.RLock() - defer p.lock.RUnlock() - return p.blockThroughput - } - return ps.idlePeers(61, 61, idle, throughput) -} - // HeaderIdlePeers retrieves a flat list of all the currently header-idle peers // within the active peer set, ordered by their reputation. func (ps *peerSet) HeaderIdlePeers() ([]*peer, int) { diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 01897af6d..fd239f7e4 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -45,7 +45,6 @@ var ( var ( errNoFetchesPending = errors.New("no fetches pending") - errStateSyncPending = errors.New("state trie sync already scheduled") errStaleDelivery = errors.New("stale delivery") ) @@ -74,10 +73,6 @@ type queue struct { mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching fastSyncPivot uint64 // Block number where the fast sync pivots into archive synchronisation mode - hashPool map[common.Hash]int // [eth/61] Pending hashes, mapping to their insertion index (priority) - hashQueue *prque.Prque // [eth/61] Priority queue of the block hashes to fetch - hashCounter int // [eth/61] Counter indexing the added hashes to ensure retrieval order - headerHead common.Hash // [eth/62] Hash of the last queued header to verify order // Headers are "special", they download in batches, supported by a skeleton chain @@ -85,7 +80,6 @@ type queue struct { headerTaskQueue *prque.Prque // [eth/62] Priority queue of the skeleton indexes to fetch the filling headers for headerPeerMiss map[string]map[uint64]struct{} // [eth/62] Set of per-peer header batches known to be unavailable headerPendPool map[string]*fetchRequest // [eth/62] Currently pending header retrieval operations - headerDonePool map[uint64]struct{} // [eth/62] Set of the completed header fetches headerResults []*types.Header // [eth/62] Result cache accumulating the completed headers headerProced int // [eth/62] Number of headers already processed from the results headerOffset uint64 // [eth/62] Number of the first header in the result cache @@ -124,8 +118,6 @@ type queue struct { func newQueue(stateDb ethdb.Database) *queue { lock := new(sync.Mutex) return &queue{ - hashPool: make(map[common.Hash]int), - hashQueue: prque.New(), headerPendPool: make(map[string]*fetchRequest), headerContCh: make(chan bool), blockTaskPool: make(map[common.Hash]*types.Header), @@ -158,10 +150,6 @@ func (q *queue) Reset() { q.mode = FullSync q.fastSyncPivot = 0 - q.hashPool = make(map[common.Hash]int) - q.hashQueue.Reset() - q.hashCounter = 0 - q.headerHead = common.Hash{} q.headerPendPool = make(map[string]*fetchRequest) @@ -208,7 +196,7 @@ func (q *queue) PendingBlocks() int { q.lock.Lock() defer q.lock.Unlock() - return q.hashQueue.Size() + q.blockTaskQueue.Size() + return q.blockTaskQueue.Size() } // PendingReceipts retrieves the number of block receipts pending for retrieval. @@ -272,7 +260,7 @@ func (q *queue) Idle() bool { q.lock.Lock() defer q.lock.Unlock() - queued := q.hashQueue.Size() + q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() + q.stateTaskQueue.Size() + queued := q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() + q.stateTaskQueue.Size() pending := len(q.blockPendPool) + len(q.receiptPendPool) + len(q.statePendPool) cached := len(q.blockDonePool) + len(q.receiptDonePool) @@ -323,34 +311,6 @@ func (q *queue) ShouldThrottleReceipts() bool { return pending >= len(q.resultCache)-len(q.receiptDonePool) } -// Schedule61 adds a set of hashes for the download queue for scheduling, returning -// the new hashes encountered. -func (q *queue) Schedule61(hashes []common.Hash, fifo bool) []common.Hash { - q.lock.Lock() - defer q.lock.Unlock() - - // Insert all the hashes prioritised in the arrival order - inserts := make([]common.Hash, 0, len(hashes)) - for _, hash := range hashes { - // Skip anything we already have - if old, ok := q.hashPool[hash]; ok { - glog.V(logger.Warn).Infof("Hash %x already scheduled at index %v", hash, old) - continue - } - // Update the counters and insert the hash - q.hashCounter = q.hashCounter + 1 - inserts = append(inserts, hash) - - q.hashPool[hash] = q.hashCounter - if fifo { - q.hashQueue.Push(hash, -float32(q.hashCounter)) // Lowest gets schedules first - } else { - q.hashQueue.Push(hash, float32(q.hashCounter)) // Highest gets schedules first - } - } - return inserts -} - // ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill // up an already retrieved header skeleton. func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { @@ -550,15 +510,6 @@ func (q *queue) ReserveHeaders(p *peer, count int) *fetchRequest { return request } -// ReserveBlocks reserves a set of block hashes for the given peer, skipping any -// previously failed download. -func (q *queue) ReserveBlocks(p *peer, count int) *fetchRequest { - q.lock.Lock() - defer q.lock.Unlock() - - return q.reserveHashes(p, count, q.hashQueue, nil, q.blockPendPool, len(q.resultCache)-len(q.blockDonePool)) -} - // ReserveNodeData reserves a set of node data hashes for the given peer, skipping // any previously failed download. func (q *queue) ReserveNodeData(p *peer, count int) *fetchRequest { @@ -753,11 +704,6 @@ func (q *queue) CancelHeaders(request *fetchRequest) { q.cancel(request, q.headerTaskQueue, q.headerPendPool) } -// CancelBlocks aborts a fetch request, returning all pending hashes to the queue. -func (q *queue) CancelBlocks(request *fetchRequest) { - q.cancel(request, q.hashQueue, q.blockPendPool) -} - // CancelBodies aborts a body fetch request, returning all pending headers to the // task queue. func (q *queue) CancelBodies(request *fetchRequest) { @@ -801,9 +747,6 @@ func (q *queue) Revoke(peerId string) { defer q.lock.Unlock() if request, ok := q.blockPendPool[peerId]; ok { - for hash, index := range request.Hashes { - q.hashQueue.Push(hash, float32(index)) - } for _, header := range request.Headers { q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) } @@ -832,15 +775,6 @@ func (q *queue) ExpireHeaders(timeout time.Duration) map[string]int { return q.expire(timeout, q.headerPendPool, q.headerTaskQueue, headerTimeoutMeter) } -// ExpireBlocks checks for in flight requests that exceeded a timeout allowance, -// canceling them and returning the responsible peers for penalisation. -func (q *queue) ExpireBlocks(timeout time.Duration) map[string]int { - q.lock.Lock() - defer q.lock.Unlock() - - return q.expire(timeout, q.blockPendPool, q.hashQueue, blockTimeoutMeter) -} - // ExpireBodies checks for in flight block body requests that exceeded a timeout // allowance, canceling them and returning the responsible peers for penalisation. func (q *queue) ExpireBodies(timeout time.Duration) map[string]int { @@ -907,74 +841,6 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, return expiries } -// DeliverBlocks injects a block retrieval response into the download queue. The -// method returns the number of blocks accepted from the delivery and also wakes -// any threads waiting for data delivery. -func (q *queue) DeliverBlocks(id string, blocks []*types.Block) (int, error) { - q.lock.Lock() - defer q.lock.Unlock() - - // Short circuit if the blocks were never requested - request := q.blockPendPool[id] - if request == nil { - return 0, errNoFetchesPending - } - blockReqTimer.UpdateSince(request.Time) - delete(q.blockPendPool, id) - - // If no blocks were retrieved, mark them as unavailable for the origin peer - if len(blocks) == 0 { - for hash, _ := range request.Hashes { - request.Peer.MarkLacking(hash) - } - } - // Iterate over the downloaded blocks and add each of them - accepted, errs := 0, make([]error, 0) - for _, block := range blocks { - // Skip any blocks that were not requested - hash := block.Hash() - if _, ok := request.Hashes[hash]; !ok { - errs = append(errs, fmt.Errorf("non-requested block %x", hash)) - continue - } - // Reconstruct the next result if contents match up - index := int(block.Number().Int64() - int64(q.resultOffset)) - if index >= len(q.resultCache) || index < 0 { - errs = []error{errInvalidChain} - break - } - q.resultCache[index] = &fetchResult{ - Header: block.Header(), - Transactions: block.Transactions(), - Uncles: block.Uncles(), - } - q.blockDonePool[block.Hash()] = struct{}{} - - delete(request.Hashes, hash) - delete(q.hashPool, hash) - accepted++ - } - // Return all failed or missing fetches to the queue - for hash, index := range request.Hashes { - q.hashQueue.Push(hash, float32(index)) - } - // Wake up WaitResults - if accepted > 0 { - q.active.Signal() - } - // If none of the blocks were good, it's a stale delivery - switch { - case len(errs) == 0: - return accepted, nil - case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBlock): - return accepted, errs[0] - case len(errs) == len(blocks): - return accepted, errStaleDelivery - default: - return accepted, fmt.Errorf("multiple failures: %v", errs) - } -} - // DeliverHeaders injects a header retrieval response into the header results // cache. This method either accepts all headers it received, or none of them // if they do not map correctly to the skeleton. diff --git a/eth/downloader/types.go b/eth/downloader/types.go index b67fff1f8..e10510486 100644 --- a/eth/downloader/types.go +++ b/eth/downloader/types.go @@ -73,26 +73,6 @@ type dataPack interface { Stats() string } -// hashPack is a batch of block hashes returned by a peer (eth/61). -type hashPack struct { - peerId string - hashes []common.Hash -} - -func (p *hashPack) PeerId() string { return p.peerId } -func (p *hashPack) Items() int { return len(p.hashes) } -func (p *hashPack) Stats() string { return fmt.Sprintf("%d", len(p.hashes)) } - -// blockPack is a batch of blocks returned by a peer (eth/61). -type blockPack struct { - peerId string - blocks []*types.Block -} - -func (p *blockPack) PeerId() string { return p.peerId } -func (p *blockPack) Items() int { return len(p.blocks) } -func (p *blockPack) Stats() string { return fmt.Sprintf("%d", len(p.blocks)) } - // headerPack is a batch of block headers returned by a peer. type headerPack struct { peerId string diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 9300717c3..bd235bb9e 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -48,9 +48,6 @@ var ( // blockRetrievalFn is a callback type for retrieving a block from the local chain. type blockRetrievalFn func(common.Hash) *types.Block -// blockRequesterFn is a callback type for sending a block retrieval request. -type blockRequesterFn func([]common.Hash) error - // headerRequesterFn is a callback type for sending a header retrieval request. type headerRequesterFn func(common.Hash) error @@ -82,7 +79,6 @@ type announce struct { origin string // Identifier of the peer originating the notification - fetch61 blockRequesterFn // [eth/61] Fetcher function to retrieve an announced block fetchHeader headerRequesterFn // [eth/62] Fetcher function to retrieve the header of an announced block fetchBodies bodyRequesterFn // [eth/62] Fetcher function to retrieve the body of an announced block } @@ -191,14 +187,12 @@ func (f *Fetcher) Stop() { // Notify announces the fetcher of the potential availability of a new block in // the network. func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - blockFetcher blockRequesterFn, // eth/61 specific whole block fetcher headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { block := &announce{ hash: hash, number: number, time: time, origin: peer, - fetch61: blockFetcher, fetchHeader: headerFetcher, fetchBodies: bodyFetcher, } @@ -224,34 +218,6 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error { } } -// FilterBlocks extracts all the blocks that were explicitly requested by the fetcher, -// returning those that should be handled differently. -func (f *Fetcher) FilterBlocks(blocks types.Blocks) types.Blocks { - glog.V(logger.Detail).Infof("[eth/61] filtering %d blocks", len(blocks)) - - // Send the filter channel to the fetcher - filter := make(chan []*types.Block) - - select { - case f.blockFilter <- filter: - case <-f.quit: - return nil - } - // Request the filtering of the block list - select { - case filter <- blocks: - case <-f.quit: - return nil - } - // Retrieve the blocks remaining after filtering - select { - case blocks := <-filter: - return blocks - case <-f.quit: - return nil - } -} - // FilterHeaders extracts all the headers that were explicitly requested by the fetcher, // returning those that should be handled differently. func (f *Fetcher) FilterHeaders(headers []*types.Header, time time.Time) []*types.Header { @@ -413,7 +379,7 @@ func (f *Fetcher) loop() { } } } - // Send out all block (eth/61) or header (eth/62) requests + // Send out all block header requests for peer, hashes := range request { if glog.V(logger.Detail) && len(hashes) > 0 { list := "[" @@ -421,29 +387,17 @@ func (f *Fetcher) loop() { list += fmt.Sprintf("%x…, ", hash[:4]) } list = list[:len(list)-2] + "]" - - if f.fetching[hashes[0]].fetch61 != nil { - glog.V(logger.Detail).Infof("[eth/61] Peer %s: fetching blocks %s", peer, list) - } else { - glog.V(logger.Detail).Infof("[eth/62] Peer %s: fetching headers %s", peer, list) - } + glog.V(logger.Detail).Infof("[eth/62] Peer %s: fetching headers %s", peer, list) } // Create a closure of the fetch and schedule in on a new thread - fetchBlocks, fetchHeader, hashes := f.fetching[hashes[0]].fetch61, f.fetching[hashes[0]].fetchHeader, hashes + fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes go func() { if f.fetchingHook != nil { f.fetchingHook(hashes) } - if fetchBlocks != nil { - // Use old eth/61 protocol to retrieve whole blocks - blockFetchMeter.Mark(int64(len(hashes))) - fetchBlocks(hashes) - } else { - // Use new eth/62 protocol to retrieve headers first - for _, hash := range hashes { - headerFetchMeter.Mark(1) - fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals - } + for _, hash := range hashes { + headerFetchMeter.Mark(1) + fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals } }() } @@ -486,46 +440,6 @@ func (f *Fetcher) loop() { // Schedule the next fetch if blocks are still pending f.rescheduleComplete(completeTimer) - case filter := <-f.blockFilter: - // Blocks arrived, extract any explicit fetches, return all else - var blocks types.Blocks - select { - case blocks = <-filter: - case <-f.quit: - return - } - blockFilterInMeter.Mark(int64(len(blocks))) - - explicit, download := []*types.Block{}, []*types.Block{} - for _, block := range blocks { - hash := block.Hash() - - // Filter explicitly requested blocks from hash announcements - if f.fetching[hash] != nil && f.queued[hash] == nil { - // Discard if already imported by other means - if f.getBlock(hash) == nil { - explicit = append(explicit, block) - } else { - f.forgetHash(hash) - } - } else { - download = append(download, block) - } - } - - blockFilterOutMeter.Mark(int64(len(download))) - select { - case filter <- download: - case <-f.quit: - return - } - // Schedule the retrieved blocks for ordered import - for _, block := range explicit { - if announce := f.fetching[block.Hash()]; announce != nil { - f.enqueue(announce.origin, block) - } - } - case filter := <-f.headerFilter: // Headers arrived from a remote peer. Extract those that were explicitly // requested by the fetcher, and return everything else so it's delivered diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index 2404c8cfa..ad955a577 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -45,7 +45,7 @@ var ( // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { - blocks, _ := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) { + blocks, _ := core.GenerateChain(nil, parent, testdb, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // If the block number is multiple of 3, send a bonus transaction to the miner @@ -151,28 +151,6 @@ func (f *fetcherTester) dropPeer(peer string) { f.drops[peer] = true } -// makeBlockFetcher retrieves a block fetcher associated with a simulated peer. -func (f *fetcherTester) makeBlockFetcher(blocks map[common.Hash]*types.Block) blockRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that returns blocks from the closure - return func(hashes []common.Hash) error { - // Gather the blocks to return - blocks := make([]*types.Block, 0, len(hashes)) - for _, hash := range hashes { - if block, ok := closure[hash]; ok { - blocks = append(blocks, block) - } - } - // Return on a new thread - go f.fetcher.FilterBlocks(blocks) - - return nil - } -} - // makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. func (f *fetcherTester) makeHeaderFetcher(blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { closure := make(map[common.Hash]*types.Block) @@ -293,7 +271,6 @@ func verifyImportDone(t *testing.T, imported chan *types.Block) { // Tests that a fetcher accepts block announcements and initiates retrievals for // them, successfully importing into the local chain. -func TestSequentialAnnouncements61(t *testing.T) { testSequentialAnnouncements(t, 61) } func TestSequentialAnnouncements62(t *testing.T) { testSequentialAnnouncements(t, 62) } func TestSequentialAnnouncements63(t *testing.T) { testSequentialAnnouncements(t, 63) } func TestSequentialAnnouncements64(t *testing.T) { testSequentialAnnouncements(t, 64) } @@ -304,7 +281,6 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { hashes, blocks := makeChain(targetBlocks, 0, genesis) tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -313,11 +289,7 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } for i := len(hashes) - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) @@ -325,7 +297,6 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { // Tests that if blocks are announced by multiple peers (or even the same buggy // peer), they will only get downloaded at most once. -func TestConcurrentAnnouncements61(t *testing.T) { testConcurrentAnnouncements(t, 61) } func TestConcurrentAnnouncements62(t *testing.T) { testConcurrentAnnouncements(t, 62) } func TestConcurrentAnnouncements63(t *testing.T) { testConcurrentAnnouncements(t, 63) } func TestConcurrentAnnouncements64(t *testing.T) { testConcurrentAnnouncements(t, 64) } @@ -337,15 +308,10 @@ func testConcurrentAnnouncements(t *testing.T, protocol int) { // Assemble a tester with a built in counter for the requests tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) counter := uint32(0) - blockWrapper := func(hashes []common.Hash) error { - atomic.AddUint32(&counter, uint32(len(hashes))) - return blockFetcher(hashes) - } headerWrapper := func(hash common.Hash) error { atomic.AddUint32(&counter, 1) return headerFetcher(hash) @@ -355,15 +321,9 @@ func testConcurrentAnnouncements(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } for i := len(hashes) - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("first", hashes[i], 0, time.Now().Add(-arriveTimeout), blockWrapper, nil, nil) - tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout+time.Millisecond), blockWrapper, nil, nil) - tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout-time.Millisecond), blockWrapper, nil, nil) - } else { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerWrapper, bodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), nil, headerWrapper, bodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), nil, headerWrapper, bodyFetcher) - } + tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), headerWrapper, bodyFetcher) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), headerWrapper, bodyFetcher) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) @@ -376,7 +336,6 @@ func testConcurrentAnnouncements(t *testing.T, protocol int) { // Tests that announcements arriving while a previous is being fetched still // results in a valid import. -func TestOverlappingAnnouncements61(t *testing.T) { testOverlappingAnnouncements(t, 61) } func TestOverlappingAnnouncements62(t *testing.T) { testOverlappingAnnouncements(t, 62) } func TestOverlappingAnnouncements63(t *testing.T) { testOverlappingAnnouncements(t, 63) } func TestOverlappingAnnouncements64(t *testing.T) { testOverlappingAnnouncements(t, 64) } @@ -387,7 +346,6 @@ func testOverlappingAnnouncements(t *testing.T, protocol int) { hashes, blocks := makeChain(targetBlocks, 0, genesis) tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -400,11 +358,7 @@ func testOverlappingAnnouncements(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } for i := len(hashes) - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) select { case <-imported: case <-time.After(time.Second): @@ -416,7 +370,6 @@ func testOverlappingAnnouncements(t *testing.T, protocol int) { } // Tests that announces already being retrieved will not be duplicated. -func TestPendingDeduplication61(t *testing.T) { testPendingDeduplication(t, 61) } func TestPendingDeduplication62(t *testing.T) { testPendingDeduplication(t, 62) } func TestPendingDeduplication63(t *testing.T) { testPendingDeduplication(t, 63) } func TestPendingDeduplication64(t *testing.T) { testPendingDeduplication(t, 64) } @@ -427,22 +380,11 @@ func testPendingDeduplication(t *testing.T, protocol int) { // Assemble a tester with a built in counter and delayed fetcher tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) delay := 50 * time.Millisecond counter := uint32(0) - blockWrapper := func(hashes []common.Hash) error { - atomic.AddUint32(&counter, uint32(len(hashes))) - - // Simulate a long running fetch - go func() { - time.Sleep(delay) - blockFetcher(hashes) - }() - return nil - } headerWrapper := func(hash common.Hash) error { atomic.AddUint32(&counter, 1) @@ -455,11 +397,7 @@ func testPendingDeduplication(t *testing.T, protocol int) { } // Announce the same block many times until it's fetched (wait for any pending ops) for tester.getBlock(hashes[0]) == nil { - if protocol < 62 { - tester.fetcher.Notify("repeater", hashes[0], 0, time.Now().Add(-arriveTimeout), blockWrapper, nil, nil) - } else { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerWrapper, bodyFetcher) - } + tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) time.Sleep(time.Millisecond) } time.Sleep(delay) @@ -475,7 +413,6 @@ func testPendingDeduplication(t *testing.T, protocol int) { // Tests that announcements retrieved in a random order are cached and eventually // imported when all the gaps are filled in. -func TestRandomArrivalImport61(t *testing.T) { testRandomArrivalImport(t, 61) } func TestRandomArrivalImport62(t *testing.T) { testRandomArrivalImport(t, 62) } func TestRandomArrivalImport63(t *testing.T) { testRandomArrivalImport(t, 63) } func TestRandomArrivalImport64(t *testing.T) { testRandomArrivalImport(t, 64) } @@ -487,7 +424,6 @@ func testRandomArrivalImport(t *testing.T, protocol int) { skip := targetBlocks / 2 tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -497,26 +433,17 @@ func testRandomArrivalImport(t *testing.T, protocol int) { for i := len(hashes) - 1; i >= 0; i-- { if i != skip { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) time.Sleep(time.Millisecond) } } // Finally announce the skipped entry and check full import - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[skip], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportCount(t, imported, len(hashes)-1) } // Tests that direct block enqueues (due to block propagation vs. hash announce) // are correctly schedule, filling and import queue gaps. -func TestQueueGapFill61(t *testing.T) { testQueueGapFill(t, 61) } func TestQueueGapFill62(t *testing.T) { testQueueGapFill(t, 62) } func TestQueueGapFill63(t *testing.T) { testQueueGapFill(t, 63) } func TestQueueGapFill64(t *testing.T) { testQueueGapFill(t, 64) } @@ -528,7 +455,6 @@ func testQueueGapFill(t *testing.T, protocol int) { skip := targetBlocks / 2 tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -538,11 +464,7 @@ func testQueueGapFill(t *testing.T, protocol int) { for i := len(hashes) - 1; i >= 0; i-- { if i != skip { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) time.Sleep(time.Millisecond) } } @@ -553,7 +475,6 @@ func testQueueGapFill(t *testing.T, protocol int) { // Tests that blocks arriving from various sources (multiple propagations, hash // announces, etc) do not get scheduled for import multiple times. -func TestImportDeduplication61(t *testing.T) { testImportDeduplication(t, 61) } func TestImportDeduplication62(t *testing.T) { testImportDeduplication(t, 62) } func TestImportDeduplication63(t *testing.T) { testImportDeduplication(t, 63) } func TestImportDeduplication64(t *testing.T) { testImportDeduplication(t, 64) } @@ -564,7 +485,6 @@ func testImportDeduplication(t *testing.T, protocol int) { // Create the tester and wrap the importer with a counter tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -580,11 +500,7 @@ func testImportDeduplication(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } // Announce the duplicating block, wait for retrieval, and also propagate directly - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[0], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) <-fetching tester.fetcher.Enqueue("valid", blocks[hashes[0]]) @@ -660,14 +576,14 @@ func testDistantAnnouncementDiscarding(t *testing.T, protocol int) { tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) select { case <-time.After(50 * time.Millisecond): case <-fetching: t.Fatalf("fetcher requested stale header") } // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) select { case <-time.After(50 * time.Millisecond): case <-fetching: @@ -693,7 +609,7 @@ func testInvalidNumberAnnouncement(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } // Announce a block with a bad number, check for immediate drop - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportEvent(t, imported, false) tester.lock.RLock() @@ -704,7 +620,7 @@ func testInvalidNumberAnnouncement(t *testing.T, protocol int) { t.Fatalf("peer with invalid numbered announcement not dropped") } // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportEvent(t, imported, true) tester.lock.RLock() @@ -743,7 +659,7 @@ func testEmptyBlockShortCircuit(t *testing.T, protocol int) { // Iteratively announce blocks until all are imported for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) // All announces should fetch the header verifyFetchingEvent(t, fetching, true) @@ -760,7 +676,6 @@ func testEmptyBlockShortCircuit(t *testing.T, protocol int) { // Tests that a peer is unable to use unbounded memory with sending infinite // block announcements to a node, but that even in the face of such an attack, // the fetcher remains operational. -func TestHashMemoryExhaustionAttack61(t *testing.T) { testHashMemoryExhaustionAttack(t, 61) } func TestHashMemoryExhaustionAttack62(t *testing.T) { testHashMemoryExhaustionAttack(t, 62) } func TestHashMemoryExhaustionAttack63(t *testing.T) { testHashMemoryExhaustionAttack(t, 63) } func TestHashMemoryExhaustionAttack64(t *testing.T) { testHashMemoryExhaustionAttack(t, 64) } @@ -781,29 +696,19 @@ func testHashMemoryExhaustionAttack(t *testing.T, protocol int) { // Create a valid chain and an infinite junk chain targetBlocks := hashLimit + 2*maxQueueDist hashes, blocks := makeChain(targetBlocks, 0, genesis) - validBlockFetcher := tester.makeBlockFetcher(blocks) validHeaderFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) validBodyFetcher := tester.makeBodyFetcher(blocks, 0) attack, _ := makeChain(targetBlocks, 0, unknownBlock) - attackerBlockFetcher := tester.makeBlockFetcher(nil) attackerHeaderFetcher := tester.makeHeaderFetcher(nil, -gatherSlack) attackerBodyFetcher := tester.makeBodyFetcher(nil, 0) // Feed the tester a huge hashset from the attacker, and a limited from the valid peer for i := 0; i < len(attack); i++ { if i < maxQueueDist { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], 0, time.Now(), validBlockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), nil, validHeaderFetcher, validBodyFetcher) - } - } - if protocol < 62 { - tester.fetcher.Notify("attacker", attack[i], 0, time.Now(), attackerBlockFetcher, nil, nil) - } else { - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), nil, attackerHeaderFetcher, attackerBodyFetcher) + tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) } + tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) } if count := atomic.LoadInt32(&announces); count != hashLimit+maxQueueDist { t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) @@ -813,11 +718,7 @@ func testHashMemoryExhaustionAttack(t *testing.T, protocol int) { // Feed the remaining valid hashes to ensure DOS protection state remains clean for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), validBlockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, validHeaderFetcher, validBodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index b82d3ca01..1ed8075bf 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -33,12 +33,9 @@ var ( propBroadcastDropMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/drop") propBroadcastDOSMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/dos") - blockFetchMeter = metrics.NewMeter("eth/fetcher/fetch/blocks") headerFetchMeter = metrics.NewMeter("eth/fetcher/fetch/headers") bodyFetchMeter = metrics.NewMeter("eth/fetcher/fetch/bodies") - blockFilterInMeter = metrics.NewMeter("eth/fetcher/filter/blocks/in") - blockFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/blocks/out") headerFilterInMeter = metrics.NewMeter("eth/fetcher/filter/headers/in") headerFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/headers/out") bodyFilterInMeter = metrics.NewMeter("eth/fetcher/filter/bodies/in") diff --git a/eth/filters/api.go b/eth/filters/api.go index 393019f8b..65c5b9380 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -68,8 +68,6 @@ type PublicFilterAPI struct { transactionMu sync.RWMutex transactionQueue map[int]*hashQueue - - transactMu sync.Mutex } // NewPublicFilterAPI returns a new PublicFilterAPI instance. @@ -100,6 +98,7 @@ done: for { select { case <-timer.C: + s.filterManager.Lock() // lock order like filterLoop() s.logMu.Lock() for id, filter := range s.logQueue { if time.Since(filter.timeout) > filterTickerTime { @@ -126,6 +125,7 @@ done: } } s.transactionMu.Unlock() + s.filterManager.Unlock() case <-s.quit: break done } @@ -135,19 +135,24 @@ done: // NewBlockFilter create a new filter that returns blocks that are included into the canonical chain. func (s *PublicFilterAPI) NewBlockFilter() (string, error) { + // protect filterManager.Add() and setting of filter fields + s.filterManager.Lock() + defer s.filterManager.Unlock() + externalId, err := newFilterId() if err != nil { return "", err } - s.blockMu.Lock() filter := New(s.chainDb) id, err := s.filterManager.Add(filter, ChainFilter) if err != nil { return "", err } + s.blockMu.Lock() s.blockQueue[id] = &hashQueue{timeout: time.Now()} + s.blockMu.Unlock() filter.BlockCallback = func(block *types.Block, logs vm.Logs) { s.blockMu.Lock() @@ -158,8 +163,6 @@ func (s *PublicFilterAPI) NewBlockFilter() (string, error) { } } - defer s.blockMu.Unlock() - s.filterMapMu.Lock() s.filterMapping[externalId] = id s.filterMapMu.Unlock() @@ -169,21 +172,24 @@ func (s *PublicFilterAPI) NewBlockFilter() (string, error) { // NewPendingTransactionFilter creates a filter that returns new pending transactions. func (s *PublicFilterAPI) NewPendingTransactionFilter() (string, error) { + // protect filterManager.Add() and setting of filter fields + s.filterManager.Lock() + defer s.filterManager.Unlock() + externalId, err := newFilterId() if err != nil { return "", err } - s.transactionMu.Lock() - defer s.transactionMu.Unlock() - filter := New(s.chainDb) id, err := s.filterManager.Add(filter, PendingTxFilter) if err != nil { return "", err } + s.transactionMu.Lock() s.transactionQueue[id] = &hashQueue{timeout: time.Now()} + s.transactionMu.Unlock() filter.TransactionCallback = func(tx *types.Transaction) { s.transactionMu.Lock() @@ -203,8 +209,9 @@ func (s *PublicFilterAPI) NewPendingTransactionFilter() (string, error) { // newLogFilter creates a new log filter. func (s *PublicFilterAPI) newLogFilter(earliest, latest int64, addresses []common.Address, topics [][]common.Hash, callback func(log *vm.Log, removed bool)) (int, error) { - s.logMu.Lock() - defer s.logMu.Unlock() + // protect filterManager.Add() and setting of filter fields + s.filterManager.Lock() + defer s.filterManager.Unlock() filter := New(s.chainDb) id, err := s.filterManager.Add(filter, LogFilter) @@ -212,7 +219,9 @@ func (s *PublicFilterAPI) newLogFilter(earliest, latest int64, addresses []commo return 0, err } + s.logMu.Lock() s.logQueue[id] = &logQueue{timeout: time.Now()} + s.logMu.Unlock() filter.SetBeginBlock(earliest) filter.SetEndBlock(latest) @@ -443,35 +452,43 @@ func (s *PublicFilterAPI) GetLogs(args NewFilterArgs) []vmlog { // UninstallFilter removes the filter with the given filter id. func (s *PublicFilterAPI) UninstallFilter(filterId string) bool { - s.filterMapMu.Lock() - defer s.filterMapMu.Unlock() + s.filterManager.Lock() + defer s.filterManager.Unlock() + s.filterMapMu.Lock() id, ok := s.filterMapping[filterId] if !ok { + s.filterMapMu.Unlock() return false } - - defer s.filterManager.Remove(id) delete(s.filterMapping, filterId) + s.filterMapMu.Unlock() + s.filterManager.Remove(id) + + s.logMu.Lock() if _, ok := s.logQueue[id]; ok { - s.logMu.Lock() - defer s.logMu.Unlock() delete(s.logQueue, id) + s.logMu.Unlock() return true } + s.logMu.Unlock() + + s.blockMu.Lock() if _, ok := s.blockQueue[id]; ok { - s.blockMu.Lock() - defer s.blockMu.Unlock() delete(s.blockQueue, id) + s.blockMu.Unlock() return true } + s.blockMu.Unlock() + + s.transactionMu.Lock() if _, ok := s.transactionQueue[id]; ok { - s.transactionMu.Lock() - defer s.transactionMu.Unlock() delete(s.transactionQueue, id) + s.transactionMu.Unlock() return true } + s.transactionMu.Unlock() return false } @@ -525,7 +542,9 @@ func (s *PublicFilterAPI) logFilterChanged(id int) []vmlog { // GetFilterLogs returns the logs for the filter with the given id. func (s *PublicFilterAPI) GetFilterLogs(filterId string) []vmlog { + s.filterMapMu.RLock() id, ok := s.filterMapping[filterId] + s.filterMapMu.RUnlock() if !ok { return toRPCLogs(nil, false) } @@ -540,9 +559,9 @@ func (s *PublicFilterAPI) GetFilterLogs(filterId string) []vmlog { // GetFilterChanges returns the logs for the filter with the given id since last time is was called. // This can be used for polling. func (s *PublicFilterAPI) GetFilterChanges(filterId string) interface{} { - s.filterMapMu.Lock() + s.filterMapMu.RLock() id, ok := s.filterMapping[filterId] - s.filterMapMu.Unlock() + s.filterMapMu.RUnlock() if !ok { // filter not found return []interface{}{} diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 4343dfa21..256464213 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -82,11 +82,20 @@ func (fs *FilterSystem) Stop() { fs.sub.Unsubscribe() } -// Add adds a filter to the filter manager -func (fs *FilterSystem) Add(filter *Filter, filterType FilterType) (int, error) { +// Acquire filter system maps lock, required to force lock acquisition +// sequence with filterMu acquired first to avoid deadlocks by callbacks +func (fs *FilterSystem) Lock() { fs.filterMu.Lock() - defer fs.filterMu.Unlock() +} + +// Release filter system maps lock +func (fs *FilterSystem) Unlock() { + fs.filterMu.Unlock() +} +// Add adds a filter to the filter manager +// Expects filterMu to be locked. +func (fs *FilterSystem) Add(filter *Filter, filterType FilterType) (int, error) { id := fs.filterId filter.created = time.Now() @@ -110,10 +119,8 @@ func (fs *FilterSystem) Add(filter *Filter, filterType FilterType) (int, error) } // Remove removes a filter by filter id +// Expects filterMu to be locked. func (fs *FilterSystem) Remove(id int) { - fs.filterMu.Lock() - defer fs.filterMu.Unlock() - delete(fs.chainFilters, id) delete(fs.pendingTxFilters, id) delete(fs.logFilters, id) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index a95adfce7..7b714f5d5 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -57,7 +57,7 @@ func BenchmarkMipmaps(b *testing.B) { defer db.Close() genesis := core.WriteGenesisBlockForTesting(db, core.GenesisAccount{Address: addr1, Balance: big.NewInt(1000000)}) - chain, receipts := core.GenerateChain(genesis, db, 100010, func(i int, gen *core.BlockGen) { + chain, receipts := core.GenerateChain(nil, genesis, db, 100010, func(i int, gen *core.BlockGen) { var receipts types.Receipts switch i { case 2403: @@ -133,7 +133,7 @@ func TestFilters(t *testing.T) { defer db.Close() genesis := core.WriteGenesisBlockForTesting(db, core.GenesisAccount{Address: addr, Balance: big.NewInt(1000000)}) - chain, receipts := core.GenerateChain(genesis, db, 1000, func(i int, gen *core.BlockGen) { + chain, receipts := core.GenerateChain(nil, genesis, db, 1000, func(i int, gen *core.BlockGen) { var receipts types.Receipts switch i { case 1: diff --git a/eth/gasprice.go b/eth/gasprice/gasprice.go index ef203f8fe..eb2df4a96 100644 --- a/eth/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package eth +package gasprice import ( "math/big" @@ -23,6 +23,8 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" ) @@ -39,10 +41,22 @@ type blockPriceInfo struct { baseGasPrice *big.Int } +type GpoParams struct { + GpoMinGasPrice *big.Int + GpoMaxGasPrice *big.Int + GpoFullBlockRatio int + GpobaseStepDown int + GpobaseStepUp int + GpobaseCorrectionFactor int +} + // GasPriceOracle recommends gas prices based on the content of recent // blocks. type GasPriceOracle struct { - eth *Ethereum + chain *core.BlockChain + db ethdb.Database + evmux *event.TypeMux + params *GpoParams initOnce sync.Once minPrice *big.Int lastBaseMutex sync.Mutex @@ -55,17 +69,20 @@ type GasPriceOracle struct { } // NewGasPriceOracle returns a new oracle. -func NewGasPriceOracle(eth *Ethereum) *GasPriceOracle { - minprice := eth.GpoMinGasPrice +func NewGasPriceOracle(chain *core.BlockChain, db ethdb.Database, evmux *event.TypeMux, params *GpoParams) *GasPriceOracle { + minprice := params.GpoMinGasPrice if minprice == nil { minprice = big.NewInt(gpoDefaultMinGasPrice) } minbase := new(big.Int).Mul(minprice, big.NewInt(100)) - if eth.GpobaseCorrectionFactor > 0 { - minbase = minbase.Div(minbase, big.NewInt(int64(eth.GpobaseCorrectionFactor))) + if params.GpobaseCorrectionFactor > 0 { + minbase = minbase.Div(minbase, big.NewInt(int64(params.GpobaseCorrectionFactor))) } return &GasPriceOracle{ - eth: eth, + chain: chain, + db: db, + evmux: evmux, + params: params, blocks: make(map[uint64]*blockPriceInfo), minBase: minbase, minPrice: minprice, @@ -75,14 +92,14 @@ func NewGasPriceOracle(eth *Ethereum) *GasPriceOracle { func (gpo *GasPriceOracle) init() { gpo.initOnce.Do(func() { - gpo.processPastBlocks(gpo.eth.BlockChain()) + gpo.processPastBlocks() go gpo.listenLoop() }) } -func (self *GasPriceOracle) processPastBlocks(chain *core.BlockChain) { +func (self *GasPriceOracle) processPastBlocks() { last := int64(-1) - cblock := chain.CurrentBlock() + cblock := self.chain.CurrentBlock() if cblock != nil { last = int64(cblock.NumberU64()) } @@ -92,7 +109,7 @@ func (self *GasPriceOracle) processPastBlocks(chain *core.BlockChain) { } self.firstProcessed = uint64(first) for i := first; i <= last; i++ { - block := chain.GetBlockByNumber(uint64(i)) + block := self.chain.GetBlockByNumber(uint64(i)) if block != nil { self.processBlock(block) } @@ -101,7 +118,7 @@ func (self *GasPriceOracle) processPastBlocks(chain *core.BlockChain) { } func (self *GasPriceOracle) listenLoop() { - events := self.eth.EventMux().Subscribe(core.ChainEvent{}, core.ChainSplitEvent{}) + events := self.evmux.Subscribe(core.ChainEvent{}, core.ChainSplitEvent{}) defer events.Unsubscribe() for event := range events.Chan() { @@ -136,9 +153,9 @@ func (self *GasPriceOracle) processBlock(block *types.Block) { } if lastBase.Cmp(lp) < 0 { - corr = self.eth.GpobaseStepUp + corr = self.params.GpobaseStepUp } else { - corr = -self.eth.GpobaseStepDown + corr = -self.params.GpobaseStepDown } crand := int64(corr * (900 + rand.Intn(201))) @@ -159,14 +176,14 @@ func (self *GasPriceOracle) processBlock(block *types.Block) { self.lastBase = newBase self.lastBaseMutex.Unlock() - glog.V(logger.Detail).Infof("Processed block #%v, base price is %v\n", block.NumberU64(), newBase.Int64()) + glog.V(logger.Detail).Infof("Processed block #%v, base price is %v\n", i, newBase.Int64()) } // returns the lowers possible price with which a tx was or could have been included func (self *GasPriceOracle) lowestPrice(block *types.Block) *big.Int { gasUsed := big.NewInt(0) - receipts := core.GetBlockReceipts(self.eth.ChainDb(), block.Hash(), block.NumberU64()) + receipts := core.GetBlockReceipts(self.db, block.Hash(), block.NumberU64()) if len(receipts) > 0 { if cgu := receipts[len(receipts)-1].CumulativeGasUsed; cgu != nil { gasUsed = receipts[len(receipts)-1].CumulativeGasUsed @@ -174,7 +191,7 @@ func (self *GasPriceOracle) lowestPrice(block *types.Block) *big.Int { } if new(big.Int).Mul(gasUsed, big.NewInt(100)).Cmp(new(big.Int).Mul(block.GasLimit(), - big.NewInt(int64(self.eth.GpoFullBlockRatio)))) < 0 { + big.NewInt(int64(self.params.GpoFullBlockRatio)))) < 0 { // block is not full, could have posted a tx with MinGasPrice return big.NewInt(0) } @@ -201,12 +218,12 @@ func (self *GasPriceOracle) SuggestPrice() *big.Int { price := new(big.Int).Set(self.lastBase) self.lastBaseMutex.Unlock() - price.Mul(price, big.NewInt(int64(self.eth.GpobaseCorrectionFactor))) + price.Mul(price, big.NewInt(int64(self.params.GpobaseCorrectionFactor))) price.Div(price, big.NewInt(100)) if price.Cmp(self.minPrice) < 0 { price.Set(self.minPrice) - } else if self.eth.GpoMaxGasPrice != nil && price.Cmp(self.eth.GpoMaxGasPrice) > 0 { - price.Set(self.eth.GpoMaxGasPrice) + } else if self.params.GpoMaxGasPrice != nil && price.Cmp(self.params.GpoMaxGasPrice) > 0 { + price.Set(self.params.GpoMaxGasPrice) } return price } diff --git a/eth/handler.go b/eth/handler.go index 47a36cc0b..9ad430976 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -45,6 +45,10 @@ const ( estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header ) +var ( + daoChallengeTimeout = 15 * time.Second // Time allowance for a node to reply to the DAO handshake challenge +) + // errIncompatibleConfig is returned if the requested protocols and configs are // not compatible (low protocol version restrictions and high requirements). var errIncompatibleConfig = errors.New("incompatible configuration") @@ -53,18 +57,16 @@ func errResp(code errCode, format string, v ...interface{}) error { return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) } -type hashFetcherFn func(common.Hash) error -type blockFetcherFn func([]common.Hash) error - type ProtocolManager struct { networkId int fastSync uint32 // Flag whether fast sync is enabled (gets disabled if we already have blocks) synced uint32 // Flag whether we're considered synchronised (enables transaction processing) - txpool txPool - blockchain *core.BlockChain - chaindb ethdb.Database + txpool txPool + blockchain *core.BlockChain + chaindb ethdb.Database + chainconfig *core.ChainConfig downloader *downloader.Downloader fetcher *fetcher.Fetcher @@ -99,6 +101,7 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, txpool: txpool, blockchain: blockchain, chaindb: chaindb, + chainconfig: config, peers: newPeerSet(), newPeerCh: make(chan *peer), noMorePeers: make(chan struct{}), @@ -269,15 +272,36 @@ func (pm *ProtocolManager) handle(p *peer) error { defer pm.removePeer(p.id) // Register the peer in the downloader. If the downloader considers it banned, we disconnect - if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(), - p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks, p.RequestHeadersByHash, - p.RequestHeadersByNumber, p.RequestBodies, p.RequestReceipts, p.RequestNodeData); err != nil { + err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(), + p.RequestHeadersByHash, p.RequestHeadersByNumber, + p.RequestBodies, p.RequestReceipts, p.RequestNodeData, + ) + if err != nil { return err } // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. pm.syncTransactions(p) + // If we're DAO hard-fork aware, validate any remote peer with regard to the hard-fork + if daoBlock := pm.chainconfig.DAOForkBlock; daoBlock != nil { + // Request the peer's DAO fork header for extra-data validation + if err := p.RequestHeadersByNumber(daoBlock.Uint64(), 1, 0, false); err != nil { + return err + } + // Start a timer to disconnect if the peer doesn't reply in time + p.forkDrop = time.AfterFunc(daoChallengeTimeout, func() { + glog.V(logger.Warn).Infof("%v: timed out DAO fork-check, dropping", p) + pm.removePeer(p.id) + }) + // Make sure it's cleaned up if the peer dies off + defer func() { + if p.forkDrop != nil { + p.forkDrop.Stop() + p.forkDrop = nil + } + }() + } // main loop. handle incoming messages. for { if err := pm.handleMsg(p); err != nil { @@ -306,108 +330,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Status messages should never arrive after the handshake return errResp(ErrExtraStatusMsg, "uncontrolled status message") - case p.version < eth62 && msg.Code == GetBlockHashesMsg: - // Retrieve the number of hashes to return and from which origin hash - var request getBlockHashesData - if err := msg.Decode(&request); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - if request.Amount > uint64(downloader.MaxHashFetch) { - request.Amount = uint64(downloader.MaxHashFetch) - } - // Retrieve the hashes from the block chain and return them - hashes := pm.blockchain.GetBlockHashesFromHash(request.Hash, request.Amount) - if len(hashes) == 0 { - glog.V(logger.Debug).Infof("invalid block hash %x", request.Hash.Bytes()[:4]) - } - return p.SendBlockHashes(hashes) - - case p.version < eth62 && msg.Code == GetBlockHashesFromNumberMsg: - // Retrieve and decode the number of hashes to return and from which origin number - var request getBlockHashesFromNumberData - if err := msg.Decode(&request); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - if request.Amount > uint64(downloader.MaxHashFetch) { - request.Amount = uint64(downloader.MaxHashFetch) - } - // Calculate the last block that should be retrieved, and short circuit if unavailable - last := pm.blockchain.GetBlockByNumber(request.Number + request.Amount - 1) - if last == nil { - last = pm.blockchain.CurrentBlock() - request.Amount = last.NumberU64() - request.Number + 1 - } - if last.NumberU64() < request.Number { - return p.SendBlockHashes(nil) - } - // Retrieve the hashes from the last block backwards, reverse and return - hashes := []common.Hash{last.Hash()} - hashes = append(hashes, pm.blockchain.GetBlockHashesFromHash(last.Hash(), request.Amount-1)...) - - for i := 0; i < len(hashes)/2; i++ { - hashes[i], hashes[len(hashes)-1-i] = hashes[len(hashes)-1-i], hashes[i] - } - return p.SendBlockHashes(hashes) - - case p.version < eth62 && msg.Code == BlockHashesMsg: - // A batch of hashes arrived to one of our previous requests - var hashes []common.Hash - if err := msg.Decode(&hashes); err != nil { - break - } - // Deliver them all to the downloader for queuing - err := pm.downloader.DeliverHashes(p.id, hashes) - if err != nil { - glog.V(logger.Debug).Infoln(err) - } - - case p.version < eth62 && msg.Code == GetBlocksMsg: - // Decode the retrieval message - msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) - if _, err := msgStream.List(); err != nil { - return err - } - // Gather blocks until the fetch or network limits is reached - var ( - hash common.Hash - bytes common.StorageSize - blocks []*types.Block - ) - for len(blocks) < downloader.MaxBlockFetch && bytes < softResponseLimit { - //Retrieve the hash of the next block - err := msgStream.Decode(&hash) - if err == rlp.EOL { - break - } else if err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Retrieve the requested block, stopping if enough was found - if block := pm.blockchain.GetBlockByHash(hash); block != nil { - blocks = append(blocks, block) - bytes += block.Size() - } - } - return p.SendBlocks(blocks) - - case p.version < eth62 && msg.Code == BlocksMsg: - // Decode the arrived block message - var blocks []*types.Block - if err := msg.Decode(&blocks); err != nil { - glog.V(logger.Detail).Infoln("Decode error", err) - blocks = nil - } - // Update the receive timestamp of each block - for _, block := range blocks { - block.ReceivedAt = msg.ReceivedAt - block.ReceivedFrom = p - } - // Filter out any explicitly requested blocks, deliver the rest to the downloader - if blocks := pm.fetcher.FilterBlocks(blocks); len(blocks) > 0 { - pm.downloader.DeliverBlocks(p.id, blocks) - } - // Block header query, collect the requested headers and reply - case p.version >= eth62 && msg.Code == GetBlockHeadersMsg: + case msg.Code == GetBlockHeadersMsg: // Decode the complex header query var query getBlockHeadersData if err := msg.Decode(&query); err != nil { @@ -475,15 +399,49 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } return p.SendBlockHeaders(headers) - case p.version >= eth62 && msg.Code == BlockHeadersMsg: + case msg.Code == BlockHeadersMsg: // A batch of headers arrived to one of our previous requests var headers []*types.Header if err := msg.Decode(&headers); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } + // If no headers were received, but we're expending a DAO fork check, maybe it's that + if len(headers) == 0 && p.forkDrop != nil { + // Possibly an empty reply to the fork header checks, sanity check TDs + verifyDAO := true + + // If we already have a DAO header, we can check the peer's TD against it. If + // the peer's ahead of this, it too must have a reply to the DAO check + if daoHeader := pm.blockchain.GetHeaderByNumber(pm.chainconfig.DAOForkBlock.Uint64()); daoHeader != nil { + if p.Td().Cmp(pm.blockchain.GetTd(daoHeader.Hash(), daoHeader.Number.Uint64())) >= 0 { + verifyDAO = false + } + } + // If we're seemingly on the same chain, disable the drop timer + if verifyDAO { + glog.V(logger.Debug).Infof("%v: seems to be on the same side of the DAO fork", p) + p.forkDrop.Stop() + p.forkDrop = nil + return nil + } + } // Filter out any explicitly requested headers, deliver the rest to the downloader filter := len(headers) == 1 if filter { + // If it's a potential DAO fork check, validate against the rules + if p.forkDrop != nil && pm.chainconfig.DAOForkBlock.Cmp(headers[0].Number) == 0 { + // Disable the fork drop timer + p.forkDrop.Stop() + p.forkDrop = nil + + // Validate the header and either drop the peer or continue + if err := core.ValidateDAOHeaderExtraData(pm.chainconfig, headers[0]); err != nil { + glog.V(logger.Debug).Infof("%v: verified to be on the other side of the DAO fork, dropping", p) + return err + } + glog.V(logger.Debug).Infof("%v: verified to be on the same side of the DAO fork", p) + } + // Irrelevant of the fork checks, send the header to the fetcher just in case headers = pm.fetcher.FilterHeaders(headers, time.Now()) } if len(headers) > 0 || !filter { @@ -493,7 +451,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } - case p.version >= eth62 && msg.Code == GetBlockBodiesMsg: + case msg.Code == GetBlockBodiesMsg: // Decode the retrieval message msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) if _, err := msgStream.List(); err != nil { @@ -520,7 +478,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } return p.SendBlockBodiesRLP(bodies) - case p.version >= eth62 && msg.Code == BlockBodiesMsg: + case msg.Code == BlockBodiesMsg: // A batch of block bodies arrived to one of our previous requests var request blockBodiesData if err := msg.Decode(&request); err != nil { @@ -671,11 +629,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } for _, block := range unknown { - if p.version < eth62 { - pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestBlocks, nil, nil) - } else { - pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), nil, p.RequestOneHeader, p.RequestBodies) - } + pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) } case msg.Code == NewBlockMsg: @@ -757,11 +711,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Otherwise if the block is indeed in out own chain, announce it if pm.blockchain.HasBlock(hash) { for _, peer := range peers { - if peer.version < eth62 { - peer.SendNewBlockHashes61([]common.Hash{hash}) - } else { - peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) - } + peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) } glog.V(logger.Detail).Infof("announced block %x to %d peers in %v", hash[:4], len(peers), time.Since(block.ReceivedAt)) } diff --git a/eth/handler_test.go b/eth/handler_test.go index 8418c28b2..91989cb8f 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -20,6 +20,7 @@ import ( "math/big" "math/rand" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" ) @@ -61,160 +63,6 @@ func TestProtocolCompatibility(t *testing.T) { } } -// Tests that hashes can be retrieved from a remote chain by hashes in reverse -// order. -func TestGetBlockHashes61(t *testing.T) { testGetBlockHashes(t, 61) } - -func testGetBlockHashes(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxHashFetch - tests := []struct { - origin common.Hash - number int - result int - }{ - {common.Hash{}, 1, 0}, // Make sure non existent hashes don't return results - {pm.blockchain.Genesis().Hash(), 1, 0}, // There are no hashes to retrieve up from the genesis - {pm.blockchain.GetBlockByNumber(5).Hash(), 5, 5}, // All the hashes including the genesis requested - {pm.blockchain.GetBlockByNumber(5).Hash(), 10, 5}, // More hashes than available till the genesis requested - {pm.blockchain.GetBlockByNumber(100).Hash(), 10, 10}, // All hashes available from the middle of the chain - {pm.blockchain.CurrentBlock().Hash(), 10, 10}, // All hashes available from the head of the chain - {pm.blockchain.CurrentBlock().Hash(), limit, limit}, // Request the maximum allowed hash count - {pm.blockchain.CurrentBlock().Hash(), limit + 1, limit}, // Request more than the maximum allowed hash count - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Assemble the hash response we would like to receive - resp := make([]common.Hash, tt.result) - if len(resp) > 0 { - from := pm.blockchain.GetBlockByHash(tt.origin).NumberU64() - 1 - for j := 0; j < len(resp); j++ { - resp[j] = pm.blockchain.GetBlockByNumber(uint64(int(from) - j)).Hash() - } - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x03, getBlockHashesData{tt.origin, uint64(tt.number)}) - if err := p2p.ExpectMsg(peer.app, 0x04, resp); err != nil { - t.Errorf("test %d: block hashes mismatch: %v", i, err) - } - } -} - -// Tests that hashes can be retrieved from a remote chain by numbers in forward -// order. -func TestGetBlockHashesFromNumber61(t *testing.T) { testGetBlockHashesFromNumber(t, 61) } - -func testGetBlockHashesFromNumber(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxHashFetch - tests := []struct { - origin uint64 - number int - result int - }{ - {pm.blockchain.CurrentBlock().NumberU64() + 1, 1, 0}, // Out of bounds requests should return empty - {pm.blockchain.CurrentBlock().NumberU64(), 1, 1}, // Make sure the head hash can be retrieved - {pm.blockchain.CurrentBlock().NumberU64() - 4, 5, 5}, // All hashes, including the head hash requested - {pm.blockchain.CurrentBlock().NumberU64() - 4, 10, 5}, // More hashes requested than available till the head - {pm.blockchain.CurrentBlock().NumberU64() - 100, 10, 10}, // All hashes available from the middle of the chain - {0, 10, 10}, // All hashes available from the root of the chain - {0, limit, limit}, // Request the maximum allowed hash count - {0, limit + 1, limit}, // Request more than the maximum allowed hash count - {0, 1, 1}, // Make sure the genesis hash can be retrieved - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Assemble the hash response we would like to receive - resp := make([]common.Hash, tt.result) - for j := 0; j < len(resp); j++ { - resp[j] = pm.blockchain.GetBlockByNumber(tt.origin + uint64(j)).Hash() - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x08, getBlockHashesFromNumberData{tt.origin, uint64(tt.number)}) - if err := p2p.ExpectMsg(peer.app, 0x04, resp); err != nil { - t.Errorf("test %d: block hashes mismatch: %v", i, err) - } - } -} - -// Tests that blocks can be retrieved from a remote chain based on their hashes. -func TestGetBlocks61(t *testing.T) { testGetBlocks(t, 61) } - -func testGetBlocks(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxBlockFetch - tests := []struct { - random int // Number of blocks to fetch randomly from the chain - explicit []common.Hash // Explicitly requested blocks - available []bool // Availability of explicitly requested blocks - expected int // Total number of existing blocks to expect - }{ - {1, nil, nil, 1}, // A single random block should be retrievable - {10, nil, nil, 10}, // Multiple random blocks should be retrievable - {limit, nil, nil, limit}, // The maximum possible blocks should be retrievable - {limit + 1, nil, nil, limit}, // No more than the possible block count should be returned - {0, []common.Hash{pm.blockchain.Genesis().Hash()}, []bool{true}, 1}, // The genesis block should be retrievable - {0, []common.Hash{pm.blockchain.CurrentBlock().Hash()}, []bool{true}, 1}, // The chains head block should be retrievable - {0, []common.Hash{common.Hash{}}, []bool{false}, 0}, // A non existent block should not be returned - - // Existing and non-existing blocks interleaved should not cause problems - {0, []common.Hash{ - common.Hash{}, - pm.blockchain.GetBlockByNumber(1).Hash(), - common.Hash{}, - pm.blockchain.GetBlockByNumber(10).Hash(), - common.Hash{}, - pm.blockchain.GetBlockByNumber(100).Hash(), - common.Hash{}, - }, []bool{false, true, false, true, false, true, false}, 3}, - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Collect the hashes to request, and the response to expect - hashes, seen := []common.Hash{}, make(map[int64]bool) - blocks := []*types.Block{} - - for j := 0; j < tt.random; j++ { - for { - num := rand.Int63n(int64(pm.blockchain.CurrentBlock().NumberU64())) - if !seen[num] { - seen[num] = true - - block := pm.blockchain.GetBlockByNumber(uint64(num)) - hashes = append(hashes, block.Hash()) - if len(blocks) < tt.expected { - blocks = append(blocks, block) - } - break - } - } - } - for j, hash := range tt.explicit { - hashes = append(hashes, hash) - if tt.available[j] && len(blocks) < tt.expected { - blocks = append(blocks, pm.blockchain.GetBlockByHash(hash)) - } - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x05, hashes) - if err := p2p.ExpectMsg(peer.app, 0x06, blocks); err != nil { - t.Errorf("test %d: blocks mismatch: %v", i, err) - } - } -} - // Tests that block headers can be retrieved from a remote chain based on user queries. func TestGetBlockHeaders62(t *testing.T) { testGetBlockHeaders(t, 62) } func TestGetBlockHeaders63(t *testing.T) { testGetBlockHeaders(t, 63) } @@ -580,3 +428,75 @@ func testGetReceipt(t *testing.T, protocol int) { t.Errorf("receipts mismatch: %v", err) } } + +// Tests that post eth protocol handshake, DAO fork-enabled clients also execute +// a DAO "challenge" verifying each others' DAO fork headers to ensure they're on +// compatible chains. +func TestDAOChallengeNoVsNo(t *testing.T) { testDAOChallenge(t, false, false, false) } +func TestDAOChallengeNoVsPro(t *testing.T) { testDAOChallenge(t, false, true, false) } +func TestDAOChallengeProVsNo(t *testing.T) { testDAOChallenge(t, true, false, false) } +func TestDAOChallengeProVsPro(t *testing.T) { testDAOChallenge(t, true, true, false) } +func TestDAOChallengeNoVsTimeout(t *testing.T) { testDAOChallenge(t, false, false, true) } +func TestDAOChallengeProVsTimeout(t *testing.T) { testDAOChallenge(t, true, true, true) } + +func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool) { + // Reduce the DAO handshake challenge timeout + if timeout { + defer func(old time.Duration) { daoChallengeTimeout = old }(daoChallengeTimeout) + daoChallengeTimeout = 500 * time.Millisecond + } + // Create a DAO aware protocol manager + var ( + evmux = new(event.TypeMux) + pow = new(core.FakePow) + db, _ = ethdb.NewMemDatabase() + genesis = core.WriteGenesisBlockForTesting(db) + config = &core.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} + blockchain, _ = core.NewBlockChain(db, config, pow, evmux) + ) + pm, err := NewProtocolManager(config, false, NetworkId, evmux, new(testTxPool), pow, blockchain, db) + if err != nil { + t.Fatalf("failed to start test protocol manager: %v", err) + } + pm.Start() + defer pm.Stop() + + // Connect a new peer and check that we receive the DAO challenge + peer, _ := newTestPeer("peer", eth63, pm, true) + defer peer.close() + + challenge := &getBlockHeadersData{ + Origin: hashOrNumber{Number: config.DAOForkBlock.Uint64()}, + Amount: 1, + Skip: 0, + Reverse: false, + } + if err := p2p.ExpectMsg(peer.app, GetBlockHeadersMsg, challenge); err != nil { + t.Fatalf("challenge mismatch: %v", err) + } + // Create a block to reply to the challenge if no timeout is simualted + if !timeout { + blocks, _ := core.GenerateChain(nil, genesis, db, 1, func(i int, block *core.BlockGen) { + if remoteForked { + block.SetExtra(params.DAOForkBlockExtra) + } + }) + if err := p2p.Send(peer.app, BlockHeadersMsg, []*types.Header{blocks[0].Header()}); err != nil { + t.Fatalf("failed to answer challenge: %v", err) + } + time.Sleep(100 * time.Millisecond) // Sleep to avoid the verification racing with the drops + } else { + // Otherwise wait until the test timeout passes + time.Sleep(daoChallengeTimeout + 500*time.Millisecond) + } + // Verify that depending on fork side, the remote peer is maintained or dropped + if localForked == remoteForked && !timeout { + if peers := pm.peers.Len(); peers != 1 { + t.Fatalf("peer count mismatch: have %d, want %d", peers, 1) + } + } else { + if peers := pm.peers.Len(); peers != 0 { + t.Fatalf("peer count mismatch: have %d, want %d", peers, 0) + } + } +} diff --git a/eth/helper_test.go b/eth/helper_test.go index dacb1593f..28ff69b17 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -56,7 +56,7 @@ func newTestProtocolManager(fastSync bool, blocks int, generator func(int, *core chainConfig = &core.ChainConfig{HomesteadBlock: big.NewInt(0)} // homestead set to 0 because of chain maker blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux) ) - chain, _ := core.GenerateChain(genesis, db, blocks, generator) + chain, _ := core.GenerateChain(nil, genesis, db, blocks, generator) if _, err := blockchain.InsertChain(chain); err != nil { panic(err) } diff --git a/eth/metrics.go b/eth/metrics.go index e1a89d3a9..5fa2597d4 100644 --- a/eth/metrics.go +++ b/eth/metrics.go @@ -34,14 +34,6 @@ var ( propBlockInTrafficMeter = metrics.NewMeter("eth/prop/blocks/in/traffic") propBlockOutPacketsMeter = metrics.NewMeter("eth/prop/blocks/out/packets") propBlockOutTrafficMeter = metrics.NewMeter("eth/prop/blocks/out/traffic") - reqHashInPacketsMeter = metrics.NewMeter("eth/req/hashes/in/packets") - reqHashInTrafficMeter = metrics.NewMeter("eth/req/hashes/in/traffic") - reqHashOutPacketsMeter = metrics.NewMeter("eth/req/hashes/out/packets") - reqHashOutTrafficMeter = metrics.NewMeter("eth/req/hashes/out/traffic") - reqBlockInPacketsMeter = metrics.NewMeter("eth/req/blocks/in/packets") - reqBlockInTrafficMeter = metrics.NewMeter("eth/req/blocks/in/traffic") - reqBlockOutPacketsMeter = metrics.NewMeter("eth/req/blocks/out/packets") - reqBlockOutTrafficMeter = metrics.NewMeter("eth/req/blocks/out/traffic") reqHeaderInPacketsMeter = metrics.NewMeter("eth/req/headers/in/packets") reqHeaderInTrafficMeter = metrics.NewMeter("eth/req/headers/in/traffic") reqHeaderOutPacketsMeter = metrics.NewMeter("eth/req/headers/out/packets") @@ -95,14 +87,9 @@ func (rw *meteredMsgReadWriter) ReadMsg() (p2p.Msg, error) { // Account for the data traffic packets, traffic := miscInPacketsMeter, miscInTrafficMeter switch { - case rw.version < eth62 && msg.Code == BlockHashesMsg: - packets, traffic = reqHashInPacketsMeter, reqHashInTrafficMeter - case rw.version < eth62 && msg.Code == BlocksMsg: - packets, traffic = reqBlockInPacketsMeter, reqBlockInTrafficMeter - - case rw.version >= eth62 && msg.Code == BlockHeadersMsg: + case msg.Code == BlockHeadersMsg: packets, traffic = reqHeaderInPacketsMeter, reqHeaderInTrafficMeter - case rw.version >= eth62 && msg.Code == BlockBodiesMsg: + case msg.Code == BlockBodiesMsg: packets, traffic = reqBodyInPacketsMeter, reqBodyInTrafficMeter case rw.version >= eth63 && msg.Code == NodeDataMsg: @@ -127,14 +114,9 @@ func (rw *meteredMsgReadWriter) WriteMsg(msg p2p.Msg) error { // Account for the data traffic packets, traffic := miscOutPacketsMeter, miscOutTrafficMeter switch { - case rw.version < eth62 && msg.Code == BlockHashesMsg: - packets, traffic = reqHashOutPacketsMeter, reqHashOutTrafficMeter - case rw.version < eth62 && msg.Code == BlocksMsg: - packets, traffic = reqBlockOutPacketsMeter, reqBlockOutTrafficMeter - - case rw.version >= eth62 && msg.Code == BlockHeadersMsg: + case msg.Code == BlockHeadersMsg: packets, traffic = reqHeaderOutPacketsMeter, reqHeaderOutTrafficMeter - case rw.version >= eth62 && msg.Code == BlockBodiesMsg: + case msg.Code == BlockBodiesMsg: packets, traffic = reqBodyOutPacketsMeter, reqBodyOutTrafficMeter case rw.version >= eth63 && msg.Code == NodeDataMsg: diff --git a/eth/peer.go b/eth/peer.go index 8eb41b0f9..c8c207ecb 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/p2p" @@ -59,10 +58,12 @@ type peer struct { *p2p.Peer rw p2p.MsgReadWriter - version int // Protocol version negotiated - head common.Hash - td *big.Int - lock sync.RWMutex + version int // Protocol version negotiated + forkDrop *time.Timer // Timed connection dropper if forks aren't validated in time + + head common.Hash + td *big.Int + lock sync.RWMutex knownTxs *set.Set // Set of transaction hashes known to be known by this peer knownBlocks *set.Set // Set of block hashes known to be known by this peer @@ -152,25 +153,6 @@ func (p *peer) SendTransactions(txs types.Transactions) error { return p2p.Send(p.rw, TxMsg, txs) } -// SendBlockHashes sends a batch of known hashes to the remote peer. -func (p *peer) SendBlockHashes(hashes []common.Hash) error { - return p2p.Send(p.rw, BlockHashesMsg, hashes) -} - -// SendBlocks sends a batch of blocks to the remote peer. -func (p *peer) SendBlocks(blocks []*types.Block) error { - return p2p.Send(p.rw, BlocksMsg, blocks) -} - -// SendNewBlockHashes61 announces the availability of a number of blocks through -// a hash notification. -func (p *peer) SendNewBlockHashes61(hashes []common.Hash) error { - for _, hash := range hashes { - p.knownBlocks.Add(hash) - } - return p2p.Send(p.rw, NewBlockHashesMsg, hashes) -} - // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -219,26 +201,6 @@ func (p *peer) SendReceiptsRLP(receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, receipts) } -// RequestHashes fetches a batch of hashes from a peer, starting at from, going -// towards the genesis block. -func (p *peer) RequestHashes(from common.Hash) error { - glog.V(logger.Debug).Infof("%v fetching hashes (%d) from %x...", p, downloader.MaxHashFetch, from[:4]) - return p2p.Send(p.rw, GetBlockHashesMsg, getBlockHashesData{from, uint64(downloader.MaxHashFetch)}) -} - -// RequestHashesFromNumber fetches a batch of hashes from a peer, starting at -// the requested block number, going upwards towards the genesis block. -func (p *peer) RequestHashesFromNumber(from uint64, count int) error { - glog.V(logger.Debug).Infof("%v fetching hashes (%d) from #%d...", p, count, from) - return p2p.Send(p.rw, GetBlockHashesFromNumberMsg, getBlockHashesFromNumberData{from, uint64(count)}) -} - -// RequestBlocks fetches a batch of blocks corresponding to the specified hashes. -func (p *peer) RequestBlocks(hashes []common.Hash) error { - glog.V(logger.Debug).Infof("%v fetching %v blocks", p, len(hashes)) - return p2p.Send(p.rw, GetBlocksMsg, hashes) -} - // RequestHeaders is a wrapper around the header query functions to fetch a // single header. It is used solely by the fetcher. func (p *peer) RequestOneHeader(hash common.Hash) error { diff --git a/eth/protocol.go b/eth/protocol.go index 808ac0601..69b3be578 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -28,7 +28,6 @@ import ( // Constants to match up protocol versions and messages const ( - eth61 = 61 eth62 = 62 eth63 = 63 ) @@ -37,10 +36,10 @@ const ( var ProtocolName = "eth" // Supported versions of the eth protocol (first is primary). -var ProtocolVersions = []uint{eth63, eth62, eth61} +var ProtocolVersions = []uint{eth63, eth62} // Number of implemented message corresponding to different protocol versions. -var ProtocolLengths = []uint64{17, 8, 9} +var ProtocolLengths = []uint64{17, 8} const ( NetworkId = 1 @@ -49,26 +48,15 @@ const ( // eth protocol message codes const ( - // Protocol messages belonging to eth/61 - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TxMsg = 0x02 - GetBlockHashesMsg = 0x03 - BlockHashesMsg = 0x04 - GetBlocksMsg = 0x05 - BlocksMsg = 0x06 - NewBlockMsg = 0x07 - GetBlockHashesFromNumberMsg = 0x08 - - // Protocol messages belonging to eth/62 (new protocol from scratch) - // StatusMsg = 0x00 (uncomment after eth/61 deprecation) - // NewBlockHashesMsg = 0x01 (uncomment after eth/61 deprecation) - // TxMsg = 0x02 (uncomment after eth/61 deprecation) + // Protocol messages belonging to eth/62 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TxMsg = 0x02 GetBlockHeadersMsg = 0x03 BlockHeadersMsg = 0x04 GetBlockBodiesMsg = 0x05 BlockBodiesMsg = 0x06 - // NewBlockMsg = 0x07 (uncomment after eth/61 deprecation) + NewBlockMsg = 0x07 // Protocol messages belonging to eth/63 GetNodeDataMsg = 0x0d @@ -117,12 +105,6 @@ type txPool interface { GetTransactions() types.Transactions } -type chainManager interface { - GetBlockHashesFromHash(hash common.Hash, amount uint64) (hashes []common.Hash) - GetBlock(hash common.Hash) (block *types.Block) - Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) -} - // statusData is the network packet for the status message. type statusData struct { ProtocolVersion uint32 @@ -138,19 +120,6 @@ type newBlockHashesData []struct { Number uint64 // Number of one particular block being announced } -// getBlockHashesData is the network packet for the hash based hash retrieval. -type getBlockHashesData struct { - Hash common.Hash - Amount uint64 -} - -// getBlockHashesFromNumberData is the network packet for the number based hash -// retrieval. -type getBlockHashesFromNumberData struct { - Number uint64 - Amount uint64 -} - // getBlockHeadersData represents a block header query. type getBlockHeadersData struct { Origin hashOrNumber // Block from which to retrieve headers @@ -209,8 +178,3 @@ type blockBody struct { // blockBodiesData is the network packet for block content distribution. type blockBodiesData []*blockBody - -// nodeDataData is the network response packet for a node data retrieval. -type nodeDataData []struct { - Value []byte -} diff --git a/eth/protocol_test.go b/eth/protocol_test.go index f860d0a35..4633344da 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -37,7 +37,6 @@ func init() { var testAccount, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") // Tests that handshake failures are detected and reported correctly. -func TestStatusMsgErrors61(t *testing.T) { testStatusMsgErrors(t, 61) } func TestStatusMsgErrors62(t *testing.T) { testStatusMsgErrors(t, 62) } func TestStatusMsgErrors63(t *testing.T) { testStatusMsgErrors(t, 63) } @@ -90,7 +89,6 @@ func testStatusMsgErrors(t *testing.T, protocol int) { } // This test checks that received transactions are added to the local pool. -func TestRecvTransactions61(t *testing.T) { testRecvTransactions(t, 61) } func TestRecvTransactions62(t *testing.T) { testRecvTransactions(t, 62) } func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) } @@ -119,7 +117,6 @@ func testRecvTransactions(t *testing.T, protocol int) { } // This test checks that pending transactions are sent. -func TestSendTransactions61(t *testing.T) { testSendTransactions(t, 61) } func TestSendTransactions62(t *testing.T) { testSendTransactions(t, 62) } func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) } |