diff options
Diffstat (limited to 'core')
36 files changed, 1275 insertions, 551 deletions
diff --git a/core/asm/compiler.go b/core/asm/compiler.go index 318c4e4d8..1b9025a54 100644 --- a/core/asm/compiler.go +++ b/core/asm/compiler.go @@ -122,7 +122,7 @@ func (c *Compiler) next() token { } // compile line compiles a single line instruction e.g. -// "push 1", "jump @labal". +// "push 1", "jump @label". func (c *Compiler) compileLine() error { n := c.next() if n.typ != lineStart { diff --git a/core/asm/lexer.go b/core/asm/lexer.go index a34b2cbd8..4d62159e5 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -48,7 +48,7 @@ const ( lineEnd // emitted when a line ends invalidStatement // any invalid statement element // any element during element parsing - label // label is emitted when a labal is found + label // label is emitted when a label is found labelDef // label definition is emitted when a new label is found number // number is emitted when a number is found stringValue // stringValue is emitted when a string has been found @@ -206,7 +206,7 @@ func lexLine(l *lexer) stateFn { return lexComment case isSpace(r): l.ignore() - case isAlphaNumeric(r) || r == '_': + case isLetter(r) || r == '_': return lexElement case isNumber(r): return lexNumber @@ -278,7 +278,7 @@ func lexElement(l *lexer) stateFn { return lexLine } -func isAlphaNumeric(t rune) bool { +func isLetter(t rune) bool { return unicode.IsLetter(t) } diff --git a/core/bench_test.go b/core/bench_test.go index f976331d1..e23f0d19d 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -173,7 +173,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. - chainman, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() @@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, params.TestChainConfig, ethash.NewFaker(), vm.Config{}) + chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator.go b/core/block_validator.go index 143728bb8..98958809b 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -50,11 +50,14 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin // validated at this point. func (v *BlockValidator) ValidateBody(block *types.Block) error { // Check whether the block's known, and if not, that it's linkable - if v.bc.HasBlockAndState(block.Hash()) { + if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { return ErrKnownBlock } - if !v.bc.HasBlockAndState(block.ParentHash()) { - return consensus.ErrUnknownAncestor + if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { + if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { + return consensus.ErrUnknownAncestor + } + return consensus.ErrPrunedAncestor } // Header validity is known at this point, check the uncles and transactions header := block.Header() diff --git a/core/block_validator_test.go b/core/block_validator_test.go index e668601f3..e334b3c3c 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -42,7 +42,7 @@ func TestHeaderVerification(t *testing.T) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), vm.Config{}) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}) defer chain.Stop() for i := 0; i < len(blocks); i++ { @@ -106,11 +106,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) { var results <-chan error if valid { - chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), vm.Config{}) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } else { - chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } @@ -173,7 +173,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { defer runtime.GOMAXPROCS(old) // Start the verifications and immediately abort - chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}) + chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}) defer chain.Stop() abort, results := chain.engine.VerifyHeaders(chain, headers, seals) diff --git a/core/blockchain.go b/core/blockchain.go index f886ffe4e..b33eb85a4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -42,10 +42,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" + "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) var ( - blockInsertTimer = metrics.NewTimer("chain/inserts") + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) ErrNoGenesis = errors.New("Genesis not found in chain") ) @@ -56,11 +57,20 @@ const ( maxFutureBlocks = 256 maxTimeFutureBlocks = 30 badBlockLimit = 10 + triesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. BlockChainVersion = 3 ) +// CacheConfig contains the configuration values for the trie caching/pruning +// that's resident in a blockchain. +type CacheConfig struct { + Disabled bool // Whether to disable trie write caching (archive node) + TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk +} + // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. // @@ -76,10 +86,14 @@ const ( // included in the canonical one where as GetBlockByNumber always represents the // canonical chain. type BlockChain struct { - config *params.ChainConfig // chain & network configuration + chainConfig *params.ChainConfig // Chain & network configuration + cacheConfig *CacheConfig // Cache configuration for pruning + + db ethdb.Database // Low level persistent database to store final content in + triegc *prque.Prque // Priority queue mapping block numbers to tries to gc + gcproc time.Duration // Accumulates canonical block processing for trie dumping hc *HeaderChain - chainDb ethdb.Database rmLogsFeed event.Feed chainFeed event.Feed chainSideFeed event.Feed @@ -93,8 +107,8 @@ type BlockChain struct { procmu sync.RWMutex // block processor lock checkpoint int // checkpoint counts towards the new checkpoint - currentBlock *types.Block // Current head of the block chain - currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!) + currentBlock atomic.Value // Current head of the block chain + currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies @@ -119,7 +133,13 @@ type BlockChain struct { // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator and // Processor. -func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) { +func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) { + if cacheConfig == nil { + cacheConfig = &CacheConfig{ + TrieNodeLimit: 256 * 1024 * 1024, + TrieTimeLimit: 5 * time.Minute, + } + } bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) @@ -127,9 +147,11 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co badBlocks, _ := lru.New(badBlockLimit) bc := &BlockChain{ - config: config, - chainDb: chainDb, - stateCache: state.NewDatabase(chainDb), + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triegc: prque.New(), + stateCache: state.NewDatabase(db), quit: make(chan struct{}), bodyCache: bodyCache, bodyRLPCache: bodyRLPCache, @@ -139,11 +161,11 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co vmConfig: vmConfig, badBlocks: badBlocks, } - bc.SetValidator(NewBlockValidator(config, bc, engine)) - bc.SetProcessor(NewStateProcessor(config, bc, engine)) + bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) + bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine)) var err error - bc.hc, err = NewHeaderChain(chainDb, config, engine, bc.getProcInterrupt) + bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { return nil, err } @@ -180,7 +202,7 @@ func (bc *BlockChain) getProcInterrupt() bool { // assumes that the chain manager mutex is held. func (bc *BlockChain) loadLastState() error { // Restore the last known head block - head := GetHeadBlockHash(bc.chainDb) + head := GetHeadBlockHash(bc.db) if head == (common.Hash{}) { // Corrupt or empty database, init from scratch log.Warn("Empty database, resetting chain") @@ -196,15 +218,17 @@ func (bc *BlockChain) loadLastState() error { // Make sure the state associated with the block is available if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { // Dangling block without a state associated, init from scratch - log.Warn("Head state missing, resetting chain", "number", currentBlock.Number(), "hash", currentBlock.Hash()) - return bc.Reset() + log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash()) + if err := bc.repair(¤tBlock); err != nil { + return err + } } // Everything seems to be fine, set as the head block - bc.currentBlock = currentBlock + bc.currentBlock.Store(currentBlock) // Restore the last known head header - currentHeader := bc.currentBlock.Header() - if head := GetHeadHeaderHash(bc.chainDb); head != (common.Hash{}) { + currentHeader := currentBlock.Header() + if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { currentHeader = header } @@ -212,21 +236,23 @@ func (bc *BlockChain) loadLastState() error { bc.hc.SetCurrentHeader(currentHeader) // Restore the last known head fast block - bc.currentFastBlock = bc.currentBlock - if head := GetHeadFastBlockHash(bc.chainDb); head != (common.Hash{}) { + bc.currentFastBlock.Store(currentBlock) + if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { - bc.currentFastBlock = block + bc.currentFastBlock.Store(block) } } // Issue a status log for the user + currentFastBlock := bc.CurrentFastBlock() + headerTd := bc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) - blockTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) - fastTd := bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()) + blockTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) + fastTd := bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()) log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd) - log.Info("Loaded most recent local full block", "number", bc.currentBlock.Number(), "hash", bc.currentBlock.Hash(), "td", blockTd) - log.Info("Loaded most recent local fast block", "number", bc.currentFastBlock.Number(), "hash", bc.currentFastBlock.Hash(), "td", fastTd) + log.Info("Loaded most recent local full block", "number", currentBlock.Number(), "hash", currentBlock.Hash(), "td", blockTd) + log.Info("Loaded most recent local fast block", "number", currentFastBlock.Number(), "hash", currentFastBlock.Hash(), "td", fastTd) return nil } @@ -243,7 +269,7 @@ func (bc *BlockChain) SetHead(head uint64) error { // Rewind the header chain, deleting all block bodies until then delFn := func(hash common.Hash, num uint64) { - DeleteBody(bc.chainDb, hash, num) + DeleteBody(bc.db, hash, num) } bc.hc.SetHead(head, delFn) currentHeader := bc.hc.CurrentHeader() @@ -255,30 +281,32 @@ func (bc *BlockChain) SetHead(head uint64) error { bc.futureBlocks.Purge() // Rewind the block chain, ensuring we don't end up with a stateless head block - if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() { - bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) + if currentBlock := bc.CurrentBlock(); currentBlock != nil && currentHeader.Number.Uint64() < currentBlock.NumberU64() { + bc.currentBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())) } - if bc.currentBlock != nil { - if _, err := state.New(bc.currentBlock.Root(), bc.stateCache); err != nil { + if currentBlock := bc.CurrentBlock(); currentBlock != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis - bc.currentBlock = nil + bc.currentBlock.Store(bc.genesisBlock) } } // Rewind the fast block in a simpleton way to the target head - if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() { - bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) + if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock != nil && currentHeader.Number.Uint64() < currentFastBlock.NumberU64() { + bc.currentFastBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())) } // If either blocks reached nil, reset to the genesis state - if bc.currentBlock == nil { - bc.currentBlock = bc.genesisBlock + if currentBlock := bc.CurrentBlock(); currentBlock == nil { + bc.currentBlock.Store(bc.genesisBlock) } - if bc.currentFastBlock == nil { - bc.currentFastBlock = bc.genesisBlock + if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock == nil { + bc.currentFastBlock.Store(bc.genesisBlock) } - if err := WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()); err != nil { + currentBlock := bc.CurrentBlock() + currentFastBlock := bc.CurrentFastBlock() + if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil { log.Crit("Failed to reset head full block", "err", err) } - if err := WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()); err != nil { + if err := WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()); err != nil { log.Crit("Failed to reset head fast block", "err", err) } return bc.loadLastState() @@ -292,12 +320,12 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x…]", hash[:4]) } - if _, err := trie.NewSecure(block.Root(), bc.chainDb, 0); err != nil { + if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB(), 0); err != nil { return err } // If all checks out, manually set the head block bc.mu.Lock() - bc.currentBlock = block + bc.currentBlock.Store(block) bc.mu.Unlock() log.Info("Committed new head block", "number", block.Number(), "hash", hash) @@ -306,45 +334,19 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { // GasLimit returns the gas limit of the current HEAD block. func (bc *BlockChain) GasLimit() uint64 { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentBlock.GasLimit() -} - -// LastBlockHash return the hash of the HEAD block. -func (bc *BlockChain) LastBlockHash() common.Hash { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentBlock.Hash() + return bc.CurrentBlock().GasLimit() } // CurrentBlock retrieves the current head block of the canonical chain. The // block is retrieved from the blockchain's internal cache. func (bc *BlockChain) CurrentBlock() *types.Block { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentBlock + return bc.currentBlock.Load().(*types.Block) } // CurrentFastBlock retrieves the current fast-sync head block of the canonical // chain. The block is retrieved from the blockchain's internal cache. func (bc *BlockChain) CurrentFastBlock() *types.Block { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentFastBlock -} - -// Status returns status information about the current chain such as the HEAD Td, -// the HEAD hash and the hash of the genesis block. -func (bc *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()), bc.currentBlock.Hash(), bc.genesisBlock.Hash() + return bc.currentFastBlock.Load().(*types.Block) } // SetProcessor sets the processor required for making state modifications. @@ -404,22 +406,40 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { log.Crit("Failed to write genesis block TD", "err", err) } - if err := WriteBlock(bc.chainDb, genesis); err != nil { + if err := WriteBlock(bc.db, genesis); err != nil { log.Crit("Failed to write genesis block", "err", err) } bc.genesisBlock = genesis bc.insert(bc.genesisBlock) - bc.currentBlock = bc.genesisBlock + bc.currentBlock.Store(bc.genesisBlock) bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) - bc.currentFastBlock = bc.genesisBlock + bc.currentFastBlock.Store(bc.genesisBlock) return nil } +// repair tries to repair the current blockchain by rolling back the current block +// until one with associated state is found. This is needed to fix incomplete db +// writes caused either by crashes/power outages, or simply non-committed tries. +// +// This method only rolls back the current block. The current header and current +// fast block are left intact. +func (bc *BlockChain) repair(head **types.Block) error { + for { + // Abort if we've rewound to a head block that does have associated state + if _, err := state.New((*head).Root(), bc.stateCache); err == nil { + log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash()) + return nil + } + // Otherwise rewind one block and recheck state availability there + (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + } +} + // Export writes the active chain to the given writer. func (bc *BlockChain) Export(w io.Writer) error { - return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64()) + return bc.ExportN(w, uint64(0), bc.CurrentBlock().NumberU64()) } // ExportN writes a subset of the active chain to the given writer. @@ -454,25 +474,25 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { // Note, this function assumes that the `mu` mutex is held! func (bc *BlockChain) insert(block *types.Block) { // If the block is on a side chain or an unknown one, force other heads onto it too - updateHeads := GetCanonicalHash(bc.chainDb, block.NumberU64()) != block.Hash() + updateHeads := GetCanonicalHash(bc.db, block.NumberU64()) != block.Hash() // Add the block to the canonical chain number scheme and mark as the head - if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil { + if err := WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()); err != nil { log.Crit("Failed to insert block number", "err", err) } - if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil { + if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil { log.Crit("Failed to insert head block hash", "err", err) } - bc.currentBlock = block + bc.currentBlock.Store(block) // If the block is better than our head or is on a different chain, force update heads if updateHeads { bc.hc.SetCurrentHeader(block.Header()) - if err := WriteHeadFastBlockHash(bc.chainDb, block.Hash()); err != nil { + if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil { log.Crit("Failed to insert head fast block hash", "err", err) } - bc.currentFastBlock = block + bc.currentFastBlock.Store(block) } } @@ -489,7 +509,7 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body { body := cached.(*types.Body) return body } - body := GetBody(bc.chainDb, hash, bc.hc.GetBlockNumber(hash)) + body := GetBody(bc.db, hash, bc.hc.GetBlockNumber(hash)) if body == nil { return nil } @@ -505,7 +525,7 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { if cached, ok := bc.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(bc.chainDb, hash, bc.hc.GetBlockNumber(hash)) + body := GetBodyRLP(bc.db, hash, bc.hc.GetBlockNumber(hash)) if len(body) == 0 { return nil } @@ -519,21 +539,25 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { return true } - ok, _ := bc.chainDb.Has(blockBodyKey(hash, number)) + ok, _ := bc.db.Has(blockBodyKey(hash, number)) return ok } +// HasState checks if state trie is fully present in the database or not. +func (bc *BlockChain) HasState(hash common.Hash) bool { + _, err := bc.stateCache.OpenTrie(hash) + return err == nil +} + // HasBlockAndState checks if a block and associated state trie is fully present // in the database or not, caching it if present. -func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { +func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { // Check first that the block itself is known - block := bc.GetBlockByHash(hash) + block := bc.GetBlock(hash, number) if block == nil { return false } - // Ensure the associated state is also present - _, err := bc.stateCache.OpenTrie(block.Root()) - return err == nil + return bc.HasState(block.Root()) } // GetBlock retrieves a block from the database by hash and number, @@ -543,7 +567,7 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { if block, ok := bc.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(bc.chainDb, hash, number) + block := GetBlock(bc.db, hash, number) if block == nil { return nil } @@ -560,13 +584,18 @@ func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { - hash := GetCanonicalHash(bc.chainDb, number) + hash := GetCanonicalHash(bc.db, number) if hash == (common.Hash{}) { return nil } return bc.GetBlock(hash, number) } +// GetReceiptsByHash retrieves the receipts for all transactions in a given block. +func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { + return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash)) +} + // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. // [deprecated by eth/62] func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { @@ -594,6 +623,12 @@ func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types. return uncles } +// TrieNode retrieves a blob of data associated with a trie node (or code hash) +// either from ephemeral in-memory cache, or from persistent storage. +func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { + return bc.stateCache.TrieDB().Node(hash) +} + // Stop stops the blockchain service. If any imports are currently in progress // it will abort them using the procInterrupt. func (bc *BlockChain) Stop() { @@ -606,6 +641,32 @@ func (bc *BlockChain) Stop() { atomic.StoreInt32(&bc.procInterrupt, 1) bc.wg.Wait() + + // Ensure the state of a recent block is also stored to disk before exiting. + // We're writing three different states to catch different restart scenarios: + // - HEAD: So we don't need to reprocess any blocks in the general case + // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle + // - HEAD-127: So we have a hard limit on the number of blocks reexecuted + if !bc.cacheConfig.Disabled { + triedb := bc.stateCache.TrieDB() + + for _, offset := range []uint64{0, 1, triesInMemory - 1} { + if number := bc.CurrentBlock().NumberU64(); number > offset { + recent := bc.GetBlockByNumber(number - offset) + + log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) + if err := triedb.Commit(recent.Root(), true); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } + } + } + for !bc.triegc.Empty() { + triedb.Dereference(bc.triegc.PopItem().(common.Hash), common.Hash{}) + } + if size := triedb.Size(); size != 0 { + log.Error("Dangling trie nodes after full cleanup") + } + } log.Info("Blockchain manager stopped") } @@ -648,22 +709,27 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { if currentHeader.Hash() == hash { bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) } - if bc.currentFastBlock.Hash() == hash { - bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1) - WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()) + if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash { + newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) + bc.currentFastBlock.Store(newFastBlock) + WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) } - if bc.currentBlock.Hash() == hash { - bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1) - WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()) + if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { + newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) + bc.currentBlock.Store(newBlock) + WriteHeadBlockHash(bc.db, newBlock.Hash()) } } } // SetReceiptsData computes all the non-consensus fields of the receipts -func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts types.Receipts) { +func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts types.Receipts) error { signer := types.MakeSigner(config, block.Number()) transactions, logIndex := block.Transactions(), uint(0) + if len(transactions) != len(receipts) { + return errors.New("transaction and receipt count mismatch") + } for j := 0; j < len(receipts); j++ { // The transaction hash can be retrieved from the transaction itself @@ -691,6 +757,7 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty logIndex++ } } + return nil } // InsertReceiptChain attempts to complete an already existing header chain with @@ -713,7 +780,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ stats = struct{ processed, ignored int32 }{} start = time.Now() bytes = 0 - batch = bc.chainDb.NewBatch() + batch = bc.db.NewBatch() ) for i, block := range blockChain { receipts := receiptChain[i] @@ -731,7 +798,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ continue } // Compute all the non-consensus fields of the receipts - SetReceiptsData(bc.config, block, receipts) + if err := SetReceiptsData(bc.chainConfig, block, receipts); err != nil { + return i, fmt.Errorf("failed to set receipts data: %v", err) + } // Write all the data out into the database if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil { return i, fmt.Errorf("failed to write block body: %v", err) @@ -749,7 +818,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return 0, err } bytes += batch.ValueSize() - batch = bc.chainDb.NewBatch() + batch.Reset() } } if batch.ValueSize() > 0 { @@ -763,11 +832,12 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ bc.mu.Lock() head := blockChain[len(blockChain)-1] if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case - if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 { - if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil { + currentFastBlock := bc.CurrentFastBlock() + if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 { + if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil { log.Crit("Failed to update head fast block hash", "err", err) } - bc.currentFastBlock = head + bc.currentFastBlock.Store(head) } } bc.mu.Unlock() @@ -775,15 +845,33 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ log.Info("Imported new block receipts", "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), - "bytes", bytes, "number", head.Number(), "hash", head.Hash(), + "size", common.StorageSize(bytes), "ignored", stats.ignored) return 0, nil } -// WriteBlock writes the block to the chain. -func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { +var lastWrite uint64 + +// WriteBlockWithoutState writes only the block and its metadata to the database, +// but does not write any state. This is used to construct competing side forks +// up to the point where they exceed the canonical total difficulty. +func (bc *BlockChain) WriteBlockWithoutState(block *types.Block, td *big.Int) (err error) { + bc.wg.Add(1) + defer bc.wg.Done() + + if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil { + return err + } + if err := WriteBlock(bc.db, block); err != nil { + return err + } + return nil +} + +// WriteBlockWithState writes the block and all associated state to the database. +func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -796,7 +884,8 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R bc.mu.Lock() defer bc.mu.Unlock() - localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) + currentBlock := bc.CurrentBlock() + localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) // Irrelevant of the canonical status, write the block itself to the database @@ -804,29 +893,82 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R return NonStatTy, err } // Write other block data using a batch. - batch := bc.chainDb.NewBatch() + batch := bc.db.NewBatch() if err := WriteBlock(batch, block); err != nil { return NonStatTy, err } - if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil { + root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) + if err != nil { return NonStatTy, err } + triedb := bc.stateCache.TrieDB() + + // If we're running an archive node, always flush + if bc.cacheConfig.Disabled { + if err := triedb.Commit(root, false); err != nil { + return NonStatTy, err + } + } else { + // Full but not archive node, do proper garbage collection + triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive + bc.triegc.Push(root, -float32(block.NumberU64())) + + if current := block.NumberU64(); current > triesInMemory { + // Find the next state trie we need to commit + header := bc.GetHeaderByNumber(current - triesInMemory) + chosen := header.Number.Uint64() + + // Only write to disk if we exceeded our memory allowance *and* also have at + // least a given number of tries gapped. + var ( + size = triedb.Size() + limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024 + ) + if size > limit || bc.gcproc > bc.cacheConfig.TrieTimeLimit { + // If we're exceeding limits but haven't reached a large enough memory gap, + // warn the user that the system is becoming unstable. + if chosen < lastWrite+triesInMemory { + switch { + case size >= 2*limit: + log.Warn("State memory usage too high, committing", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory) + case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit: + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory) + } + } + // If optimum or critical limits reached, write to disk + if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + triedb.Commit(header.Root, true) + lastWrite = chosen + bc.gcproc = 0 + } + } + // Garbage collect anything below our required write retention + for !bc.triegc.Empty() { + root, number := bc.triegc.Pop() + if uint64(-number) > chosen { + bc.triegc.Push(root, number) + break + } + triedb.Dereference(root.(common.Hash), common.Hash{}) + } + } + } if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { return NonStatTy, err } - // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf reorg := externTd.Cmp(localTd) > 0 + currentBlock = bc.CurrentBlock() if !reorg && externTd.Cmp(localTd) == 0 { // Split same-difficulty blocks by number, then at random - reorg = block.NumberU64() < bc.currentBlock.NumberU64() || (block.NumberU64() == bc.currentBlock.NumberU64() && mrand.Float64() < 0.5) + reorg = block.NumberU64() < currentBlock.NumberU64() || (block.NumberU64() == currentBlock.NumberU64() && mrand.Float64() < 0.5) } if reorg { // Reorganise the chain if the parent is not the head block - if block.ParentHash() != bc.currentBlock.Hash() { - if err := bc.reorg(bc.currentBlock, block); err != nil { + if block.ParentHash() != currentBlock.Hash() { + if err := bc.reorg(currentBlock, block); err != nil { return NonStatTy, err } } @@ -835,7 +977,7 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R return NonStatTy, err } // Write hash preimages - if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { + if err := WritePreimages(bc.db, block.NumberU64(), state.Preimages()); err != nil { return NonStatTy, err } status = CanonStatTy @@ -927,31 +1069,65 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty if err == nil { err = bc.Validator().ValidateBody(block) } - if err != nil { - if err == ErrKnownBlock { + switch { + case err == ErrKnownBlock: + // Block and state both already known. However if the current block is below + // this number we did a rollback and we should reimport it nonetheless. + if bc.CurrentBlock().NumberU64() >= block.NumberU64() { stats.ignored++ continue } - if err == consensus.ErrFutureBlock { - // Allow up to MaxFuture second in the future blocks. If this limit - // is exceeded the chain is discarded and processed at a later time - // if given. - max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) - if block.Time().Cmp(max) > 0 { - return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max) + case err == consensus.ErrFutureBlock: + // Allow up to MaxFuture second in the future blocks. If this limit is exceeded + // the chain is discarded and processed at a later time if given. + max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) + if block.Time().Cmp(max) > 0 { + return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max) + } + bc.futureBlocks.Add(block.Hash(), block) + stats.queued++ + continue + + case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()): + bc.futureBlocks.Add(block.Hash(), block) + stats.queued++ + continue + + case err == consensus.ErrPrunedAncestor: + // Block competing with the canonical chain, store in the db, but don't process + // until the competitor TD goes above the canonical TD + currentBlock := bc.CurrentBlock() + localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) + externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty()) + if localTd.Cmp(externTd) > 0 { + if err = bc.WriteBlockWithoutState(block, externTd); err != nil { + return i, events, coalescedLogs, err } - bc.futureBlocks.Add(block.Hash(), block) - stats.queued++ continue } + // Competitor chain beat canonical, gather all blocks from the common ancestor + var winner []*types.Block - if err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()) { - bc.futureBlocks.Add(block.Hash(), block) - stats.queued++ - continue + parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1) + for !bc.HasState(parent.Root()) { + winner = append(winner, parent) + parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) + } + for j := 0; j < len(winner)/2; j++ { + winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j] + } + // Import all the pruned blocks to make the state available + bc.chainmu.Unlock() + _, evs, logs, err := bc.insertChain(winner) + bc.chainmu.Lock() + events, coalescedLogs = evs, logs + + if err != nil { + return i, events, coalescedLogs, err } + case err != nil: bc.reportBlock(block, nil, err) return i, events, coalescedLogs, err } @@ -979,8 +1155,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty bc.reportBlock(block, receipts, err) return i, events, coalescedLogs, err } + proctime := time.Since(bstart) + // Write the block to the chain and get the status. - status, err := bc.WriteBlockAndState(block, receipts, state) + status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { return i, events, coalescedLogs, err } @@ -994,6 +1172,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty events = append(events, ChainEvent{block, block.Hash(), logs}) lastCanon = block + // Only count canonical blocks for GC processing time + bc.gcproc += proctime + case SideStatTy: log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles())) @@ -1003,10 +1184,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } stats.processed++ stats.usedGas += usedGas - stats.report(chain, i) + stats.report(chain, i, bc.stateCache.TrieDB().Size()) } // Append a single chain head event if we've progressed the chain - if lastCanon != nil && bc.LastBlockHash() == lastCanon.Hash() { + if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() { events = append(events, ChainHeadEvent{lastCanon}) } return 0, events, coalescedLogs, nil @@ -1026,7 +1207,7 @@ const statsReportLimit = 8 * time.Second // report prints statistics if some number of blocks have been processed // or more than a few seconds have passed since the last message. -func (st *insertStats) report(chain []*types.Block, index int) { +func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) { // Fetch the timings for the batch var ( now = mclock.Now() @@ -1041,7 +1222,7 @@ func (st *insertStats) report(chain []*types.Block, index int) { context := []interface{}{ "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000, "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed), - "number", end.Number(), "hash", end.Hash(), + "number", end.Number(), "hash", end.Hash(), "cache", cache, } if st.queued > 0 { context = append(context, []interface{}{"queued", st.queued}...) @@ -1077,7 +1258,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // These logs are later announced as deleted. collectLogs = func(h common.Hash) { // Coalesce logs and set 'Removed'. - receipts := GetBlockReceipts(bc.chainDb, h, bc.hc.GetBlockNumber(h)) + receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h)) for _, receipt := range receipts { for _, log := range receipt.Logs { del := *log @@ -1146,7 +1327,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // insert the block in the canonical way, re-writing history bc.insert(newChain[i]) // write lookup entries for hash based transaction/receipt searches - if err := WriteTxLookupEntries(bc.chainDb, newChain[i]); err != nil { + if err := WriteTxLookupEntries(bc.db, newChain[i]); err != nil { return err } addedTxs = append(addedTxs, newChain[i].Transactions()...) @@ -1156,7 +1337,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // When transactions get deleted from the database that means the // receipts that were created in the fork must also be deleted for _, tx := range diff { - DeleteTxLookupEntry(bc.chainDb, tx.Hash()) + DeleteTxLookupEntry(bc.db, tx.Hash()) } if len(deletedLogs) > 0 { go bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) @@ -1248,7 +1429,7 @@ Hash: 0x%x Error: %v ############################## -`, bc.config, block.Number(), block.Hash(), receiptString, err)) +`, bc.chainConfig, block.Number(), block.Hash(), receiptString, err)) } // InsertHeaderChain attempts to insert the given header chain in to the local @@ -1306,9 +1487,6 @@ func (bc *BlockChain) writeHeader(header *types.Header) error { // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. func (bc *BlockChain) CurrentHeader() *types.Header { - bc.mu.RLock() - defer bc.mu.RUnlock() - return bc.hc.CurrentHeader() } @@ -1355,7 +1533,7 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { } // Config retrieves the blockchain's chain configuration. -func (bc *BlockChain) Config() *params.ChainConfig { return bc.config } +func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig } // Engine retrieves the blockchain's consensus engine. func (bc *BlockChain) Engine() consensus.Engine { return bc.engine } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index cbde3bcd2..748cdc5c7 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -34,26 +34,6 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// newTestBlockChain creates a blockchain without validation. -func newTestBlockChain(fake bool) *BlockChain { - db, _ := ethdb.NewMemDatabase() - gspec := &Genesis{ - Config: params.TestChainConfig, - Difficulty: big.NewInt(1), - } - gspec.MustCommit(db) - engine := ethash.NewFullFaker() - if !fake { - engine = ethash.NewTester() - } - blockchain, err := NewBlockChain(db, gspec.Config, engine, vm.Config{}) - if err != nil { - panic(err) - } - blockchain.SetValidator(bproc{}) - return blockchain -} - // Test fork of length N starting from block i func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db @@ -148,9 +128,9 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { return err } blockchain.mu.Lock() - WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) - WriteBlock(blockchain.chainDb, block) - statedb.CommitTo(blockchain.chainDb, false) + WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) + WriteBlock(blockchain.db, block) + statedb.Commit(false) blockchain.mu.Unlock() } return nil @@ -166,8 +146,8 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } // Manually insert the header into the database, but don't reorganise (allows subsequent testing) blockchain.mu.Lock() - WriteTd(blockchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) - WriteHeader(blockchain.chainDb, header) + WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) + WriteHeader(blockchain.db, header) blockchain.mu.Unlock() } return nil @@ -183,13 +163,18 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t * } func TestLastBlock(t *testing.T) { - bchain := newTestBlockChain(false) - defer bchain.Stop() + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() - block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.chainDb, 0)[0] - bchain.insert(block) - if block.Hash() != GetHeadBlockHash(bchain.chainDb) { - t.Errorf("Write/Get HeadBlockHash failed") + blocks := makeBlockChain(blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), blockchain.db, 0) + if _, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("Failed to insert block: %v", err) + } + if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) { + t.Fatalf("Write/Get HeadBlockHash failed") } } @@ -337,55 +322,13 @@ func testBrokenChain(t *testing.T, full bool) { } } -type bproc struct{} - -func (bproc) ValidateBody(*types.Block) error { return nil } -func (bproc) ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error { - return nil -} -func (bproc) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) { - return nil, nil, 0, nil -} - -func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header { - blocks := makeBlockChainWithDiff(genesis, d, seed) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - return headers -} - -func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block { - var chain []*types.Block - for i, difficulty := range d { - header := &types.Header{ - Coinbase: common.Address{seed}, - Number: big.NewInt(int64(i + 1)), - Difficulty: big.NewInt(int64(difficulty)), - UncleHash: types.EmptyUncleHash, - TxHash: types.EmptyRootHash, - ReceiptHash: types.EmptyRootHash, - Time: big.NewInt(int64(i) + 1), - } - if i == 0 { - header.ParentHash = genesis.Hash() - } else { - header.ParentHash = chain[i-1].Hash() - } - block := types.NewBlockWithHeader(header) - chain = append(chain, block) - } - return chain -} - // Tests that reorganising a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) } func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true) } func testReorgLong(t *testing.T, full bool) { - testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full) + testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280, full) } // Tests that reorganising a short difficult chain after a long easy one @@ -394,45 +337,82 @@ func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) } func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } func testReorgShort(t *testing.T, full bool) { - testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full) + // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment + // we need a fairly long chain of blocks with different difficulties for a short + // one to become heavyer than a long one. The 96 is an empirical value. + easy := make([]int64, 96) + for i := 0; i < len(easy); i++ { + easy[i] = 60 + } + diff := make([]int64, len(easy)-1) + for i := 0; i < len(diff); i++ { + diff[i] = -9 + } + testReorg(t, easy, diff, 12615120, full) } -func testReorg(t *testing.T, first, second []int, td int64, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() +func testReorg(t *testing.T, first, second []int64, td int64, full bool) { + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() // Insert an easy and a difficult chain afterwards + easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(first), func(i int, b *BlockGen) { + b.OffsetTime(first[i]) + }) + diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(second), func(i int, b *BlockGen) { + b.OffsetTime(second[i]) + }) if full { - bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, first, 11)) - bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, second, 22)) + if _, err := blockchain.InsertChain(easyBlocks); err != nil { + t.Fatalf("failed to insert easy chain: %v", err) + } + if _, err := blockchain.InsertChain(diffBlocks); err != nil { + t.Fatalf("failed to insert difficult chain: %v", err) + } } else { - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, first, 11), 1) - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, second, 22), 1) + easyHeaders := make([]*types.Header, len(easyBlocks)) + for i, block := range easyBlocks { + easyHeaders[i] = block.Header() + } + diffHeaders := make([]*types.Header, len(diffBlocks)) + for i, block := range diffBlocks { + diffHeaders[i] = block.Header() + } + if _, err := blockchain.InsertHeaderChain(easyHeaders, 1); err != nil { + t.Fatalf("failed to insert easy chain: %v", err) + } + if _, err := blockchain.InsertHeaderChain(diffHeaders, 1); err != nil { + t.Fatalf("failed to insert difficult chain: %v", err) + } } // Check that the chain is valid number and link wise if full { - prev := bc.CurrentBlock() - for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) { + prev := blockchain.CurrentBlock() + for block := blockchain.GetBlockByNumber(blockchain.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, blockchain.GetBlockByNumber(block.NumberU64()-1) { if prev.ParentHash() != block.Hash() { t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash()) } } } else { - prev := bc.CurrentHeader() - for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) { + prev := blockchain.CurrentHeader() + for header := blockchain.GetHeaderByNumber(blockchain.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, blockchain.GetHeaderByNumber(header.Number.Uint64()-1) { if prev.ParentHash != header.Hash() { t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash()) } } } // Make sure the chain total difficulty is the correct one - want := new(big.Int).Add(bc.genesisBlock.Difficulty(), big.NewInt(td)) + want := new(big.Int).Add(blockchain.genesisBlock.Difficulty(), big.NewInt(td)) if full { - if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { + if have := blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } else { - if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { + if have := blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } @@ -443,19 +423,28 @@ func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) } func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) } func testBadHashes(t *testing.T, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() // Create a chain, ban a hash and try to import - var err error if full { - blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) + blocks := makeBlockChain(blockchain.CurrentBlock(), 3, ethash.NewFaker(), db, 10) + BadHashes[blocks[2].Header().Hash()] = true - _, err = bc.InsertChain(blocks) + defer func() { delete(BadHashes, blocks[2].Header().Hash()) }() + + _, err = blockchain.InsertChain(blocks) } else { - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) + headers := makeHeaderChain(blockchain.CurrentHeader(), 3, ethash.NewFaker(), db, 10) + BadHashes[headers[2].Hash()] = true - _, err = bc.InsertHeaderChain(headers, 1) + defer func() { delete(BadHashes, headers[2].Hash()) }() + + _, err = blockchain.InsertHeaderChain(headers, 1) } if err != ErrBlacklistedHash { t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash) @@ -468,40 +457,41 @@ func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) } func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) } func testReorgBadHashes(t *testing.T, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() - + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } // Create a chain, import and ban afterwards - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) - blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) + headers := makeHeaderChain(blockchain.CurrentHeader(), 4, ethash.NewFaker(), db, 10) + blocks := makeBlockChain(blockchain.CurrentBlock(), 4, ethash.NewFaker(), db, 10) if full { - if _, err := bc.InsertChain(blocks); err != nil { - t.Fatalf("failed to import blocks: %v", err) + if _, err = blockchain.InsertChain(blocks); err != nil { + t.Errorf("failed to import blocks: %v", err) } - if bc.CurrentBlock().Hash() != blocks[3].Hash() { - t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash()) + if blockchain.CurrentBlock().Hash() != blocks[3].Hash() { + t.Errorf("last block hash mismatch: have: %x, want %x", blockchain.CurrentBlock().Hash(), blocks[3].Header().Hash()) } BadHashes[blocks[3].Header().Hash()] = true defer func() { delete(BadHashes, blocks[3].Header().Hash()) }() } else { - if _, err := bc.InsertHeaderChain(headers, 1); err != nil { - t.Fatalf("failed to import headers: %v", err) + if _, err = blockchain.InsertHeaderChain(headers, 1); err != nil { + t.Errorf("failed to import headers: %v", err) } - if bc.CurrentHeader().Hash() != headers[3].Hash() { - t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash()) + if blockchain.CurrentHeader().Hash() != headers[3].Hash() { + t.Errorf("last header hash mismatch: have: %x, want %x", blockchain.CurrentHeader().Hash(), headers[3].Hash()) } BadHashes[headers[3].Hash()] = true defer func() { delete(BadHashes, headers[3].Hash()) }() } + blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(bc.chainDb, bc.config, ethash.NewFaker(), vm.Config{}) + ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } - defer ncm.Stop() - if full { if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() { t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash()) @@ -514,6 +504,7 @@ func testReorgBadHashes(t *testing.T, full bool) { t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash()) } } + ncm.Stop() } // Tests chain insertions in the face of one entity containing an invalid nonce. @@ -609,7 +600,7 @@ func TestFastVsFullChains(t *testing.T) { // Import the chain as an archive node for the comparison baseline archiveDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) - archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), vm.Config{}) + archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer archive.Stop() if n, err := archive.InsertChain(blocks); err != nil { @@ -618,7 +609,7 @@ func TestFastVsFullChains(t *testing.T) { // Fast import the chain as a non-archive node to test fastDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) - fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), vm.Config{}) + fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -696,7 +687,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) - archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), vm.Config{}) + archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) } @@ -709,7 +700,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) - fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), vm.Config{}) + fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -730,7 +721,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { lightDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(lightDb) - light, _ := NewBlockChain(lightDb, gspec.Config, ethash.NewFaker(), vm.Config{}) + light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -799,7 +790,7 @@ func TestChainTxReorgs(t *testing.T) { } }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -870,7 +861,7 @@ func TestLogReorgs(t *testing.T) { signer = types.NewEIP155Signer(gspec.Config.ChainId) ) - blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) @@ -917,7 +908,7 @@ func TestReorgSideEvent(t *testing.T) { signer = types.NewEIP155Signer(gspec.Config.ChainId) ) - blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer blockchain.Stop() chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}) @@ -989,10 +980,13 @@ done: // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { - bc := newTestBlockChain(true) - defer bc.Stop() + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() - chain, _ := GenerateChain(bc.config, bc.genesisBlock, ethash.NewFaker(), bc.chainDb, 10, func(i int, gen *BlockGen) {}) + chain, _ := GenerateChain(blockchain.chainConfig, blockchain.genesisBlock, ethash.NewFaker(), blockchain.db, 10, func(i int, gen *BlockGen) {}) var pend sync.WaitGroup pend.Add(len(chain)) @@ -1003,14 +997,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) { // try to retrieve a block by its canonical hash and see if the block data can be retrieved. for { - ch := GetCanonicalHash(bc.chainDb, block.NumberU64()) + ch := GetCanonicalHash(blockchain.db, block.NumberU64()) if ch == (common.Hash{}) { continue // busy wait for canonical hash to be written } if ch != block.Hash() { t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex()) } - fb := GetBlock(bc.chainDb, ch, block.NumberU64()) + fb := GetBlock(blockchain.db, ch, block.NumberU64()) if fb == nil { t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex()) } @@ -1021,7 +1015,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { } }(chain[i]) - if _, err := bc.InsertChain(types.Blocks{chain[i]}); err != nil { + if _, err := blockchain.InsertChain(types.Blocks{chain[i]}); err != nil { t.Fatalf("failed to insert block %d: %v", i, err) } } @@ -1043,7 +1037,7 @@ func TestEIP155Transition(t *testing.T) { genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { @@ -1151,7 +1145,7 @@ func TestEIP161AccountRemoval(t *testing.T) { } genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) { @@ -1226,7 +1220,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { diskdb, _ := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, params.TestChainConfig, engine, vm.Config{}) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1245,3 +1239,102 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } } } + +// Tests that importing small side forks doesn't leave junk in the trie database +// cache (which would eventually cause memory issues). +func TestTrieForkGC(t *testing.T) { + // Generate a canonical chain to act as the main dataset + engine := ethash.NewFaker() + + db, _ := ethdb.NewMemDatabase() + genesis := new(Genesis).MustCommit(db) + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + + // Generate a bunch of fork blocks, each side forking from the canonical chain + forks := make([]*types.Block, len(blocks)) + for i := 0; i < len(forks); i++ { + parent := genesis + if i > 0 { + parent = blocks[i-1] + } + fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + forks[i] = fork[0] + } + // Import the canonical and fork chain side by side, forcing the trie cache to cache both + diskdb, _ := ethdb.NewMemDatabase() + new(Genesis).MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + for i := 0; i < len(blocks); i++ { + if _, err := chain.InsertChain(blocks[i : i+1]); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", i, err) + } + if _, err := chain.InsertChain(forks[i : i+1]); err != nil { + t.Fatalf("fork %d: failed to insert into chain: %v", i, err) + } + } + // Dereference all the recent tries and ensure no past trie is left in + for i := 0; i < triesInMemory; i++ { + chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root(), common.Hash{}) + chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root(), common.Hash{}) + } + if len(chain.stateCache.TrieDB().Nodes()) > 0 { + t.Fatalf("stale tries still alive after garbase collection") + } +} + +// Tests that doing large reorgs works even if the state associated with the +// forking point is not available any more. +func TestLargeReorgTrieGC(t *testing.T) { + // Generate the original common chain segment and the two competing forks + engine := ethash.NewFaker() + + db, _ := ethdb.NewMemDatabase() + genesis := new(Genesis).MustCommit(db) + + shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) + + // Import the shared chain and the original canonical one + diskdb, _ := ethdb.NewMemDatabase() + new(Genesis).MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + if _, err := chain.InsertChain(shared); err != nil { + t.Fatalf("failed to insert shared chain: %v", err) + } + if _, err := chain.InsertChain(original); err != nil { + t.Fatalf("failed to insert shared chain: %v", err) + } + // Ensure that the state associated with the forking point is pruned away + if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil { + t.Fatalf("common-but-old ancestor still cache") + } + // Import the competitor chain without exceeding the canonical's TD and ensure + // we have not processed any of the blocks (protection against malicious blocks) + if _, err := chain.InsertChain(competitor[:len(competitor)-2]); err != nil { + t.Fatalf("failed to insert competitor chain: %v", err) + } + for i, block := range competitor[:len(competitor)-2] { + if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { + t.Fatalf("competitor %d: low TD chain became processed", i) + } + } + // Import the head of the competitor chain, triggering the reorg and ensure we + // successfully reprocess all the stashed away blocks. + if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil { + t.Fatalf("failed to finalize competitor chain: %v", err) + } + for i, block := range competitor[:len(competitor)-triesInMemory] { + if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { + t.Fatalf("competitor %d: competing chain state missing", i) + } + } +} diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 7fb184aaa..158ed8324 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -203,6 +203,9 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE if header.ParentHash != prevHash { // Reorg to the common ancestor (might not exist in light sync mode, skip reorg then) // TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly? + + // TODO(karalabe): This operation is expensive and might block, causing the event system to + // potentially also lock up. We need to do with on a different thread somehow. if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { c.newHead(h.Number.Uint64(), true) } diff --git a/core/chain_makers.go b/core/chain_makers.go index 9bd3b2aee..31c9e3fb7 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -178,7 +178,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { // TODO(karalabe): This is needed for clique, which depends on multiple blocks. // It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow. - blockchain, _ := NewBlockChain(db, config, engine, vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}) defer blockchain.Stop() b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine} @@ -204,10 +204,13 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if b.engine != nil { block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts) // Write state changes to db - _, err := statedb.CommitTo(db, config.IsEIP158(b.header.Number)) + root, err := statedb.Commit(config.IsEIP158(b.header.Number)) if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } + if err := statedb.Database().TrieDB().Commit(root, false); err != nil { + panic(fmt.Sprintf("trie write error: %v", err)) + } return block, b.receipts } return nil, nil @@ -258,7 +261,7 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B db, _ := ethdb.NewMemDatabase() genesis := gspec.MustCommit(db) - blockchain, _ := NewBlockChain(db, params.AllEthashProtocolChanges, engine, vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}) // Create and inject the requested chain if n == 0 { return db, blockchain, nil diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index a3b80da29..93be43ddc 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -79,7 +79,7 @@ func ExampleGenerateChain() { }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}) + blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain); err != nil { diff --git a/core/dao_test.go b/core/dao_test.go index 43e2982a5..e0a3e3ff3 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -45,7 +45,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proConf.DAOForkBlock = forkBlock proConf.DAOForkSupport = true - proBc, _ := NewBlockChain(proDb, &proConf, ethash.NewFaker(), vm.Config{}) + proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() conDb, _ := ethdb.NewMemDatabase() @@ -55,7 +55,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { conConf.DAOForkBlock = forkBlock conConf.DAOForkSupport = false - conBc, _ := NewBlockChain(conDb, &conConf, ethash.NewFaker(), vm.Config{}) + conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -69,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -79,6 +79,9 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + t.Fatalf("failed to commit contra-fork head for expansion: %v", err) + } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err == nil { t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0]) @@ -91,7 +94,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a no-fork block, and try to feed into the pro-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -101,6 +104,9 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + t.Fatalf("failed to commit pro-fork head for expansion: %v", err) + } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err == nil { t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0]) @@ -114,7 +120,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -124,6 +130,9 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + t.Fatalf("failed to commit contra-fork head for expansion: %v", err) + } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) @@ -131,7 +140,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -141,6 +150,9 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + t.Fatalf("failed to commit pro-fork head for expansion: %v", err) + } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err) diff --git a/core/database_util.go b/core/database_util.go index c6b125dae..8c4698985 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -47,6 +47,7 @@ var ( headHeaderKey = []byte("LastHeader") headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") + trieSyncKey = []byte("TrieSync") // Data item prefixes (use single byte to avoid mixing data types, avoid `i`). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header @@ -70,8 +71,8 @@ var ( ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error - preimageCounter = metrics.NewCounter("db/preimage/total") - preimageHitCounter = metrics.NewCounter("db/preimage/hits") + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) + preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) // TxLookupEntry is a positional metadata to help looking up the data content of @@ -146,6 +147,16 @@ func GetHeadFastBlockHash(db DatabaseReader) common.Hash { return common.BytesToHash(data) } +// GetTrieSyncProgress retrieves the number of tries nodes fast synced to allow +// reportinc correct numbers across restarts. +func GetTrieSyncProgress(db DatabaseReader) uint64 { + data, _ := db.Get(trieSyncKey) + if len(data) == 0 { + return 0 + } + return new(big.Int).SetBytes(data).Uint64() +} + // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { @@ -374,6 +385,15 @@ func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { return nil } +// WriteTrieSyncProgress stores the fast sync trie process counter to support +// retrieving it across restarts. +func WriteTrieSyncProgress(db ethdb.Putter, count uint64) error { + if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil { + log.Crit("Failed to store fast sync trie progress", "err", err) + } + return nil +} + // WriteHeader serializes a block header into the database. func WriteHeader(db ethdb.Putter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) diff --git a/core/fees.go b/core/fees.go deleted file mode 100644 index 83275ea36..000000000 --- a/core/fees.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package core - -import ( - "math/big" -) - -var BlockReward = big.NewInt(5e+18) diff --git a/core/genesis.go b/core/genesis.go index e22985b80..b6ead2250 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -169,10 +169,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Check whether the genesis block is already written. if genesis != nil { - block, _ := genesis.ToBlock() - hash := block.Hash() + hash := genesis.ToBlock(nil).Hash() if hash != stored { - return genesis.Config, block.Hash(), &GenesisMismatchError{stored, hash} + return genesis.Config, hash, &GenesisMismatchError{stored, hash} } } @@ -220,9 +219,12 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { } } -// ToBlock creates the block and state of a genesis specification. -func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) { - db, _ := ethdb.NewMemDatabase() +// ToBlock creates the genesis block and writes state of a genesis specification +// to the given database (or discards it if nil). +func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { + if db == nil { + db, _ = ethdb.NewMemDatabase() + } statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) for addr, account := range g.Alloc { statedb.AddBalance(addr, account.Balance) @@ -252,19 +254,19 @@ func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) { if g.Difficulty == nil { head.Difficulty = params.GenesisDifficulty } - return types.NewBlock(head, nil, nil, nil), statedb + statedb.Commit(false) + statedb.Database().TrieDB().Commit(root, true) + + return types.NewBlock(head, nil, nil, nil) } // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { - block, statedb := g.ToBlock() + block := g.ToBlock(db) if block.Number().Sign() != 0 { return nil, fmt.Errorf("can't commit genesis block with number > 0") } - if _, err := statedb.CommitTo(db, false); err != nil { - return nil, fmt.Errorf("cannot write state: %v", err) - } if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil { return nil, err } diff --git a/core/genesis_test.go b/core/genesis_test.go index 2fe931b24..052ded699 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -30,11 +30,11 @@ import ( ) func TestDefaultGenesisBlock(t *testing.T) { - block, _ := DefaultGenesisBlock().ToBlock() + block := DefaultGenesisBlock().ToBlock(nil) if block.Hash() != params.MainnetGenesisHash { t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash) } - block, _ = DefaultTestnetGenesisBlock().ToBlock() + block = DefaultTestnetGenesisBlock().ToBlock(nil) if block.Hash() != params.TestnetGenesisHash { t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash) } @@ -118,10 +118,12 @@ func TestSetupGenesis(t *testing.T) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. genesis := oldcustomg.MustCommit(db) - bc, _ := NewBlockChain(db, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}) + + bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}) defer bc.Stop() - bc.SetValidator(bproc{}) - bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0)) + + blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) + bc.InsertChain(blocks) bc.CurrentBlock() // This should return a compatibility error. return SetupGenesisBlock(db, &customg) diff --git a/core/headerchain.go b/core/headerchain.go index 0e5215293..73cd5d2c4 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/hashicorp/golang-lru" + "sync/atomic" ) const ( @@ -51,8 +52,8 @@ type HeaderChain struct { chainDb ethdb.Database genesisHeader *types.Header - currentHeader *types.Header // Current head of the header chain (may be above the block chain!) - currentHeaderHash common.Hash // Hash of the current head of the header chain (prevent recomputing all the time) + currentHeader atomic.Value // Current head of the header chain (may be above the block chain!) + currentHeaderHash common.Hash // Hash of the current head of the header chain (prevent recomputing all the time) headerCache *lru.Cache // Cache for the most recent block headers tdCache *lru.Cache // Cache for the most recent block total difficulties @@ -95,13 +96,13 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c return nil, ErrNoGenesis } - hc.currentHeader = hc.genesisHeader + hc.currentHeader.Store(hc.genesisHeader) if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) { if chead := hc.GetHeaderByHash(head); chead != nil { - hc.currentHeader = chead + hc.currentHeader.Store(chead) } } - hc.currentHeaderHash = hc.currentHeader.Hash() + hc.currentHeaderHash = hc.CurrentHeader().Hash() return hc, nil } @@ -139,7 +140,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if ptd == nil { return NonStatTy, consensus.ErrUnknownAncestor } - localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64()) + localTd := hc.GetTd(hc.currentHeaderHash, hc.CurrentHeader().Number.Uint64()) externTd := new(big.Int).Add(header.Difficulty, ptd) // Irrelevant of the canonical status, write the td and header to the database @@ -181,7 +182,8 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil { log.Crit("Failed to insert head header hash", "err", err) } - hc.currentHeaderHash, hc.currentHeader = hash, types.CopyHeader(header) + hc.currentHeaderHash = hash + hc.currentHeader.Store(types.CopyHeader(header)) status = CanonStatTy } else { @@ -383,7 +385,7 @@ func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header { // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. func (hc *HeaderChain) CurrentHeader() *types.Header { - return hc.currentHeader + return hc.currentHeader.Load().(*types.Header) } // SetCurrentHeader sets the current head header of the canonical chain. @@ -391,7 +393,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil { log.Crit("Failed to insert head header hash", "err", err) } - hc.currentHeader = head + hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() } @@ -403,19 +405,20 @@ type DeleteCallback func(common.Hash, uint64) // will be deleted and the new one set. func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { height := uint64(0) - if hc.currentHeader != nil { - height = hc.currentHeader.Number.Uint64() + + if hdr := hc.CurrentHeader(); hdr != nil { + height = hdr.Number.Uint64() } - for hc.currentHeader != nil && hc.currentHeader.Number.Uint64() > head { - hash := hc.currentHeader.Hash() - num := hc.currentHeader.Number.Uint64() + for hdr := hc.CurrentHeader(); hdr != nil && hdr.Number.Uint64() > head; hdr = hc.CurrentHeader() { + hash := hdr.Hash() + num := hdr.Number.Uint64() if delFn != nil { delFn(hash, num) } DeleteHeader(hc.chainDb, hash, num) DeleteTd(hc.chainDb, hash, num) - hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash, hc.currentHeader.Number.Uint64()-1) + hc.currentHeader.Store(hc.GetHeader(hdr.ParentHash, hdr.Number.Uint64()-1)) } // Roll back the canonical chain numbering for i := height; i > head; i-- { @@ -426,10 +429,10 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { hc.tdCache.Purge() hc.numberCache.Purge() - if hc.currentHeader == nil { - hc.currentHeader = hc.genesisHeader + if hc.CurrentHeader() == nil { + hc.currentHeader.Store(hc.genesisHeader) } - hc.currentHeaderHash = hc.currentHeader.Hash() + hc.currentHeaderHash = hc.CurrentHeader().Hash() if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil { log.Crit("Failed to reset head header hash", "err", err) diff --git a/core/state/database.go b/core/state/database.go index 946625e76..36926ec69 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -40,16 +40,23 @@ const ( // Database wraps access to tries and contract code. type Database interface { - // Accessing tries: // OpenTrie opens the main account trie. - // OpenStorageTrie opens the storage trie of an account. OpenTrie(root common.Hash) (Trie, error) + + // OpenStorageTrie opens the storage trie of an account. OpenStorageTrie(addrHash, root common.Hash) (Trie, error) - // Accessing contract code: - ContractCode(addrHash, codeHash common.Hash) ([]byte, error) - ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie + + // ContractCode retrieves a particular contract's code. + ContractCode(addrHash, codeHash common.Hash) ([]byte, error) + + // ContractCodeSize retrieves a particular contracts code's size. + ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + + // TrieDB retrieves the low level trie database used for data storage. + TrieDB() *trie.Database } // Trie is a Ethereum Merkle Trie. @@ -57,26 +64,33 @@ type Trie interface { TryGet(key []byte) ([]byte, error) TryUpdate(key, value []byte) error TryDelete(key []byte) error - CommitTo(trie.DatabaseWriter) (common.Hash, error) + Commit(onleaf trie.LeafCallback) (common.Hash, error) Hash() common.Hash NodeIterator(startKey []byte) trie.NodeIterator GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed + Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error } // NewDatabase creates a backing store for state. The returned database is safe for -// concurrent use and retains cached trie nodes in memory. +// concurrent use and retains cached trie nodes in memory. The pool is an optional +// intermediate trie-node memory pool between the low level storage layer and the +// high level trie abstraction. func NewDatabase(db ethdb.Database) Database { csc, _ := lru.New(codeSizeCacheSize) - return &cachingDB{db: db, codeSizeCache: csc} + return &cachingDB{ + db: trie.NewDatabase(db), + codeSizeCache: csc, + } } type cachingDB struct { - db ethdb.Database + db *trie.Database mu sync.Mutex pastTries []*trie.SecureTrie codeSizeCache *lru.Cache } +// OpenTrie opens the main account trie. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { db.mu.Lock() defer db.mu.Unlock() @@ -105,10 +119,12 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) { } } +// OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { return trie.NewSecure(root, db.db, 0) } +// CopyTrie returns an independent copy of the given trie. func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case cachedTrie: @@ -120,14 +136,16 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { } } +// ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - code, err := db.db.Get(codeHash[:]) + code, err := db.db.Node(codeHash) if err == nil { db.codeSizeCache.Add(codeHash, len(code)) } return code, err } +// ContractCodeSize retrieves a particular contracts code's size. func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) { if cached, ok := db.codeSizeCache.Get(codeHash); ok { return cached.(int), nil @@ -139,16 +157,25 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro return len(code), err } +// TrieDB retrieves any intermediate trie-node caching layer. +func (db *cachingDB) TrieDB() *trie.Database { + return db.db +} + // cachedTrie inserts its trie into a cachingDB on commit. type cachedTrie struct { *trie.SecureTrie db *cachingDB } -func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) { - root, err := m.SecureTrie.CommitTo(dbw) +func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) { + root, err := m.SecureTrie.Commit(onleaf) if err == nil { m.db.pushTrie(m.SecureTrie) } return root, err } + +func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { + return m.SecureTrie.Prove(key, fromLevel, proofDb) +} diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index ff66ba7a9..9e46c851c 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -21,12 +21,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { // Create some arbitrary test state to iterate - db, mem, root, _ := makeTestState() + db, root, _ := makeTestState() state, err := New(root, db) if err != nil { @@ -39,14 +40,18 @@ func TestNodeIteratorCoverage(t *testing.T) { hashes[it.Hash] = struct{}{} } } - - // Cross check the hashes and the database itself + // Cross check the iterated hashes and the database/nodepool content for hash := range hashes { - if _, err := mem.Get(hash.Bytes()); err != nil { - t.Errorf("failed to retrieve reported node %x: %v", hash, err) + if _, err := db.TrieDB().Node(hash); err != nil { + t.Errorf("failed to retrieve reported node %x", hash) + } + } + for _, hash := range db.TrieDB().Nodes() { + if _, ok := hashes[hash]; !ok { + t.Errorf("state entry not reported %x", hash) } } - for _, key := range mem.Keys() { + for _, key := range db.TrieDB().DiskDB().(*ethdb.MemDatabase).Keys() { if bytes.HasPrefix(key, []byte("secure-key-")) { continue } diff --git a/core/state/state_object.go b/core/state/state_object.go index b2378c69c..b2112bfae 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -238,12 +237,12 @@ func (self *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to dwb. // This updates the trie root. -func (self *stateObject) CommitTrie(db Database, dbw trie.DatabaseWriter) error { +func (self *stateObject) CommitTrie(db Database) error { self.updateTrie(db) if self.dbErr != nil { return self.dbErr } - root, err := self.trie.CommitTo(dbw) + root, err := self.trie.Commit(nil) if err == nil { self.data.Root = root } diff --git a/core/state/state_test.go b/core/state/state_test.go index bbae3685b..6d42d63d8 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -48,7 +48,7 @@ func (s *StateSuite) TestDump(c *checker.C) { // write some of them to the trie s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) - s.state.CommitTo(s.db, false) + s.state.Commit(false) // check that dump contains the state objects that are in trie got := string(s.state.Dump()) @@ -97,7 +97,7 @@ func (s *StateSuite) TestNull(c *checker.C) { //value := common.FromHex("0x823140710bf13990e4500136726d8b55") var value common.Hash s.state.SetState(address, common.Hash{}, value) - s.state.CommitTo(s.db, false) + s.state.Commit(false) value = s.state.GetState(address, common.Hash{}) if !common.EmptyHash(value) { c.Errorf("expected empty hash. got %x", value) @@ -155,7 +155,7 @@ func TestSnapshot2(t *testing.T) { so0.deleted = false state.setStateObject(so0) - root, _ := state.CommitTo(db, false) + root, _ := state.Commit(false) state.Reset(root) // and one with deleted == true diff --git a/core/state/statedb.go b/core/state/statedb.go index 8e29104d5..776693e24 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -36,6 +36,14 @@ type revision struct { journalIndex int } +var ( + // emptyState is the known hash of an empty state trie entry. + emptyState = crypto.Keccak256Hash(nil) + + // emptyCode is the known hash of the empty EVM bytecode. + emptyCode = crypto.Keccak256Hash(nil) +) + // StateDBs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: @@ -235,6 +243,11 @@ func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash { return common.Hash{} } +// Database retrieves the low level database supporting the lower level trie ops. +func (self *StateDB) Database() Database { + return self.db +} + // StorageTrie returns the storage trie of an account. // The return value is a copy and is nil for non-existent accounts. func (self *StateDB) StorageTrie(a common.Address) Trie { @@ -568,8 +581,8 @@ func (s *StateDB) clearJournalAndRefund() { s.refund = 0 } -// CommitTo writes the state to the given database. -func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (root common.Hash, err error) { +// Commit writes the state to the underlying in-memory trie database. +func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) { defer s.clearJournalAndRefund() // Commit objects to the trie. @@ -583,13 +596,11 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro case isDirty: // Write any contract code associated with the state object if stateObject.code != nil && stateObject.dirtyCode { - if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil { - return common.Hash{}, err - } + s.db.TrieDB().Insert(common.BytesToHash(stateObject.CodeHash()), stateObject.code) stateObject.dirtyCode = false } // Write any storage changes in the state object to its storage trie. - if err := stateObject.CommitTrie(s.db, dbw); err != nil { + if err := stateObject.CommitTrie(s.db); err != nil { return common.Hash{}, err } // Update the object in the main account trie. @@ -598,7 +609,20 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro delete(s.stateObjectsDirty, addr) } // Write trie changes. - root, err = s.trie.CommitTo(dbw) + root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error { + var account Account + if err := rlp.DecodeBytes(leaf, &account); err != nil { + return nil + } + if account.Root != emptyState { + s.db.TrieDB().Reference(account.Root, parent) + } + code := common.BytesToHash(account.CodeHash) + if code != emptyCode { + s.db.TrieDB().Reference(code, parent) + } + return nil + }) log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads()) return root, err } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 5c80e3aa5..d9e3d9b79 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -97,10 +97,10 @@ func TestIntermediateLeaks(t *testing.T) { } // Commit and cross check the databases. - if _, err := transState.CommitTo(transDb, false); err != nil { + if _, err := transState.Commit(false); err != nil { t.Fatalf("failed to commit transition state: %v", err) } - if _, err := finalState.CommitTo(finalDb, false); err != nil { + if _, err := finalState.Commit(false); err != nil { t.Fatalf("failed to commit final state: %v", err) } for _, key := range finalDb.Keys() { @@ -122,8 +122,8 @@ func TestIntermediateLeaks(t *testing.T) { // https://github.com/ethereum/go-ethereum/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - mem, _ := ethdb.NewMemDatabase() - orig, _ := New(common.Hash{}, NewDatabase(mem)) + db, _ := ethdb.NewMemDatabase() + orig, _ := New(common.Hash{}, NewDatabase(db)) for i := byte(0); i < 255; i++ { obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) @@ -346,11 +346,10 @@ func (test *snapshotTest) run() bool { } action.fn(action, state) } - // Revert all snapshots in reverse order. Each revert must yield a state // that is equivalent to fresh state with all actions up the snapshot applied. for sindex--; sindex >= 0; sindex-- { - checkstate, _ := New(common.Hash{}, NewDatabase(db)) + checkstate, _ := New(common.Hash{}, state.Database()) for _, action := range test.actions[:test.snapshots[sindex]] { action.fn(action, checkstate) } @@ -409,7 +408,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { func (s *StateSuite) TestTouchDelete(c *check.C) { s.state.GetOrNewStateObject(common.Address{}) - root, _ := s.state.CommitTo(s.db, false) + root, _ := s.state.Commit(false) s.state.Reset(root) snapshot := s.state.Snapshot() @@ -417,7 +416,6 @@ func (s *StateSuite) TestTouchDelete(c *check.C) { if len(s.state.stateObjectsDirty) != 1 { c.Fatal("expected one dirty state object") } - s.state.RevertToSnapshot(snapshot) if len(s.state.stateObjectsDirty) != 0 { c.Fatal("expected no dirty state object") diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 06c572ea6..8f14a44e7 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -36,10 +36,10 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) { +func makeTestState() (Database, common.Hash, []*testAccount) { // Create an empty state - mem, _ := ethdb.NewMemDatabase() - db := NewDatabase(mem) + diskdb, _ := ethdb.NewMemDatabase() + db := NewDatabase(diskdb) state, _ := New(common.Hash{}, db) // Fill it with some arbitrary data @@ -61,10 +61,10 @@ func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) state.updateStateObject(obj) accounts = append(accounts, acc) } - root, _ := state.CommitTo(mem, false) + root, _ := state.Commit(false) // Return the generated state - return db, mem, root, accounts + return db, root, accounts } // checkStateAccounts cross references a reconstructed state with an expected @@ -96,7 +96,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error { if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(root, db) + trie, err := trie.New(root, trie.NewDatabase(db)) if err != nil { return err } @@ -138,7 +138,7 @@ func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, func testIterativeStateSync(t *testing.T, batch int) { // Create a random state to copy - _, srcMem, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -148,9 +148,9 @@ func testIterativeStateSync(t *testing.T, batch int) { for len(queue) > 0 { results := make([]trie.SyncResult, len(queue)) for i, hash := range queue { - data, err := srcMem.Get(hash.Bytes()) + data, err := srcDb.TrieDB().Node(hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x", hash) } results[i] = trie.SyncResult{Hash: hash, Data: data} } @@ -170,7 +170,7 @@ func testIterativeStateSync(t *testing.T, batch int) { // partial results are returned, and the others sent only later. func TestIterativeDelayedStateSync(t *testing.T) { // Create a random state to copy - _, srcMem, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -181,9 +181,9 @@ func TestIterativeDelayedStateSync(t *testing.T) { // Sync only half of the scheduled nodes results := make([]trie.SyncResult, len(queue)/2+1) for i, hash := range queue[:len(results)] { - data, err := srcMem.Get(hash.Bytes()) + data, err := srcDb.TrieDB().Node(hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x", hash) } results[i] = trie.SyncResult{Hash: hash, Data: data} } @@ -207,7 +207,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomS func testIterativeRandomStateSync(t *testing.T, batch int) { // Create a random state to copy - _, srcMem, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -221,9 +221,9 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { // Fetch all the queued nodes in a random order results := make([]trie.SyncResult, 0, len(queue)) for hash := range queue { - data, err := srcMem.Get(hash.Bytes()) + data, err := srcDb.TrieDB().Node(hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.SyncResult{Hash: hash, Data: data}) } @@ -247,7 +247,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { // partial results are returned (Even those randomly), others sent only later. func TestIterativeRandomDelayedStateSync(t *testing.T) { // Create a random state to copy - _, srcMem, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -263,9 +263,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { for hash := range queue { delete(queue, hash) - data, err := srcMem.Get(hash.Bytes()) + data, err := srcDb.TrieDB().Node(hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.SyncResult{Hash: hash, Data: data}) @@ -292,9 +292,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { // the database. func TestIncompleteStateSync(t *testing.T) { // Create a random state to copy - _, srcMem, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState() - checkTrieConsistency(srcMem, srcRoot) + checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -306,9 +306,9 @@ func TestIncompleteStateSync(t *testing.T) { // Fetch a batch of state nodes results := make([]trie.SyncResult, len(queue)) for i, hash := range queue { - data, err := srcMem.Get(hash.Bytes()) + data, err := srcDb.TrieDB().Node(hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x", hash) } results[i] = trie.SyncResult{Hash: hash, Data: data} } diff --git a/core/state_transition.go b/core/state_transition.go index 390473fff..b19bc12e4 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -215,6 +215,9 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo // Pay intrinsic gas gas, err := IntrinsicGas(st.data, contractCreation, homestead) + if err != nil { + return nil, 0, false, err + } if err = st.useGas(gas); err != nil { return nil, 0, false, err } diff --git a/core/tx_pool.go b/core/tx_pool.go index dc3ddc423..089bd215a 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -87,20 +87,20 @@ var ( var ( // Metrics for the pending pool - pendingDiscardCounter = metrics.NewCounter("txpool/pending/discard") - pendingReplaceCounter = metrics.NewCounter("txpool/pending/replace") - pendingRateLimitCounter = metrics.NewCounter("txpool/pending/ratelimit") // Dropped due to rate limiting - pendingNofundsCounter = metrics.NewCounter("txpool/pending/nofunds") // Dropped due to out-of-funds + pendingDiscardCounter = metrics.NewRegisteredCounter("txpool/pending/discard", nil) + pendingReplaceCounter = metrics.NewRegisteredCounter("txpool/pending/replace", nil) + pendingRateLimitCounter = metrics.NewRegisteredCounter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting + pendingNofundsCounter = metrics.NewRegisteredCounter("txpool/pending/nofunds", nil) // Dropped due to out-of-funds // Metrics for the queued pool - queuedDiscardCounter = metrics.NewCounter("txpool/queued/discard") - queuedReplaceCounter = metrics.NewCounter("txpool/queued/replace") - queuedRateLimitCounter = metrics.NewCounter("txpool/queued/ratelimit") // Dropped due to rate limiting - queuedNofundsCounter = metrics.NewCounter("txpool/queued/nofunds") // Dropped due to out-of-funds + queuedDiscardCounter = metrics.NewRegisteredCounter("txpool/queued/discard", nil) + queuedReplaceCounter = metrics.NewRegisteredCounter("txpool/queued/replace", nil) + queuedRateLimitCounter = metrics.NewRegisteredCounter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting + queuedNofundsCounter = metrics.NewRegisteredCounter("txpool/queued/nofunds", nil) // Dropped due to out-of-funds // General tx metrics - invalidTxCounter = metrics.NewCounter("txpool/invalid") - underpricedTxCounter = metrics.NewCounter("txpool/underpriced") + invalidTxCounter = metrics.NewRegisteredCounter("txpool/invalid", nil) + underpricedTxCounter = metrics.NewRegisteredCounter("txpool/underpriced", nil) ) // TxStatus is the current status of a transaction as seen by the pool. @@ -877,15 +877,14 @@ func (pool *TxPool) removeTx(hash common.Hash) { // Remove the transaction from the pending lists and reset the account nonce if pending := pool.pending[addr]; pending != nil { if removed, invalids := pending.Remove(tx); removed { - // If no more transactions are left, remove the list + // If no more pending transactions are left, remove the list if pending.Empty() { delete(pool.pending, addr) delete(pool.beats, addr) - } else { - // Otherwise postpone any invalidated transactions - for _, tx := range invalids { - pool.enqueueTx(tx.Hash(), tx) - } + } + // Postpone any invalidated transactions + for _, tx := range invalids { + pool.enqueueTx(tx.Hash(), tx) } // Update the account nonce if needed if nonce := tx.Nonce(); pool.pendingState.GetNonce(addr) > nonce { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index cd11f2ba2..1cf533aa6 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -78,8 +78,8 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec } func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + diskdb, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, state.NewDatabase(diskdb)) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} key, _ := crypto.GenerateKey() @@ -557,74 +557,112 @@ func TestTransactionDropping(t *testing.T) { func TestTransactionPostponing(t *testing.T) { t.Parallel() - // Create a test account and fund it - pool, key := setupTxPool() + // Create the pool to test the postponing with + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + + pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) - pool.currentState.AddBalance(account, big.NewInt(1000)) + // Create two test accounts to produce different gap profiles with + keys := make([]*ecdsa.PrivateKey, 2) + accs := make([]common.Address, len(keys)) + for i := 0; i < len(keys); i++ { + keys[i], _ = crypto.GenerateKey() + accs[i] = crypto.PubkeyToAddress(keys[i].PublicKey) + + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(50100)) + } // Add a batch consecutive pending transactions for validation - txns := []*types.Transaction{} - for i := 0; i < 100; i++ { - var tx *types.Transaction - if i%2 == 0 { - tx = transaction(uint64(i), 100, key) - } else { - tx = transaction(uint64(i), 500, key) + txs := []*types.Transaction{} + for i, key := range keys { + + for j := 0; j < 100; j++ { + var tx *types.Transaction + if (i+j)%2 == 0 { + tx = transaction(uint64(j), 25000, key) + } else { + tx = transaction(uint64(j), 50000, key) + } + txs = append(txs, tx) + } + } + for i, err := range pool.AddRemotes(txs) { + if err != nil { + t.Fatalf("tx %d: failed to add transactions: %v", i, err) } - pool.promoteTx(account, tx.Hash(), tx) - txns = append(txns, tx) } // Check that pre and post validations leave the pool as is - if pool.pending[account].Len() != len(txns) { - t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), len(txns)) + if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { + t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs)) } if len(pool.queue) != 0 { - t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 0) + t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } - if len(pool.all) != len(txns) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)) + if len(pool.all) != len(txs) { + t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)) } pool.lockedReset(nil, nil) - if pool.pending[account].Len() != len(txns) { - t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), len(txns)) + if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { + t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs)) } if len(pool.queue) != 0 { - t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 0) + t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } - if len(pool.all) != len(txns) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)) + if len(pool.all) != len(txs) { + t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)) } // Reduce the balance of the account, and check that transactions are reorganised - pool.currentState.AddBalance(account, big.NewInt(-750)) + for _, addr := range accs { + pool.currentState.AddBalance(addr, big.NewInt(-1)) + } pool.lockedReset(nil, nil) - if _, ok := pool.pending[account].txs.items[txns[0].Nonce()]; !ok { - t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txns[0]) + // The first account's first transaction remains valid, check that subsequent + // ones are either filtered out, or queued up for later. + if _, ok := pool.pending[accs[0]].txs.items[txs[0].Nonce()]; !ok { + t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txs[0]) } - if _, ok := pool.queue[account].txs.items[txns[0].Nonce()]; ok { - t.Errorf("tx %d: valid and funded transaction present in future queue: %v", 0, txns[0]) + if _, ok := pool.queue[accs[0]].txs.items[txs[0].Nonce()]; ok { + t.Errorf("tx %d: valid and funded transaction present in future queue: %v", 0, txs[0]) } - for i, tx := range txns[1:] { + for i, tx := range txs[1:100] { if i%2 == 1 { - if _, ok := pool.pending[account].txs.items[tx.Nonce()]; ok { + if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: valid but future transaction present in pending pool: %v", i+1, tx) } - if _, ok := pool.queue[account].txs.items[tx.Nonce()]; !ok { + if _, ok := pool.queue[accs[0]].txs.items[tx.Nonce()]; !ok { t.Errorf("tx %d: valid but future transaction missing from future queue: %v", i+1, tx) } } else { - if _, ok := pool.pending[account].txs.items[tx.Nonce()]; ok { + if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: out-of-fund transaction present in pending pool: %v", i+1, tx) } - if _, ok := pool.queue[account].txs.items[tx.Nonce()]; ok { + if _, ok := pool.queue[accs[0]].txs.items[tx.Nonce()]; ok { t.Errorf("tx %d: out-of-fund transaction present in future queue: %v", i+1, tx) } } } - if len(pool.all) != len(txns)/2 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)/2) + // The second account's first transaction got invalid, check that all transactions + // are either filtered out, or queued up for later. + if pool.pending[accs[1]] != nil { + t.Errorf("invalidated account still has pending transactions") + } + for i, tx := range txs[100:] { + if i%2 == 1 { + if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; !ok { + t.Errorf("tx %d: valid but future transaction missing from future queue: %v", 100+i, tx) + } + } else { + if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; ok { + t.Errorf("tx %d: out-of-fund transaction present in future queue: %v", 100+i, tx) + } + } + } + if len(pool.all) != len(txs)/2 { + t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)/2) } } @@ -949,11 +987,11 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { account2, _ := deriveSender(transaction(0, 0, key2)) pool2.currentState.AddBalance(account2, big.NewInt(1000000)) - txns := []*types.Transaction{} + txs := []*types.Transaction{} for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { - txns = append(txns, transaction(origin+i, 100000, key2)) + txs = append(txs, transaction(origin+i, 100000, key2)) } - pool2.AddRemotes(txns) + pool2.AddRemotes(txs) // Ensure the batch optimization honors the same pool mechanics if len(pool1.pending) != len(pool2.pending) { @@ -1124,7 +1162,7 @@ func TestTransactionPoolRepricing(t *testing.T) { defer sub.Unsubscribe() // Create a number of test accounts and fund them - keys := make([]*ecdsa.PrivateKey, 3) + keys := make([]*ecdsa.PrivateKey, 4) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) @@ -1136,24 +1174,28 @@ func TestTransactionPoolRepricing(t *testing.T) { txs = append(txs, pricedTransaction(1, 100000, big.NewInt(1), keys[0])) txs = append(txs, pricedTransaction(2, 100000, big.NewInt(2), keys[0])) + txs = append(txs, pricedTransaction(0, 100000, big.NewInt(1), keys[1])) txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[1])) - txs = append(txs, pricedTransaction(2, 100000, big.NewInt(1), keys[1])) - txs = append(txs, pricedTransaction(3, 100000, big.NewInt(2), keys[1])) + txs = append(txs, pricedTransaction(2, 100000, big.NewInt(2), keys[1])) - ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[2]) + txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[2])) + txs = append(txs, pricedTransaction(2, 100000, big.NewInt(1), keys[2])) + txs = append(txs, pricedTransaction(3, 100000, big.NewInt(2), keys[2])) + + ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[3]) // Import the batch and that both pending and queued transactions match up pool.AddRemotes(txs) pool.AddLocal(ltx) pending, queued := pool.Stats() - if pending != 4 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 4) + if pending != 7 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 7) } if queued != 3 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) } - if err := validateEvents(events, 4); err != nil { + if err := validateEvents(events, 7); err != nil { t.Fatalf("original event firing failed: %v", err) } if err := validateTxPoolInternals(pool); err != nil { @@ -1166,8 +1208,8 @@ func TestTransactionPoolRepricing(t *testing.T) { if pending != 2 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) } - if queued != 3 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) + if queued != 5 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 5) } if err := validateEvents(events, 0); err != nil { t.Fatalf("reprice event firing failed: %v", err) @@ -1179,7 +1221,10 @@ func TestTransactionPoolRepricing(t *testing.T) { if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); err != ErrUnderpriced { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[1])); err != ErrUnderpriced { + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); err != ErrUnderpriced { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + } + if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); err != ErrUnderpriced { t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } if err := validateEvents(events, 0); err != nil { @@ -1189,7 +1234,7 @@ func TestTransactionPoolRepricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // However we can add local underpriced transactions - tx := pricedTransaction(1, 100000, big.NewInt(1), keys[2]) + tx := pricedTransaction(1, 100000, big.NewInt(1), keys[3]) if err := pool.AddLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } @@ -1202,6 +1247,22 @@ func TestTransactionPoolRepricing(t *testing.T) { if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // And we can fill gaps with properly priced transactions + if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[0])); err != nil { + t.Fatalf("failed to add pending transaction: %v", err) + } + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(2), keys[1])); err != nil { + t.Fatalf("failed to add pending transaction: %v", err) + } + if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil { + t.Fatalf("failed to add queued transaction: %v", err) + } + if err := validateEvents(events, 5); err != nil { + t.Fatalf("post-reprice event firing failed: %v", err) + } + if err := validateTxPoolInternals(pool); err != nil { + t.Fatalf("pool internal state corrupted: %v", err) + } } // Tests that setting the transaction pool gas price to a higher value does not diff --git a/core/types/block.go b/core/types/block.go index ffe317342..92b868d9d 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -25,6 +25,7 @@ import ( "sort" "sync/atomic" "time" + "unsafe" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -121,6 +122,12 @@ func (h *Header) HashNoNonce() common.Hash { }) } +// Size returns the approximate memory used by all internal contents. It is used +// to approximate and limit the memory consumption of various caches. +func (h *Header) Size() common.StorageSize { + return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8) +} + func rlpHash(x interface{}) (h common.Hash) { hw := sha3.NewKeccak256() rlp.Encode(hw, x) @@ -322,6 +329,8 @@ func (b *Block) HashNoNonce() common.Hash { return b.header.HashNoNonce() } +// Size returns the true RLP encoded storage size of the block, either by encoding +// and returning it, or returning a previsouly cached value. func (b *Block) Size() common.StorageSize { if size := b.size.Load(); size != nil { return size.(common.StorageSize) diff --git a/core/types/receipt.go b/core/types/receipt.go index 208d54aaa..f945f6f6a 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "io" + "unsafe" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -136,6 +137,18 @@ func (r *Receipt) statusEncoding() []byte { return r.PostState } +// Size returns the approximate memory used by all internal contents. It is used +// to approximate and limit the memory consumption of various caches. +func (r *Receipt) Size() common.StorageSize { + size := common.StorageSize(unsafe.Sizeof(*r)) + common.StorageSize(len(r.PostState)) + + size += common.StorageSize(len(r.Logs)) * common.StorageSize(unsafe.Sizeof(Log{})) + for _, log := range r.Logs { + size += common.StorageSize(len(log.Topics)*common.HashLength + len(log.Data)) + } + return size +} + // String implements the Stringer interface. func (r *Receipt) String() string { if len(r.PostState) == 0 { diff --git a/core/types/transaction.go b/core/types/transaction.go index a7ed211e4..5660582ba 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -206,6 +206,8 @@ func (tx *Transaction) Hash() common.Hash { return v } +// Size returns the true RLP encoded storage size of the transaction, either by +// encoding and returning it, or returning a previsouly cached value. func (tx *Transaction) Size() common.StorageSize { if size := tx.size.Load(); size != nil { return size.(common.StorageSize) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 7344b6043..237450ea9 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -251,26 +251,12 @@ func (c *bigModExp) Run(input []byte) ([]byte, error) { return common.LeftPadBytes(base.Exp(base, exp, mod).Bytes(), int(modLen)), nil } -var ( - // errNotOnCurve is returned if a point being unmarshalled as a bn256 elliptic - // curve point is not on the curve. - errNotOnCurve = errors.New("point not on elliptic curve") - - // errInvalidCurvePoint is returned if a point being unmarshalled as a bn256 - // elliptic curve point is invalid. - errInvalidCurvePoint = errors.New("invalid elliptic curve point") -) - // newCurvePoint unmarshals a binary blob into a bn256 elliptic curve point, // returning it, or an error if the point is invalid. func newCurvePoint(blob []byte) (*bn256.G1, error) { - p, onCurve := new(bn256.G1).Unmarshal(blob) - if !onCurve { - return nil, errNotOnCurve - } - gx, gy, _, _ := p.CurvePoints() - if gx.Cmp(bn256.P) >= 0 || gy.Cmp(bn256.P) >= 0 { - return nil, errInvalidCurvePoint + p := new(bn256.G1) + if _, err := p.Unmarshal(blob); err != nil { + return nil, err } return p, nil } @@ -278,14 +264,9 @@ func newCurvePoint(blob []byte) (*bn256.G1, error) { // newTwistPoint unmarshals a binary blob into a bn256 elliptic curve point, // returning it, or an error if the point is invalid. func newTwistPoint(blob []byte) (*bn256.G2, error) { - p, onCurve := new(bn256.G2).Unmarshal(blob) - if !onCurve { - return nil, errNotOnCurve - } - x2, y2, _, _ := p.CurvePoints() - if x2.Real().Cmp(bn256.P) >= 0 || x2.Imag().Cmp(bn256.P) >= 0 || - y2.Real().Cmp(bn256.P) >= 0 || y2.Imag().Cmp(bn256.P) >= 0 { - return nil, errInvalidCurvePoint + p := new(bn256.G2) + if _, err := p.Unmarshal(blob); err != nil { + return nil, err } return p, nil } diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 513651835..96083337c 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -1,3 +1,19 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + package vm import ( diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 766172501..66e804fb7 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -30,6 +30,8 @@ import ( var ( bigZero = new(big.Int) + tt255 = math.BigPow(2, 255) + tt256 = math.BigPow(2, 256) errWriteProtection = errors.New("evm: write protection") errReturnDataOutOfBounds = errors.New("evm: return data out of bounds") errExecutionReverted = errors.New("evm: execution reverted") @@ -191,50 +193,71 @@ func opGt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack } func opSlt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := math.S256(stack.pop()), math.S256(stack.pop()) - if x.Cmp(math.S256(y)) < 0 { - stack.push(evm.interpreter.intPool.get().SetUint64(1)) - } else { - stack.push(new(big.Int)) - } + x, y := stack.pop(), stack.peek() - evm.interpreter.intPool.put(x, y) + xSign := x.Cmp(tt255) + ySign := y.Cmp(tt255) + + switch { + case xSign >= 0 && ySign < 0: + y.SetUint64(1) + + case xSign < 0 && ySign >= 0: + y.SetUint64(0) + + default: + if x.Cmp(y) < 0 { + y.SetUint64(1) + } else { + y.SetUint64(0) + } + } + evm.interpreter.intPool.put(x) return nil, nil } func opSgt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := math.S256(stack.pop()), math.S256(stack.pop()) - if x.Cmp(y) > 0 { - stack.push(evm.interpreter.intPool.get().SetUint64(1)) - } else { - stack.push(new(big.Int)) - } + x, y := stack.pop(), stack.peek() - evm.interpreter.intPool.put(x, y) + xSign := x.Cmp(tt255) + ySign := y.Cmp(tt255) + + switch { + case xSign >= 0 && ySign < 0: + y.SetUint64(0) + + case xSign < 0 && ySign >= 0: + y.SetUint64(1) + + default: + if x.Cmp(y) > 0 { + y.SetUint64(1) + } else { + y.SetUint64(0) + } + } + evm.interpreter.intPool.put(x) return nil, nil } func opEq(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() + x, y := stack.pop(), stack.peek() if x.Cmp(y) == 0 { - stack.push(evm.interpreter.intPool.get().SetUint64(1)) + y.SetUint64(1) } else { - stack.push(new(big.Int)) + y.SetUint64(0) } - - evm.interpreter.intPool.put(x, y) + evm.interpreter.intPool.put(x) return nil, nil } func opIszero(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x := stack.pop() + x := stack.peek() if x.Sign() > 0 { - stack.push(new(big.Int)) + x.SetUint64(0) } else { - stack.push(evm.interpreter.intPool.get().SetUint64(1)) + x.SetUint64(1) } - - evm.interpreter.intPool.put(x) return nil, nil } @@ -302,6 +325,66 @@ func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S return nil, nil } +// opSHL implements Shift Left +// The SHL instruction (shift left) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the left by arg1 number of bits. +func opSHL(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards + shift, value := math.U256(stack.pop()), math.U256(stack.peek()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + value.SetUint64(0) + return nil, nil + } + n := uint(shift.Uint64()) + math.U256(value.Lsh(value, n)) + + return nil, nil +} + +// opSHR implements Logical Shift Right +// The SHR instruction (logical shift right) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the right by arg1 number of bits with zero fill. +func opSHR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards + shift, value := math.U256(stack.pop()), math.U256(stack.peek()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + value.SetUint64(0) + return nil, nil + } + n := uint(shift.Uint64()) + math.U256(value.Rsh(value, n)) + + return nil, nil +} + +// opSAR implements Arithmetic Shift Right +// The SAR instruction (arithmetic shift right) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the right by arg1 number of bits with sign extension. +func opSAR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, S256 returns (potentially) a new bigint, so we're popping, not peeking this one + shift, value := math.U256(stack.pop()), math.S256(stack.pop()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + if value.Sign() > 0 { + value.SetUint64(0) + } else { + value.SetInt64(-1) + } + stack.push(math.U256(value)) + return nil, nil + } + n := uint(shift.Uint64()) + value.Rsh(value, n) + stack.push(math.U256(value)) + + return nil, nil +} + func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() data := memory.Get(offset.Int64(), size.Int64()) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 18644989c..134363bb7 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -1,3 +1,19 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + package vm import ( @@ -8,6 +24,48 @@ import ( "github.com/ethereum/go-ethereum/params" ) +type twoOperandTest struct { + x string + y string + expected string +} + +func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + stack = newstack() + pc = uint64(0) + ) + for i, test := range tests { + x := new(big.Int).SetBytes(common.Hex2Bytes(test.x)) + shift := new(big.Int).SetBytes(common.Hex2Bytes(test.y)) + expected := new(big.Int).SetBytes(common.Hex2Bytes(test.expected)) + stack.push(x) + stack.push(shift) + opFn(&pc, env, nil, nil, stack) + actual := stack.pop() + if actual.Cmp(expected) != 0 { + t.Errorf("Testcase %d, expected %v, got %v", i, expected, actual) + } + // Check pool usage + // 1.pool is not allowed to contain anything on the stack + // 2.pool is not allowed to contain the same pointers twice + if env.interpreter.intPool.pool.len() > 0 { + + poolvals := make(map[*big.Int]struct{}) + poolvals[actual] = struct{}{} + + for env.interpreter.intPool.pool.len() > 0 { + key := env.interpreter.intPool.get() + if _, exist := poolvals[key]; exist { + t.Errorf("Testcase %d, pool contains double-entry", i) + } + poolvals[key] = struct{}{} + } + } + } +} + func TestByteOp(t *testing.T) { var ( env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) @@ -41,6 +99,103 @@ func TestByteOp(t *testing.T) { } } +func TestSHL(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shl-shift-left + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000002"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ff", "8000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "0101", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "8000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"}, + } + testTwoOperandOp(t, tests, opSHL) +} + +func TestSHR(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shr-logical-shift-right + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "01", "4000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "ff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0101", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + testTwoOperandOp(t, tests, opSHR) +} + +func TestSAR(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#sar-arithmetic-shift-right + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "01", "c000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0101", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"4000000000000000000000000000000000000000000000000000000000000000", "fe", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "f8", "000000000000000000000000000000000000000000000000000000000000007f"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "fe", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + + testTwoOperandOp(t, tests, opSAR) +} + +func TestSGT(t *testing.T) { + tests := []twoOperandTest{ + + {"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + testTwoOperandOp(t, tests, opSgt) +} + +func TestSLT(t *testing.T) { + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb", "0000000000000000000000000000000000000000000000000000000000000001"}, + } + testTwoOperandOp(t, tests, opSlt) +} + func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) { var ( env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) @@ -199,7 +354,11 @@ func BenchmarkOpEq(b *testing.B) { opBenchmark(b, opEq, x, y) } - +func BenchmarkOpEq2(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201fffffffe" + opBenchmark(b, opEq, x, y) +} func BenchmarkOpAnd(b *testing.B) { x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" @@ -243,3 +402,26 @@ func BenchmarkOpMulmod(b *testing.B) { opBenchmark(b, opMulmod, x, y, z) } + +func BenchmarkOpSHL(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSHL, x, y) +} +func BenchmarkOpSHR(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSHR, x, y) +} +func BenchmarkOpSAR(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSAR, x, y) +} +func BenchmarkOpIsZero(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + opBenchmark(b, opIszero, x) +} diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 482e67a3a..95490adfc 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -20,9 +20,7 @@ import ( "fmt" "sync/atomic" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" ) @@ -39,8 +37,6 @@ type Config struct { // NoRecursion disabled Interpreter call, callcode, // delegate call and create. NoRecursion bool - // Disable gas metering - DisableGasMetering bool // Enable recording of SHA3/keccak preimages EnablePreimageRecording bool // JumpTable contains the EVM instruction table. This @@ -70,6 +66,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { switch { + case evm.ChainConfig().IsConstantinople(evm.BlockNumber): + cfg.JumpTable = constantinopleInstructionSet case evm.ChainConfig().IsByzantium(evm.BlockNumber): cfg.JumpTable = byzantiumInstructionSet case evm.ChainConfig().IsHomestead(evm.BlockNumber): @@ -123,11 +121,6 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er return nil, nil } - codehash := contract.CodeHash // codehash is used when doing jump dest caching - if codehash == (common.Hash{}) { - codehash = crypto.Keccak256Hash(contract.Code) - } - var ( op OpCode // current opcode mem = NewMemory() // bound memory @@ -194,14 +187,11 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er return nil, errGasUintOverflow } } - - if !in.cfg.DisableGasMetering { - // consume the gas and return an error if not enough gas is available. - // cost is explicitly set so that the capture state defer method cas get the proper cost - cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize) - if err != nil || !contract.UseGas(cost) { - return nil, ErrOutOfGas - } + // consume the gas and return an error if not enough gas is available. + // cost is explicitly set so that the capture state defer method cas get the proper cost + cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize) + if err != nil || !contract.UseGas(cost) { + return nil, ErrOutOfGas } if memorySize > 0 { mem.Resize(memorySize) diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index a1c5ad9c6..338994135 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -51,11 +51,38 @@ type operation struct { } var ( - frontierInstructionSet = NewFrontierInstructionSet() - homesteadInstructionSet = NewHomesteadInstructionSet() - byzantiumInstructionSet = NewByzantiumInstructionSet() + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() + byzantiumInstructionSet = NewByzantiumInstructionSet() + constantinopleInstructionSet = NewConstantinopleInstructionSet() ) +// NewConstantinopleInstructionSet returns the frontier, homestead +// byzantium and contantinople instructions. +func NewConstantinopleInstructionSet() [256]operation { + // instructions that can be executed during the byzantium phase. + instructionSet := NewByzantiumInstructionSet() + instructionSet[SHL] = operation{ + execute: opSHL, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + instructionSet[SHR] = operation{ + execute: opSHR, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + instructionSet[SAR] = operation{ + execute: opSAR, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + return instructionSet +} + // NewByzantiumInstructionSet returns the frontier, homestead and // byzantium instructions. func NewByzantiumInstructionSet() [256]operation { diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 0c6550735..7fe55b72f 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -63,6 +63,9 @@ const ( XOR NOT BYTE + SHL + SHR + SAR SHA3 = 0x20 ) @@ -234,6 +237,9 @@ var opCodeToString = map[OpCode]string{ OR: "OR", XOR: "XOR", BYTE: "BYTE", + SHL: "SHL", + SHR: "SHR", + SAR: "SAR", ADDMOD: "ADDMOD", MULMOD: "MULMOD", @@ -400,6 +406,9 @@ var stringToOp = map[string]OpCode{ "OR": OR, "XOR": XOR, "BYTE": BYTE, + "SHL": SHL, + "SHR": SHR, + "SAR": SAR, "ADDMOD": ADDMOD, "MULMOD": MULMOD, "SHA3": SHA3, |