diff options
Diffstat (limited to 'eth/downloader')
| -rw-r--r-- | eth/downloader/downloader_test.go | 814 | ||||
| -rw-r--r-- | eth/downloader/testchain_test.go | 221 |
2 files changed, 491 insertions, 544 deletions
diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index dad626e89..1fe02d884 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -25,22 +25,14 @@ import ( "testing" "time" + ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) -) - // Reduce some of the parameters to make the tester faster. func init() { MaxForkAncestry = uint64(10000) @@ -55,6 +47,7 @@ type downloadTester struct { genesis *types.Block // Genesis blocks used by the tester and peers stateDb ethdb.Database // Database used by the tester for syncing from peers peerDb ethdb.Database // Database of the peers containing all data + peers map[string]*downloadTesterPeer ownHashes []common.Hash // Hash chain belonging to the tester ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester @@ -62,129 +55,27 @@ type downloadTester struct { ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain - peerHashes map[string][]common.Hash // Hash chain belonging to different test peers - peerHeaders map[string]map[common.Hash]*types.Header // Headers belonging to different test peers - peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers - peerReceipts map[string]map[common.Hash]types.Receipts // Receipts belonging to different test peers - peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains - - peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return - lock sync.RWMutex } // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb := ethdb.NewMemDatabase() - genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) - tester := &downloadTester{ - genesis: genesis, - peerDb: testdb, - ownHashes: []common.Hash{genesis.Hash()}, - ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil}, - ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, - peerHashes: make(map[string][]common.Hash), - peerHeaders: make(map[string]map[common.Hash]*types.Header), - peerBlocks: make(map[string]map[common.Hash]*types.Block), - peerReceipts: make(map[string]map[common.Hash]types.Receipts), - peerChainTds: make(map[string]map[common.Hash]*big.Int), - peerMissingStates: make(map[string]map[common.Hash]bool), + genesis: testGenesis, + peerDb: testDB, + peers: make(map[string]*downloadTesterPeer), + ownHashes: []common.Hash{testGenesis.Hash()}, + ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, + ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, + ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, + ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, } tester.stateDb = ethdb.NewMemDatabase() - tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) - + tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) - return tester } -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts, heavy bool) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) { - // Generate the block chain - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), dl.peerDb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If a heavy chain is requested, delay blocks to raise difficulty - if heavy { - block.OffsetTime(-1) - } - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == dl.genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ - ParentHash: block.PrevBlock(i - 1).Hash(), - Number: big.NewInt(block.Number().Int64() - 1), - }) - } - }) - // Convert the block-chain into a hash-chain and header/block maps - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - - headerm := make(map[common.Hash]*types.Header, n+1) - headerm[parent.Hash()] = parent.Header() - - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - - receiptm := make(map[common.Hash]types.Receipts, n+1) - receiptm[parent.Hash()] = parentReceipts - - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - headerm[b.Hash()] = b.Header() - blockm[b.Hash()] = b - receiptm[b.Hash()] = receipts[i] - } - return hashes, headerm, blockm, receiptm -} - -// makeChainFork creates two chains of length n, such that h1[:f] and -// h2[:f] are different but have a common suffix of length n-f. -func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentReceipts types.Receipts, balanced bool) ([]common.Hash, []common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]*types.Block, map[common.Hash]types.Receipts, map[common.Hash]types.Receipts) { - // Create the common suffix - hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false) - - // Create the forks, making the second heavier if non balanced forks were requested - hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false) - hashes1 = append(hashes1, hashes[1:]...) - - heavy := false - if !balanced { - heavy = true - } - hashes2, headers2, blocks2, receipts2 := dl.makeChain(f, 2, blocks[hashes[0]], receipts[hashes[0]], heavy) - hashes2 = append(hashes2, hashes[1:]...) - - for hash, header := range headers { - headers1[hash] = header - headers2[hash] = header - } - for hash, block := range blocks { - blocks1[hash] = block - blocks2[hash] = block - } - for hash, receipt := range receipts { - receipts1[hash] = receipt - receipts2[hash] = receipt - } - return hashes1, hashes2, headers1, headers2, blocks1, blocks2, receipts1, receipts2 -} - // terminate aborts any operations on the embedded downloader and releases all // held resources. func (dl *downloadTester) terminate() { @@ -194,13 +85,10 @@ func (dl *downloadTester) terminate() { // sync starts synchronizing with a remote peer, blocking until it completes. func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { dl.lock.RLock() - hash := dl.peerHashes[id][0] + hash := dl.peers[id].chain.headBlock().Hash() // If no particular TD was requested, load from the peer's blockchain if td == nil { - td = big.NewInt(1) - if diff, ok := dl.peerChainTds[id][hash]; ok { - td = diff - } + td = dl.peers[id].chain.td(hash) } dl.lock.RUnlock() @@ -302,7 +190,7 @@ func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int { } // InsertHeaderChain injects a new batch of headers into the simulated chain. -func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (int, error) { +func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -331,7 +219,7 @@ func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq i } // InsertChain injects a new batch of blocks into the simulated chain. -func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { +func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -353,7 +241,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { } // InsertReceiptChain injects a new batch of receipts into the simulated chain. -func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (int, error) { +func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -387,60 +275,13 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error { - return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0) -} - -// newSlowPeer registers a new block download source into the downloader, with a -// specific delay time on processing the network packets sent to it, simulating -// potentially slow network IO. -func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts, delay time.Duration) error { +func (dl *downloadTester) newPeer(id string, version int, chain *testChain) error { dl.lock.Lock() defer dl.lock.Unlock() - var err = dl.downloader.RegisterPeer(id, version, &downloadTesterPeer{dl: dl, id: id, delay: delay}) - if err == nil { - // Assign the owned hashes, headers and blocks to the peer (deep copy) - dl.peerHashes[id] = make([]common.Hash, len(hashes)) - copy(dl.peerHashes[id], hashes) - - dl.peerHeaders[id] = make(map[common.Hash]*types.Header) - dl.peerBlocks[id] = make(map[common.Hash]*types.Block) - dl.peerReceipts[id] = make(map[common.Hash]types.Receipts) - dl.peerChainTds[id] = make(map[common.Hash]*big.Int) - dl.peerMissingStates[id] = make(map[common.Hash]bool) - - genesis := hashes[len(hashes)-1] - if header := headers[genesis]; header != nil { - dl.peerHeaders[id][genesis] = header - dl.peerChainTds[id][genesis] = header.Difficulty - } - if block := blocks[genesis]; block != nil { - dl.peerBlocks[id][genesis] = block - dl.peerChainTds[id][genesis] = block.Difficulty() - } - - for i := len(hashes) - 2; i >= 0; i-- { - hash := hashes[i] - - if header, ok := headers[hash]; ok { - dl.peerHeaders[id][hash] = header - if _, ok := dl.peerHeaders[id][header.ParentHash]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(header.Difficulty, dl.peerChainTds[id][header.ParentHash]) - } - } - if block, ok := blocks[hash]; ok { - dl.peerBlocks[id][hash] = block - if _, ok := dl.peerBlocks[id][block.ParentHash()]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][block.ParentHash()]) - } - } - if receipt, ok := receipts[hash]; ok { - dl.peerReceipts[id][hash] = receipt - } - } - } - return err + peer := &downloadTesterPeer{dl: dl, id: id, chain: chain} + dl.peers[id] = peer + return dl.downloader.RegisterPeer(id, version, peer) } // dropPeer simulates a hard peer removal from the connection pool. @@ -448,89 +289,48 @@ func (dl *downloadTester) dropPeer(id string) { dl.lock.Lock() defer dl.lock.Unlock() - delete(dl.peerHashes, id) - delete(dl.peerHeaders, id) - delete(dl.peerBlocks, id) - delete(dl.peerChainTds, id) - + delete(dl.peers, id) dl.downloader.UnregisterPeer(id) } type downloadTesterPeer struct { - dl *downloadTester - id string - delay time.Duration - lock sync.RWMutex -} - -// setDelay is a thread safe setter for the network delay value. -func (dlp *downloadTesterPeer) setDelay(delay time.Duration) { - dlp.lock.Lock() - defer dlp.lock.Unlock() - - dlp.delay = delay -} - -// waitDelay is a thread safe way to sleep for the configured time. -func (dlp *downloadTesterPeer) waitDelay() { - dlp.lock.RLock() - delay := dlp.delay - dlp.lock.RUnlock() - - time.Sleep(delay) + dl *downloadTester + id string + lock sync.RWMutex + chain *testChain + missingStates map[common.Hash]bool // State entries that fast sync should not return } // Head constructs a function to retrieve a peer's current head hash // and total difficulty. func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - return dlp.dl.peerHashes[dlp.id][0], nil + b := dlp.chain.headBlock() + return b.Hash(), dlp.chain.td(b.Hash()) } // RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - // Find the canonical number of the hash - dlp.dl.lock.RLock() - number := uint64(0) - for num, hash := range dlp.dl.peerHashes[dlp.id] { - if hash == origin { - number = uint64(len(dlp.dl.peerHashes[dlp.id]) - num - 1) - break - } + if reverse { + panic("reverse header requests not supported") } - dlp.dl.lock.RUnlock() - // Use the absolute header fetcher to satisfy the query - return dlp.RequestHeadersByNumber(number, amount, skip, reverse) + result := dlp.chain.headersByHash(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) + return nil } // RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - // Gather the next batch of headers - hashes := dlp.dl.peerHashes[dlp.id] - headers := dlp.dl.peerHeaders[dlp.id] - result := make([]*types.Header, 0, amount) - for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ { - if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok { - result = append(result, header) - } + if reverse { + panic("reverse header requests not supported") } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dlp.dl.downloader.DeliverHeaders(dlp.id, result) - }() + + result := dlp.chain.headersByNumber(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) return nil } @@ -538,24 +338,8 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, // peer in the download tester. The returned function can be used to retrieve // batches of block bodies from the particularly requested peer. func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - blocks := dlp.dl.peerBlocks[dlp.id] - - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - go dlp.dl.downloader.DeliverBodies(dlp.id, transactions, uncles) - + txs, uncles := dlp.chain.bodies(hashes) + go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles) return nil } @@ -563,21 +347,8 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - receipts := dlp.dl.peerReceipts[dlp.id] - - results := make([][]*types.Receipt, 0, len(hashes)) - for _, hash := range hashes { - if receipt, ok := receipts[hash]; ok { - results = append(results, receipt) - } - } - go dlp.dl.downloader.DeliverReceipts(dlp.id, results) - + receipts := dlp.chain.receipts(hashes) + go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts) return nil } @@ -585,21 +356,18 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of node state data from the particularly requested peer. func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { - dlp.waitDelay() - dlp.dl.lock.RLock() defer dlp.dl.lock.RUnlock() results := make([][]byte, 0, len(hashes)) for _, hash := range hashes { if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { - if !dlp.dl.peerMissingStates[dlp.id][hash] { + if !dlp.missingStates[hash] { results = append(results, data) } } } go dlp.dl.downloader.DeliverNodeData(dlp.id, results) - return nil } @@ -639,21 +407,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng if rs := len(tester.ownReceipts); rs != receipts { t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts) } - // Verify the state trie too for fast syncs - /*if tester.downloader.mode == FastSync { - pivot := uint64(0) - var index int - if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common { - index = pivot - } else { - index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot) - } - if index > 0 { - if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil { - t.Fatalf("state reconstruction failed: %v", err) - } - } - }*/ } // Tests that simple synchronization against a canonical chain works correctly. @@ -673,16 +426,14 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(blockCacheItems - 15) + tester.newPeer("peer", protocol, chain) // Synchronise with the peer and make sure all relevant data was retrieved if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if a large batch of blocks are being downloaded, it is throttled @@ -699,10 +450,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long block chain to download and the tester - targetBlocks := 8 * blockCacheItems - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + targetBlocks := testChainBase.len() - 1 + tester.newPeer("peer", protocol, testChainBase) // Wrap the importer to allow stepping blocked, proceed := uint32(0), make(chan struct{}) @@ -734,9 +483,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { cached = len(tester.downloader.queue.blockDonePool) if mode == FastSync { if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached { - //if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot { cached = receipts - //} } } frozen = int(atomic.LoadUint32(&blocked)) @@ -786,24 +533,22 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkLightB.shorten(testChainBase.len() + 80) + tester.newPeer("fork A", protocol, chainA) + tester.newPeer("fork B", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("fork A", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("fork B", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that synchronising against a much shorter but much heavyer fork works @@ -821,24 +566,22 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 4*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkHeavy.shorten(testChainBase.len() + 80) + tester.newPeer("light", protocol, chainA) + tester.newPeer("heavy", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("light", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("heavy", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that chain forks are contained within a certain interval of the current @@ -857,18 +600,16 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA + chainB := testChainForkLightB + tester.newPeer("original", protocol, chainA) + tester.newPeer("rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { @@ -893,17 +634,16 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit + chainA := testChainForkLightA + chainB := testChainForkHeavy + tester.newPeer("original", protocol, chainA) + tester.newPeer("heavy-rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { @@ -924,7 +664,7 @@ func TestInactiveDownloader62(t *testing.T) { t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } } @@ -962,17 +702,8 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download and the tester - targetBlocks := blockCacheItems - 15 - if targetBlocks >= MaxHashFetch { - targetBlocks = MaxHashFetch - 15 - } - if targetBlocks >= MaxHeaderFetch { - targetBlocks = MaxHeaderFetch - 15 - } - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(MaxHeaderFetch) + tester.newPeer("peer", protocol, chain) // Make sure canceling works with a pristine downloader tester.downloader.Cancel() @@ -1005,17 +736,16 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Create various peers with various parts of the chain targetPeers := 8 - targetBlocks := targetPeers*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetPeers * 100) for i := 0; i < targetPeers; i++ { id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts) + tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1))) } if err := tester.sync("peer #0", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that synchronisations behave well in multi-version protocol environments @@ -1034,24 +764,23 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Create peers of every type - tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) - tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) - tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) + tester.newPeer("peer 62", 62, chain) + tester.newPeer("peer 63", 63, chain) + tester.newPeer("peer 64", 64, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peerHashes[peer]; !ok { + if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) } } @@ -1073,10 +802,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a block chain to download - targetBlocks := 2*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase + tester.newPeer("peer", protocol, chain) // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) @@ -1090,16 +817,16 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Validate the number of block bodies that should have been requested bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range blocks { + for _, block := range chain.blockm { if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { bodiesNeeded++ } } - for _, receipt := range receipts { + for _, receipt := range chain.receiptm { if mode == FastSync && len(receipt) > 0 { receiptsNeeded++ } @@ -1127,24 +854,20 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - // Attempt a full sync with an attacker feeding gapped headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["attack"], hashes[missing]) + chain := testChainBase.shorten(blockCacheItems - 15) + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if requested headers are shifted (i.e. first is missing), the queue @@ -1162,25 +885,24 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Attempt a full sync with an attacker feeding shifted headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - delete(tester.peerHeaders["attack"], hashes[len(hashes)-2]) - delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) - delete(tester.peerReceipts["attack"], hashes[len(hashes)-2]) - + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[1]) + delete(brokenChain.blockm, brokenChain.chain[1]) + delete(brokenChain.receiptm, brokenChain.chain[1]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } + // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that upon detecting an invalid header, the recent ones are rolled back @@ -1198,13 +920,14 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Create a small enough block chain to download targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetBlocks) // Attempt to sync with an attacker that feeds junk during the fast sync phase. // This should result in the last fsHeaderSafetyNet headers being rolled back. - tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts) missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) + fastAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) + tester.newPeer("fast-attack", protocol, fastAttackChain) if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") @@ -1212,13 +935,15 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } + // Attempt to sync with an attacker that feeds junk during the block import phase. // This should result in both the last fsHeaderSafetyNet number of headers being // rolled back, and also the pivot point being reverted to a non-block status. - tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts) missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in - delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing]) + blockAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in + delete(blockAttackChain.headerm, blockAttackChain.chain[missing]) + tester.newPeer("block-attack", protocol, blockAttackChain) if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") @@ -1231,19 +956,18 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } + // Attempt to sync with an attacker that withholds promised blocks after the // fast sync pivot point. This could be a trial to leave the node with a bad // but already imported pivot block. - tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts) - missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - + withholdAttackChain := chain.shorten(chain.len()) + tester.newPeer("withhold-attack", protocol, withholdAttackChain) tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i <= len(hashes); i++ { - delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i]) + for i := missing; i < withholdAttackChain.len(); i++ { + delete(withholdAttackChain.headerm, withholdAttackChain.chain[i]) } tester.downloader.syncInitHook = nil } - if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } @@ -1255,20 +979,21 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } - // Synchronise with the valid peer and make sure sync succeeds. Since the last - // rollback should also disable fast syncing for this process, verify that we - // did a fresh full sync. Note, we can't assert anything about the receipts - // since we won't purge the database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + + // synchronise with the valid peer and make sure sync succeeds. Since the last rollback + // should also disable fast syncing for this process, verify that we did a fresh full + // sync. Note, we can't assert anything about the receipts since we won't purge the + // database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - if hs := len(tester.ownHeaders); hs != len(headers) { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers)) + if hs := len(tester.ownHeaders); hs != chain.len() { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len()) } if mode != LightSync { - if bs := len(tester.ownBlocks); bs != len(blocks) { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks)) + if bs := len(tester.ownBlocks); bs != chain.len() { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len()) } } } @@ -1288,9 +1013,8 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false) - tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts) - + chain := testChainBase.shorten(1) + tester.newPeer("attack", protocol, chain) if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) } @@ -1333,21 +1057,22 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Run the tests and check disconnection status tester := newTester() defer tester.terminate() + chain := testChainBase.shorten(1) for i, tt := range tests { // Register a new peer and ensure it's presence id := fmt.Sprintf("test %d", i) - if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil { + if err := tester.newPeer(id, protocol, chain); err != nil { t.Fatalf("test %d: failed to register new peer: %v", i, err) } - if _, ok := tester.peerHashes[id]; !ok { + if _, ok := tester.peers[id]; !ok { t.Fatalf("test %d: registered peer not found", i) } // Simulate a synchronisation and check the required result tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync) - if _, ok := tester.peerHashes[id]; !ok != tt.drop { + if _, ok := tester.peers[id]; !ok != tt.drop { t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) } } @@ -1367,10 +1092,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1380,12 +1102,10 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) + tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2)) pending := new(sync.WaitGroup) pending.Add(1) @@ -1396,16 +1116,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chain.len()/2 - 1), + }) progress <- struct{}{} pending.Wait() // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts) + tester.newPeer("peer-full", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("peer-full", nil, mode); err != nil { @@ -1413,15 +1132,29 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len()/2 - 1), + HighestBlock: uint64(chain.len() - 1), + }) + + // Check final progress after successful sync progress <- struct{}{} pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) +} - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) +func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + t.Helper() + p := d.Progress() + p.KnownStates, p.PulledStates = 0, 0 + want.KnownStates, want.PulledStates = 0, 0 + if p != want { + t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) } } @@ -1440,10 +1173,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a forked chain to simulate origin revertal - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHashFetch) + chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHashFetch) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1453,15 +1184,12 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("fork A", protocol, chainA) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork A", nil, mode); err != nil { @@ -1469,9 +1197,10 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) - } + + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chainA.len() - 1), + }) progress <- struct{}{} pending.Wait() @@ -1479,9 +1208,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + tester.newPeer("fork B", protocol, chainB) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork B", nil, mode); err != nil { @@ -1489,16 +1217,20 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainA.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainB.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) } // Tests that if synchronisation is aborted due to some failure, then the progress @@ -1516,10 +1248,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1529,20 +1258,18 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Attempt a full sync with a faulty peer - tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["faulty"], hashes[missing]) - delete(tester.peerBlocks["faulty"], hashes[missing]) - delete(tester.peerReceipts["faulty"], hashes[missing]) + brokenChain := chain.shorten(chain.len()) + missing := brokenChain.len() / 2 + delete(brokenChain.headerm, brokenChain.chain[missing]) + delete(brokenChain.blockm, brokenChain.chain[missing]) + delete(brokenChain.receiptm, brokenChain.chain[missing]) + tester.newPeer("faulty", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("faulty", nil, mode); err == nil { @@ -1550,16 +1277,17 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress origin remind the same after a failure - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + // Synchronise with a good peer and check that the progress origin remind the same + // after a failure + tester.newPeer("valid", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("valid", nil, mode); err != nil { @@ -1567,16 +1295,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", afterFailedSync) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) } // Tests that if an attacker fakes a chain height, after the attack is detected, @@ -1593,34 +1320,27 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small block chain - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) progress := make(chan struct{}) - tester.downloader.syncInitHook = func(origin, latest uint64) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } - // Create and sync with an attacker that promises a higher chain than available - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - for i := 1; i < 3; i++ { - delete(tester.peerHeaders["attack"], hashes[i]) - delete(tester.peerBlocks["attack"], hashes[i]) - delete(tester.peerReceipts["attack"], hashes[i]) + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Create and sync with an attacker that promises a higher chain than available. + brokenChain := chain.shorten(chain.len()) + numMissing := 5 + for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- { + delete(brokenChain.headerm, brokenChain.chain[i]) } + tester.newPeer("attack", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("attack", nil, mode); err == nil { @@ -1628,14 +1348,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress height has been reduced to the true value - tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts) + // Synchronise with a good peer and check that the progress height has been reduced to + // the true value. + validChain := chain.shorten(chain.len() - numMissing) + tester.newPeer("valid", protocol, validChain) pending.Add(1) go func() { @@ -1645,23 +1368,25 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + CurrentBlock: afterFailedSync.CurrentBlock, + HighestBlock: uint64(validChain.len() - 1), + }) + + // Check final progress after successful sync. progress <- struct{}{} pending.Wait() - - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(validChain.len() - 1), + HighestBlock: uint64(validChain.len() - 1), + }) } // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -// We use data driven subtests to manage this so that it will be parallel on its own -// and not with the other tests, avoiding intermittent failures. func TestDeliverHeadersHang(t *testing.T) { + t.Parallel() + testCases := []struct { protocol int syncMode SyncMode @@ -1675,15 +1400,38 @@ func TestDeliverHeadersHang(t *testing.T) { } for _, tc := range testCases { t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) { + t.Parallel() testDeliverHeadersHang(t, tc.protocol, tc.syncMode) }) } } +func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { + master := newTester() + defer master.terminate() + chain := testChainBase.shorten(15) + + for i := 0; i < 200; i++ { + tester := newTester() + tester.peerDb = master.peerDb + tester.newPeer("peer", protocol, chain) + + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + } +} + type floodingTestPeer struct { peer Peer tester *downloadTester - pend sync.WaitGroup } func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } @@ -1702,54 +1450,32 @@ func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { deliveriesDone := make(chan struct{}, 500) - for i := 0; i < cap(deliveriesDone); i++ { + for i := 0; i < cap(deliveriesDone)-1; i++ { peer := fmt.Sprintf("fake-peer%d", i) - ftp.pend.Add(1) - go func() { ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) deliveriesDone <- struct{}{} - ftp.pend.Done() }() } - // Deliver the actual requested headers. - go ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + // None of the extra deliveries should block. timeout := time.After(60 * time.Second) + launched := false for i := 0; i < cap(deliveriesDone); i++ { select { case <-deliveriesDone: + if !launched { + // Start delivering the requested headers + // after one of the flooding responses has arrived. + go func() { + ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + deliveriesDone <- struct{}{} + }() + launched = true + } case <-timeout: panic("blocked") } } return nil } - -func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - master := newTester() - defer master.terminate() - - hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) - for i := 0; i < 200; i++ { - tester := newTester() - tester.peerDb = master.peerDb - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, - } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() - - // Flush all goroutines to prevent messing with subsequent tests - tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait() - } -} diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go new file mode 100644 index 000000000..0b5a21425 --- /dev/null +++ b/eth/downloader/testchain_test.go @@ -0,0 +1,221 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package downloader + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +// Test chain parameters. +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + testDB = ethdb.NewMemDatabase() + testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000)) +) + +// The common prefix of all test chains: +var testChainBase = newTestChain(blockCacheItems+200, testGenesis) + +// Different forks on top of the base chain: +var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain + +func init() { + var forkLen = int(MaxForkAncestry + 50) + var wg sync.WaitGroup + wg.Add(3) + go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() + go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() + go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() + wg.Wait() +} + +type testChain struct { + genesis *types.Block + chain []common.Hash + headerm map[common.Hash]*types.Header + blockm map[common.Hash]*types.Block + receiptm map[common.Hash][]*types.Receipt + tdm map[common.Hash]*big.Int +} + +// newTestChain creates a blockchain of the given length. +func newTestChain(length int, genesis *types.Block) *testChain { + tc := new(testChain).copy(length) + tc.genesis = genesis + tc.chain = append(tc.chain, genesis.Hash()) + tc.headerm[tc.genesis.Hash()] = tc.genesis.Header() + tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty() + tc.blockm[tc.genesis.Hash()] = tc.genesis + tc.generate(length-1, 0, genesis, false) + return tc +} + +// makeFork creates a fork on top of the test chain. +func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { + fork := tc.copy(tc.len() + length) + fork.generate(length, seed, tc.headBlock(), heavy) + return fork +} + +// shorten creates a copy of the chain with the given length. It panics if the +// length is longer than the number of available blocks. +func (tc *testChain) shorten(length int) *testChain { + if length > tc.len() { + panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len())) + } + return tc.copy(length) +} + +func (tc *testChain) copy(newlen int) *testChain { + cpy := &testChain{ + genesis: tc.genesis, + headerm: make(map[common.Hash]*types.Header, newlen), + blockm: make(map[common.Hash]*types.Block, newlen), + receiptm: make(map[common.Hash][]*types.Receipt, newlen), + tdm: make(map[common.Hash]*big.Int, newlen), + } + for i := 0; i < len(tc.chain) && i < newlen; i++ { + hash := tc.chain[i] + cpy.chain = append(cpy.chain, tc.chain[i]) + cpy.tdm[hash] = tc.tdm[hash] + cpy.blockm[hash] = tc.blockm[hash] + cpy.headerm[hash] = tc.headerm[hash] + cpy.receiptm[hash] = tc.receiptm[hash] + } + return cpy +} + +// generate creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 22th block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { + // start := time.Now() + // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }() + + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // If a heavy chain is requested, delay blocks to raise difficulty + if heavy { + block.OffsetTime(-1) + } + // Include transactions to the miner to make blocks more interesting. + if parent == tc.genesis && i%22 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // if the block number is a multiple of 5, add a bonus uncle to the block + if i > 0 && i%5 == 0 { + block.AddUncle(&types.Header{ + ParentHash: block.PrevBlock(i - 1).Hash(), + Number: big.NewInt(block.Number().Int64() - 1), + }) + } + }) + + // Convert the block-chain into a hash-chain and header/block maps + td := new(big.Int).Set(tc.td(parent.Hash())) + for i, b := range blocks { + td := td.Add(td, b.Difficulty()) + hash := b.Hash() + tc.chain = append(tc.chain, hash) + tc.blockm[hash] = b + tc.headerm[hash] = b.Header() + tc.receiptm[hash] = receipts[i] + tc.tdm[hash] = new(big.Int).Set(td) + } +} + +// len returns the total number of blocks in the chain. +func (tc *testChain) len() int { + return len(tc.chain) +} + +// headBlock returns the head of the chain. +func (tc *testChain) headBlock() *types.Block { + return tc.blockm[tc.chain[len(tc.chain)-1]] +} + +// td returns the total difficulty of the given block. +func (tc *testChain) td(hash common.Hash) *big.Int { + return tc.tdm[hash] +} + +// headersByHash returns headers in ascending order from the given hash. +func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header { + num, _ := tc.hashToNumber(origin) + return tc.headersByNumber(num, amount, skip) +} + +// headersByNumber returns headers in ascending order from the given number. +func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header { + result := make([]*types.Header, 0, amount) + for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 { + if header, ok := tc.headerm[tc.chain[int(num)]]; ok { + result = append(result, header) + } + } + return result +} + +// receipts returns the receipts of the given block hashes. +func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt { + results := make([][]*types.Receipt, 0, len(hashes)) + for _, hash := range hashes { + if receipt, ok := tc.receiptm[hash]; ok { + results = append(results, receipt) + } + } + return results +} + +// bodies returns the block bodies of the given block hashes. +func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) { + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) + for _, hash := range hashes { + if block, ok := tc.blockm[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) + } + } + return transactions, uncles +} + +func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) { + for num, hash := range tc.chain { + if hash == target { + return uint64(num), true + } + } + return 0, false +} |