From f9917c8c7b6d16daadebd72977e56a8adc0382b0 Mon Sep 17 00:00:00 2001 From: zsfelfoldi Date: Tue, 5 Apr 2016 15:22:04 +0200 Subject: core: improved chainDb using sequential keys --- eth/api.go | 26 ++-- eth/backend.go | 118 ++-------------- eth/backend_test.go | 2 +- eth/db_upgrade.go | 343 +++++++++++++++++++++++++++++++++++++++++++++ eth/filters/api.go | 2 +- eth/filters/filter.go | 7 +- eth/filters/filter_test.go | 4 +- eth/gasprice.go | 2 +- eth/handler.go | 32 +++-- eth/handler_test.go | 10 +- eth/sync.go | 3 +- 11 files changed, 400 insertions(+), 149 deletions(-) create mode 100644 eth/db_upgrade.go (limited to 'eth') diff --git a/eth/api.go b/eth/api.go index f5f942c27..3cb6c4d10 100644 --- a/eth/api.go +++ b/eth/api.go @@ -594,7 +594,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(blockNr rpc.BlockNumber, fullTx b // GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full // detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByHash(blockHash common.Hash, fullTx bool) (map[string]interface{}, error) { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return s.rpcOutputBlock(block, true, fullTx) } return nil, nil @@ -618,7 +618,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(blockNr rpc.BlockNum // GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. When fullTx is true // all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(blockHash common.Hash, index rpc.HexNumber) (map[string]interface{}, error) { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { uncles := block.Uncles() if index.Int() < 0 || index.Int() >= len(uncles) { glog.V(logger.Debug).Infof("uncle block on index %d not found for block %s", index.Int(), blockHash.Hex()) @@ -640,7 +640,7 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(blockNr rpc.BlockNumber // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(blockHash common.Hash) *rpc.HexNumber { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return rpc.NewHexNumber(len(block.Uncles())) } return nil @@ -814,7 +814,7 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "stateRoot": b.Root(), "miner": b.Coinbase(), "difficulty": rpc.NewHexNumber(b.Difficulty()), - "totalDifficulty": rpc.NewHexNumber(s.bc.GetTd(b.Hash())), + "totalDifficulty": rpc.NewHexNumber(s.bc.GetTd(b.Hash(), b.NumberU64())), "extraData": fmt.Sprintf("0x%x", b.Extra()), "size": rpc.NewHexNumber(b.Size().Int64()), "gasLimit": rpc.NewHexNumber(b.GasLimit()), @@ -1004,7 +1004,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(blockNr rpc. // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(blockHash common.Hash) *rpc.HexNumber { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return rpc.NewHexNumber(len(block.Transactions())) } return nil @@ -1020,7 +1020,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(blockNr r // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(blockHash common.Hash, index rpc.HexNumber) (*RPCTransaction, error) { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return newRPCTransactionFromBlockIndex(block, index.Int()) } return nil, nil @@ -1080,7 +1080,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByHash(txHash common.Hash) (*RP return nil, nil } - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return newRPCTransaction(block, txHash) } @@ -1705,7 +1705,7 @@ func (api *PrivateDebugAPI) TraceBlockByNumber(number uint64, config *vm.Config) // TraceBlockByHash processes the block by hash. func (api *PrivateDebugAPI) TraceBlockByHash(hash common.Hash, config *vm.Config) BlockTraceResult { // Fetch the block that we aim to reprocess - block := api.eth.BlockChain().GetBlock(hash) + block := api.eth.BlockChain().GetBlockByHash(hash) if block == nil { return BlockTraceResult{Error: fmt.Sprintf("block #%x not found", hash)} } @@ -1745,10 +1745,10 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, config *vm.Config) (b config.Debug = true // make sure debug is set. config.Logger.Collector = collector - if err := core.ValidateHeader(api.config, blockchain.AuxValidator(), block.Header(), blockchain.GetHeader(block.ParentHash()), true, false); err != nil { + if err := core.ValidateHeader(api.config, blockchain.AuxValidator(), block.Header(), blockchain.GetHeader(block.ParentHash(), block.NumberU64()-1), true, false); err != nil { return false, collector.traces, err } - statedb, err := state.New(blockchain.GetBlock(block.ParentHash()).Root(), api.eth.ChainDb()) + statedb, err := state.New(blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), api.eth.ChainDb()) if err != nil { return false, collector.traces, err } @@ -1757,7 +1757,7 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, config *vm.Config) (b if err != nil { return false, collector.traces, err } - if err := validator.ValidateState(block, blockchain.GetBlock(block.ParentHash()), statedb, receipts, usedGas); err != nil { + if err := validator.ValidateState(block, blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1), statedb, receipts, usedGas); err != nil { return false, collector.traces, err } return true, collector.traces, nil @@ -1841,12 +1841,12 @@ func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogC if tx == nil { return nil, fmt.Errorf("transaction %x not found", txHash) } - block := api.eth.BlockChain().GetBlock(blockHash) + block := api.eth.BlockChain().GetBlockByHash(blockHash) if block == nil { return nil, fmt.Errorf("block %x not found", blockHash) } // Create the state database to mutate and eventually trace - parent := api.eth.BlockChain().GetBlock(block.ParentHash()) + parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { return nil, fmt.Errorf("block parent %x not found", block.ParentHash()) } diff --git a/eth/backend.go b/eth/backend.go index bb487650b..006523484 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -18,7 +18,6 @@ package eth import ( - "bytes" "errors" "fmt" "math/big" @@ -47,7 +46,6 @@ import ( "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -104,9 +102,9 @@ type Config struct { } type Ethereum struct { - chainConfig *core.ChainConfig - // Channel for shutting down the ethereum - shutdownChan chan bool + chainConfig *core.ChainConfig + shutdownChan chan bool // Channel for shutting down the ethereum + stopDbUpgrade func() // stop chain db sequential key upgrade // DB interfaces chainDb ethdb.Database // Block chain database @@ -161,6 +159,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err := addMipmapBloomBins(chainDb); err != nil { return nil, err } + stopDbUpgrade := upgradeSequentialKeys(chainDb) dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache, config.DatabaseHandles) if err != nil { @@ -185,7 +184,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { chainDb = config.TestGenesisState } if config.TestGenesisBlock != nil { - core.WriteTd(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.Difficulty()) + core.WriteTd(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64(), config.TestGenesisBlock.Difficulty()) core.WriteBlock(chainDb, config.TestGenesisBlock) core.WriteCanonicalHash(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64()) core.WriteHeadBlockHash(chainDb, config.TestGenesisBlock.Hash()) @@ -202,6 +201,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth := &Ethereum{ shutdownChan: make(chan bool), + stopDbUpgrade: stopDbUpgrade, chainDb: chainDb, dappDb: dappDb, eventMux: ctx.EventMux, @@ -238,7 +238,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { // load the genesis block or write a new one if no genesis // block is prenent in the database. - genesis := core.GetBlock(chainDb, core.GetCanonicalHash(chainDb, 0)) + genesis := core.GetBlock(chainDb, core.GetCanonicalHash(chainDb, 0), 0) if genesis == nil { genesis, err = core.WriteDefaultGenesisBlock(chainDb) if err != nil { @@ -415,6 +415,9 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { // Stop implements node.Service, terminating all internal goroutines used by the // Ethereum protocol. func (s *Ethereum) Stop() error { + if s.stopDbUpgrade != nil { + s.stopDbUpgrade() + } s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() @@ -526,104 +529,3 @@ func dagFiles(epoch uint64) (string, string) { dag := fmt.Sprintf("full-R%d-%x", ethashRevision, seedHash[:8]) return dag, "full-R" + dag } - -// upgradeChainDatabase ensures that the chain database stores block split into -// separate header and body entries. -func upgradeChainDatabase(db ethdb.Database) error { - // Short circuit if the head block is stored already as separate header and body - data, err := db.Get([]byte("LastBlock")) - if err != nil { - return nil - } - head := common.BytesToHash(data) - - if block := core.GetBlockByHashOld(db, head); block == nil { - return nil - } - // At least some of the database is still the old format, upgrade (skip the head block!) - glog.V(logger.Info).Info("Old database detected, upgrading...") - - if db, ok := db.(*ethdb.LDBDatabase); ok { - blockPrefix := []byte("block-hash-") - for it := db.NewIterator(); it.Next(); { - // Skip anything other than a combined block - if !bytes.HasPrefix(it.Key(), blockPrefix) { - continue - } - // Skip the head block (merge last to signal upgrade completion) - if bytes.HasSuffix(it.Key(), head.Bytes()) { - continue - } - // Load the block, split and serialize (order!) - block := core.GetBlockByHashOld(db, common.BytesToHash(bytes.TrimPrefix(it.Key(), blockPrefix))) - - if err := core.WriteTd(db, block.Hash(), block.DeprecatedTd()); err != nil { - return err - } - if err := core.WriteBody(db, block.Hash(), block.Body()); err != nil { - return err - } - if err := core.WriteHeader(db, block.Header()); err != nil { - return err - } - if err := db.Delete(it.Key()); err != nil { - return err - } - } - // Lastly, upgrade the head block, disabling the upgrade mechanism - current := core.GetBlockByHashOld(db, head) - - if err := core.WriteTd(db, current.Hash(), current.DeprecatedTd()); err != nil { - return err - } - if err := core.WriteBody(db, current.Hash(), current.Body()); err != nil { - return err - } - if err := core.WriteHeader(db, current.Header()); err != nil { - return err - } - } - return nil -} - -func addMipmapBloomBins(db ethdb.Database) (err error) { - const mipmapVersion uint = 2 - - // check if the version is set. We ignore data for now since there's - // only one version so we can easily ignore it for now - var data []byte - data, _ = db.Get([]byte("setting-mipmap-version")) - if len(data) > 0 { - var version uint - if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion { - return nil - } - } - - defer func() { - if err == nil { - var val []byte - val, err = rlp.EncodeToBytes(mipmapVersion) - if err == nil { - err = db.Put([]byte("setting-mipmap-version"), val) - } - return - } - }() - latestBlock := core.GetBlock(db, core.GetHeadBlockHash(db)) - if latestBlock == nil { // clean database - return - } - - tstart := time.Now() - glog.V(logger.Info).Infoln("upgrading db log bloom bins") - for i := uint64(0); i <= latestBlock.NumberU64(); i++ { - hash := core.GetCanonicalHash(db, i) - if (hash == common.Hash{}) { - return fmt.Errorf("chain db corrupted. Could not find block %d.", i) - } - core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash)) - } - glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) - return nil -} diff --git a/eth/backend_test.go b/eth/backend_test.go index c2bed879b..cb94adbf0 100644 --- a/eth/backend_test.go +++ b/eth/backend_test.go @@ -61,7 +61,7 @@ func TestMipmapUpgrade(t *testing.T) { if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { t.Fatalf("failed to insert block number: %v", err) } - if err := core.WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { t.Fatal("error writing block receipts:", err) } } diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go new file mode 100644 index 000000000..12de60fe7 --- /dev/null +++ b/eth/db_upgrade.go @@ -0,0 +1,343 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package eth implements the Ethereum protocol. +package eth + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/rlp" +) + +var useSequentialKeys = []byte("dbUpgrade_20160530sequentialKeys") + +// upgradeSequentialKeys checks the chain database version and +// starts a background process to make upgrades if necessary. +// Returns a stop function that blocks until the process has +// been safely stopped. +func upgradeSequentialKeys(db ethdb.Database) (stopFn func()) { + data, _ := db.Get(useSequentialKeys) + if len(data) > 0 && data[0] == 42 { + return nil // already converted + } + + if data, _ := db.Get([]byte("LastHeader")); len(data) == 0 { + db.Put(useSequentialKeys, []byte{42}) + return nil // empty database, nothing to do + } + + glog.V(logger.Info).Infof("Upgrading chain database to use sequential keys") + + stopChn := make(chan struct{}) + stoppedChn := make(chan struct{}) + + go func() { + stopFn := func() bool { + select { + case <-time.After(time.Microsecond * 100): // make sure other processes don't get starved + case <-stopChn: + return true + } + return false + } + + err, stopped := upgradeSequentialCanonicalNumbers(db, stopFn) + if err == nil && !stopped { + err, stopped = upgradeSequentialBlocks(db, stopFn) + } + if err == nil && !stopped { + err, stopped = upgradeSequentialOrphanedReceipts(db, stopFn) + } + if err == nil && !stopped { + glog.V(logger.Info).Infof("Database conversion successful") + db.Put(useSequentialKeys, []byte{42}) + } + if err != nil { + glog.V(logger.Error).Infof("Database conversion failed: %v", err) + } + close(stoppedChn) + }() + + return func() { + close(stopChn) + <-stoppedChn + } +} + +// upgradeSequentialCanonicalNumbers reads all old format canonical numbers from +// the database, writes them in new format and deletes the old ones if successful. +func upgradeSequentialCanonicalNumbers(db ethdb.Database, stopFn func() bool) (error, bool) { + prefix := []byte("block-num-") + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(prefix) + cnt := 0 + for bytes.HasPrefix(it.Key(), prefix) { + keyPtr := it.Key() + if len(keyPtr) < 20 { + cnt++ + if cnt%100000 == 0 { + glog.V(logger.Info).Infof("converting %d canonical numbers...", cnt) + } + number := big.NewInt(0).SetBytes(keyPtr[10:]).Uint64() + newKey := []byte("h12345678n") + binary.BigEndian.PutUint64(newKey[1:9], number) + if err := db.Put(newKey, it.Value()); err != nil { + return err, false + } + if err := db.Delete(keyPtr); err != nil { + return err, false + } + } + + if stopFn() { + return nil, true + } + it.Next() + } + if cnt > 0 { + glog.V(logger.Info).Infof("converted %d canonical numbers...", cnt) + } + return nil, false +} + +// upgradeSequentialBlocks reads all old format block headers, bodies, TDs and block +// receipts from the database, writes them in new format and deletes the old ones +// if successful. +func upgradeSequentialBlocks(db ethdb.Database, stopFn func() bool) (error, bool) { + prefix := []byte("block-") + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(prefix) + cnt := 0 + for bytes.HasPrefix(it.Key(), prefix) { + keyPtr := it.Key() + if len(keyPtr) >= 38 { + cnt++ + if cnt%10000 == 0 { + glog.V(logger.Info).Infof("converting %d blocks...", cnt) + } + // convert header, body, td and block receipts + var keyPrefix [38]byte + copy(keyPrefix[:], keyPtr[0:38]) + hash := keyPrefix[6:38] + if err := upgradeSequentialBlockData(db, hash); err != nil { + return err, false + } + // delete old db entries belonging to this hash + for bytes.HasPrefix(it.Key(), keyPrefix[:]) { + if err := db.Delete(it.Key()); err != nil { + return err, false + } + it.Next() + } + if err := db.Delete(append([]byte("receipts-block-"), hash...)); err != nil { + return err, false + } + } else { + it.Next() + } + + if stopFn() { + return nil, true + } + } + if cnt > 0 { + glog.V(logger.Info).Infof("converted %d blocks...", cnt) + } + return nil, false +} + +// upgradeSequentialOrphanedReceipts removes any old format block receipts from the +// database that did not have a corresponding block +func upgradeSequentialOrphanedReceipts(db ethdb.Database, stopFn func() bool) (error, bool) { + prefix := []byte("receipts-block-") + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(prefix) + cnt := 0 + for bytes.HasPrefix(it.Key(), prefix) { + // phase 2 already converted receipts belonging to existing + // blocks, just remove if there's anything left + cnt++ + if err := db.Delete(it.Key()); err != nil { + return err, false + } + + if stopFn() { + return nil, true + } + it.Next() + } + if cnt > 0 { + glog.V(logger.Info).Infof("removed %d orphaned block receipts...", cnt) + } + return nil, false +} + +// upgradeSequentialBlockData upgrades the header, body, td and block receipts +// database entries belonging to a single hash (doesn't delete old data). +func upgradeSequentialBlockData(db ethdb.Database, hash []byte) error { + // get old chain data and block number + headerRLP, _ := db.Get(append(append([]byte("block-"), hash...), []byte("-header")...)) + if len(headerRLP) == 0 { + return nil + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(headerRLP), header); err != nil { + return err + } + number := header.Number.Uint64() + bodyRLP, _ := db.Get(append(append([]byte("block-"), hash...), []byte("-body")...)) + tdRLP, _ := db.Get(append(append([]byte("block-"), hash...), []byte("-td")...)) + receiptsRLP, _ := db.Get(append([]byte("receipts-block-"), hash...)) + // store new hash -> number association + encNum := make([]byte, 8) + binary.BigEndian.PutUint64(encNum, number) + if err := db.Put(append([]byte("H"), hash...), encNum); err != nil { + return err + } + // store new chain data + if err := db.Put(append(append([]byte("h"), encNum...), hash...), headerRLP); err != nil { + return err + } + if len(tdRLP) != 0 { + if err := db.Put(append(append(append([]byte("h"), encNum...), hash...), []byte("t")...), tdRLP); err != nil { + return err + } + } + if len(bodyRLP) != 0 { + if err := db.Put(append(append([]byte("b"), encNum...), hash...), bodyRLP); err != nil { + return err + } + } + if len(receiptsRLP) != 0 { + if err := db.Put(append(append([]byte("r"), encNum...), hash...), receiptsRLP); err != nil { + return err + } + } + return nil +} + +// upgradeChainDatabase ensures that the chain database stores block split into +// separate header and body entries. +func upgradeChainDatabase(db ethdb.Database) error { + // Short circuit if the head block is stored already as separate header and body + data, err := db.Get([]byte("LastBlock")) + if err != nil { + return nil + } + head := common.BytesToHash(data) + + if block := core.GetBlockByHashOld(db, head); block == nil { + return nil + } + // At least some of the database is still the old format, upgrade (skip the head block!) + glog.V(logger.Info).Info("Old database detected, upgrading...") + + if db, ok := db.(*ethdb.LDBDatabase); ok { + blockPrefix := []byte("block-hash-") + for it := db.NewIterator(); it.Next(); { + // Skip anything other than a combined block + if !bytes.HasPrefix(it.Key(), blockPrefix) { + continue + } + // Skip the head block (merge last to signal upgrade completion) + if bytes.HasSuffix(it.Key(), head.Bytes()) { + continue + } + // Load the block, split and serialize (order!) + block := core.GetBlockByHashOld(db, common.BytesToHash(bytes.TrimPrefix(it.Key(), blockPrefix))) + + if err := core.WriteTd(db, block.Hash(), block.NumberU64(), block.DeprecatedTd()); err != nil { + return err + } + if err := core.WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { + return err + } + if err := core.WriteHeader(db, block.Header()); err != nil { + return err + } + if err := db.Delete(it.Key()); err != nil { + return err + } + } + // Lastly, upgrade the head block, disabling the upgrade mechanism + current := core.GetBlockByHashOld(db, head) + + if err := core.WriteTd(db, current.Hash(), current.NumberU64(), current.DeprecatedTd()); err != nil { + return err + } + if err := core.WriteBody(db, current.Hash(), current.NumberU64(), current.Body()); err != nil { + return err + } + if err := core.WriteHeader(db, current.Header()); err != nil { + return err + } + } + return nil +} + +func addMipmapBloomBins(db ethdb.Database) (err error) { + const mipmapVersion uint = 2 + + // check if the version is set. We ignore data for now since there's + // only one version so we can easily ignore it for now + var data []byte + data, _ = db.Get([]byte("setting-mipmap-version")) + if len(data) > 0 { + var version uint + if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion { + return nil + } + } + + defer func() { + if err == nil { + var val []byte + val, err = rlp.EncodeToBytes(mipmapVersion) + if err == nil { + err = db.Put([]byte("setting-mipmap-version"), val) + } + return + } + }() + latestHash := core.GetHeadBlockHash(db) + latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash)) + if latestBlock == nil { // clean database + return + } + + tstart := time.Now() + glog.V(logger.Info).Infoln("upgrading db log bloom bins") + for i := uint64(0); i <= latestBlock.NumberU64(); i++ { + hash := core.GetCanonicalHash(db, i) + if (hash == common.Hash{}) { + return fmt.Errorf("chain db corrupted. Could not find block %d.", i) + } + core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i)) + } + glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) + return nil +} diff --git a/eth/filters/api.go b/eth/filters/api.go index 7278e20b9..393019f8b 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -361,7 +361,7 @@ func (args *NewFilterArgs) UnmarshalJSON(data []byte) error { if len(raw) >= 2 && raw[0] == '0' && (raw[1] == 'x' || raw[1] == 'X') { raw = raw[2:] } - if len(raw) != 2 * common.HashLength { + if len(raw) != 2*common.HashLength { return common.Hash{}, errors.New("invalid topic(s)") } if decAddr, err := hex.DecodeString(raw); err == nil { diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 469dfba4d..995b588fb 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -72,7 +72,8 @@ func (self *Filter) SetTopics(topics [][]common.Hash) { // Run filters logs with the current parameters set func (self *Filter) Find() vm.Logs { - latestBlock := core.GetBlock(self.db, core.GetHeadBlockHash(self.db)) + latestHash := core.GetHeadBlockHash(self.db) + latestBlock := core.GetBlock(self.db, latestHash, core.GetBlockNumber(self.db, latestHash)) var beginBlockNo uint64 = uint64(self.begin) if self.begin == -1 { beginBlockNo = latestBlock.NumberU64() @@ -127,7 +128,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { for i := start; i <= end; i++ { hash := core.GetCanonicalHash(self.db, i) if hash != (common.Hash{}) { - block = core.GetBlock(self.db, hash) + block = core.GetBlock(self.db, hash, i) } else { // block not found return logs } @@ -137,7 +138,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { if self.bloomFilter(block) { // Get the logs of the block var ( - receipts = core.GetBlockReceipts(self.db, block.Hash()) + receipts = core.GetBlockReceipts(self.db, block.Hash(), i) unfiltered vm.Logs ) for _, receipt := range receipts { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index b0f88ffeb..a95adfce7 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -94,7 +94,7 @@ func BenchmarkMipmaps(b *testing.B) { if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { b.Fatalf("failed to insert block number: %v", err) } - if err := core.WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { b.Fatal("error writing block receipts:", err) } } @@ -196,7 +196,7 @@ func TestFilters(t *testing.T) { if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { t.Fatalf("failed to insert block number: %v", err) } - if err := core.WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { t.Fatal("error writing block receipts:", err) } } diff --git a/eth/gasprice.go b/eth/gasprice.go index e0de89e62..ef203f8fe 100644 --- a/eth/gasprice.go +++ b/eth/gasprice.go @@ -166,7 +166,7 @@ func (self *GasPriceOracle) processBlock(block *types.Block) { func (self *GasPriceOracle) lowestPrice(block *types.Block) *big.Int { gasUsed := big.NewInt(0) - receipts := core.GetBlockReceipts(self.eth.ChainDb(), block.Hash()) + receipts := core.GetBlockReceipts(self.eth.ChainDb(), block.Hash(), block.NumberU64()) if len(receipts) > 0 { if cgu := receipts[len(receipts)-1].CumulativeGasUsed; cgu != nil { gasUsed = receipts[len(receipts)-1].CumulativeGasUsed diff --git a/eth/handler.go b/eth/handler.go index 1e4dc1289..47a36cc0b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -152,9 +152,9 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, return nil, errIncompatibleConfig } // Construct the different synchronisation mechanisms - manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeader, - blockchain.GetBlock, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, - blockchain.GetTd, blockchain.InsertHeaderChain, manager.insertChain, blockchain.InsertReceiptChain, blockchain.Rollback, + manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeaderByHash, + blockchain.GetBlockByHash, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, + blockchain.GetTdByHash, blockchain.InsertHeaderChain, manager.insertChain, blockchain.InsertReceiptChain, blockchain.Rollback, manager.removePeer) validator := func(block *types.Block, parent *types.Block) error { @@ -167,7 +167,7 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, atomic.StoreUint32(&manager.synced, 1) // Mark initial sync done on any fetcher import return manager.insertChain(blocks) } - manager.fetcher = fetcher.New(blockchain.GetBlock, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) + manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) if blockchain.Genesis().Hash().Hex() == defaultGenesisHash && networkId == 1 { glog.V(logger.Debug).Infoln("Bad Block Reporting is enabled") @@ -382,7 +382,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Retrieve the requested block, stopping if enough was found - if block := pm.blockchain.GetBlock(hash); block != nil { + if block := pm.blockchain.GetBlockByHash(hash); block != nil { blocks = append(blocks, block) bytes += block.Size() } @@ -425,13 +425,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Retrieve the next header satisfying the query var origin *types.Header if hashMode { - origin = pm.blockchain.GetHeader(query.Origin.Hash) + origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) } else { origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) } if origin == nil { break } + number := origin.Number.Uint64() headers = append(headers, origin) bytes += estHeaderRlpSize @@ -440,8 +441,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { case query.Origin.Hash != (common.Hash{}) && query.Reverse: // Hash based traversal towards the genesis block for i := 0; i < int(query.Skip)+1; i++ { - if header := pm.blockchain.GetHeader(query.Origin.Hash); header != nil { + if header := pm.blockchain.GetHeader(query.Origin.Hash, number); header != nil { query.Origin.Hash = header.ParentHash + number-- } else { unknown = true break @@ -602,9 +604,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Retrieve the requested block's receipts, skipping if unknown to us - results := core.GetBlockReceipts(pm.chaindb, hash) + results := core.GetBlockReceipts(pm.chaindb, hash, core.GetBlockNumber(pm.chaindb, hash)) if results == nil { - if header := pm.blockchain.GetHeader(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { continue } } @@ -697,7 +699,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Update the peers total difficulty if needed, schedule a download if gapped if request.TD.Cmp(p.Td()) > 0 { p.SetTd(request.TD) - td := pm.blockchain.GetTd(pm.blockchain.CurrentBlock().Hash()) + currentBlock := pm.blockchain.CurrentBlock() + td := pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) if request.TD.Cmp(new(big.Int).Add(td, request.Block.Difficulty())) > 0 { go pm.synchronise(p) } @@ -738,8 +741,8 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { if propagate { // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) var td *big.Int - if parent := pm.blockchain.GetBlock(block.ParentHash()); parent != nil { - td = new(big.Int).Add(block.Difficulty(), pm.blockchain.GetTd(block.ParentHash())) + if parent := pm.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { + td = new(big.Int).Add(block.Difficulty(), pm.blockchain.GetTd(block.ParentHash(), block.NumberU64()-1)) } else { glog.V(logger.Error).Infof("propagating dangling block #%d [%x]", block.NumberU64(), hash[:4]) return @@ -807,10 +810,11 @@ type EthNodeInfo struct { // NodeInfo retrieves some protocol metadata about the running host node. func (self *ProtocolManager) NodeInfo() *EthNodeInfo { + currentBlock := self.blockchain.CurrentBlock() return &EthNodeInfo{ Network: self.networkId, - Difficulty: self.blockchain.GetTd(self.blockchain.CurrentBlock().Hash()), + Difficulty: self.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), Genesis: self.blockchain.Genesis().Hash(), - Head: self.blockchain.CurrentBlock().Hash(), + Head: currentBlock.Hash(), } } diff --git a/eth/handler_test.go b/eth/handler_test.go index 9e593f040..8418c28b2 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -91,7 +91,7 @@ func testGetBlockHashes(t *testing.T, protocol int) { // Assemble the hash response we would like to receive resp := make([]common.Hash, tt.result) if len(resp) > 0 { - from := pm.blockchain.GetBlock(tt.origin).NumberU64() - 1 + from := pm.blockchain.GetBlockByHash(tt.origin).NumberU64() - 1 for j := 0; j < len(resp); j++ { resp[j] = pm.blockchain.GetBlockByNumber(uint64(int(from) - j)).Hash() } @@ -204,7 +204,7 @@ func testGetBlocks(t *testing.T, protocol int) { for j, hash := range tt.explicit { hashes = append(hashes, hash) if tt.available[j] && len(blocks) < tt.expected { - blocks = append(blocks, pm.blockchain.GetBlock(hash)) + blocks = append(blocks, pm.blockchain.GetBlockByHash(hash)) } } // Send the hash request and verify the response @@ -339,7 +339,7 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Collect the headers to expect in the response headers := []*types.Header{} for _, hash := range tt.expect { - headers = append(headers, pm.blockchain.GetBlock(hash).Header()) + headers = append(headers, pm.blockchain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response p2p.Send(peer.app, 0x03, tt.query) @@ -420,7 +420,7 @@ func testGetBlockBodies(t *testing.T, protocol int) { for j, hash := range tt.explicit { hashes = append(hashes, hash) if tt.available[j] && len(bodies) < tt.expected { - block := pm.blockchain.GetBlock(hash) + block := pm.blockchain.GetBlockByHash(hash) bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) } } @@ -572,7 +572,7 @@ func testGetReceipt(t *testing.T, protocol int) { block := pm.blockchain.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash())) + receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response p2p.Send(peer.app, 0x0f, hashes) diff --git a/eth/sync.go b/eth/sync.go index 52f7e90e7..23cf18c8d 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -162,7 +162,8 @@ func (pm *ProtocolManager) synchronise(peer *peer) { return } // Make sure the peer's TD is higher than our own. If not drop. - td := pm.blockchain.GetTd(pm.blockchain.CurrentBlock().Hash()) + currentBlock := pm.blockchain.CurrentBlock() + td := pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) if peer.Td().Cmp(td) <= 0 { return } -- cgit v1.2.3