From f9917c8c7b6d16daadebd72977e56a8adc0382b0 Mon Sep 17 00:00:00 2001 From: zsfelfoldi Date: Tue, 5 Apr 2016 15:22:04 +0200 Subject: core: improved chainDb using sequential keys --- core/database_util.go | 179 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 114 insertions(+), 65 deletions(-) (limited to 'core/database_util.go') diff --git a/core/database_util.go b/core/database_util.go index 3ba80062c..1529d7369 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -36,34 +36,72 @@ var ( headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - blockPrefix = []byte("block-") - blockNumPrefix = []byte("block-num-") + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - headerSuffix = []byte("-header") - bodySuffix = []byte("-body") - tdSuffix = []byte("-td") - - txMetaSuffix = []byte{0x01} - receiptsPrefix = []byte("receipts-") - blockReceiptsPrefix = []byte("receipts-block-") + txMetaSuffix = []byte{0x01} + receiptsPrefix = []byte("receipts-") mipmapPre = []byte("mipmap-log-bloom-") MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000} - blockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] - configPrefix = []byte("ethereum-config-") // config prefix for the db + + // used by old (non-sequential keys) db, now only used for conversion + oldBlockPrefix = []byte("block-") + oldHeaderSuffix = []byte("-header") + oldTdSuffix = []byte("-td") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + oldBodySuffix = []byte("-body") + oldBlockNumPrefix = []byte("block-num-") + oldBlockReceiptsPrefix = []byte("receipts-block-") + oldBlockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] ) +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + // GetCanonicalHash retrieves a hash assigned to a canonical block number. func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { - data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) if len(data) == 0 { - return common.Hash{} + data, _ = db.Get(append(oldBlockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + if len(data) == 0 { + return common.Hash{} + } } return common.BytesToHash(data) } +// missingNumber is returned by GetBlockNumber if no header with the +// given block hash has been stored in the database +const missingNumber = uint64(0xffffffffffffffff) + +// GetBlockNumber returns the block number assigned to a block hash +// if the corresponding header is present in the database +func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { + data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) + if len(data) != 8 { + data, _ := db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldHeaderSuffix...)) + if len(data) == 0 { + return missingNumber + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + glog.Fatalf("failed to decode block header: %v", err) + } + return header.Number.Uint64() + } + return binary.BigEndian.Uint64(data) +} + // GetHeadHeaderHash retrieves the hash of the current canonical head block's // header. The difference between this and GetHeadBlockHash is that whereas the // last block hash is only updated upon a full block import, the last header @@ -100,15 +138,18 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. -func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), headerSuffix...)) +func GetHeaderRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if len(data) == 0 { + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldHeaderSuffix...)) + } return data } // GetHeader retrieves the block header corresponding to the hash, nil if none // found. -func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { - data := GetHeaderRLP(db, hash) +func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header { + data := GetHeaderRLP(db, hash, number) if len(data) == 0 { return nil } @@ -121,15 +162,18 @@ func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { } // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...)) +func GetBodyRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if len(data) == 0 { + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldBodySuffix...)) + } return data } // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash) *types.Body { - data := GetBodyRLP(db, hash) +func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { + data := GetBodyRLP(db, hash, number) if len(data) == 0 { return nil } @@ -143,10 +187,13 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body { // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. -func GetTd(db ethdb.Database, hash common.Hash) *big.Int { - data, _ := db.Get(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { + data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), tdSuffix...)) if len(data) == 0 { - return nil + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldTdSuffix...)) + if len(data) == 0 { + return nil + } } td := new(big.Int) if err := rlp.Decode(bytes.NewReader(data), td); err != nil { @@ -158,13 +205,13 @@ func GetTd(db ethdb.Database, hash common.Hash) *big.Int { // GetBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. -func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { +func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { // Retrieve the block header and body contents - header := GetHeader(db, hash) + header := GetHeader(db, hash, number) if header == nil { return nil } - body := GetBody(db, hash) + body := GetBody(db, hash, number) if body == nil { return nil } @@ -174,10 +221,13 @@ func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { - data, _ := db.Get(append(blockReceiptsPrefix, hash[:]...)) +func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types.Receipts { + data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { - return nil + data, _ = db.Get(append(oldBlockReceiptsPrefix, hash.Bytes()...)) + if len(data) == 0 { + return nil + } } storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { @@ -235,10 +285,9 @@ func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { // WriteCanonicalHash stores the canonical hash for the given block number. func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { - key := append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...) + key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { glog.Fatalf("failed to store number to hash mapping into database: %v", err) - return err } return nil } @@ -247,7 +296,6 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last header's hash into database: %v", err) - return err } return nil } @@ -256,7 +304,6 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last block's hash into database: %v", err) - return err } return nil } @@ -265,7 +312,6 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last fast block's hash into database: %v", err) - return err } return nil } @@ -276,40 +322,44 @@ func WriteHeader(db ethdb.Database, header *types.Header) error { if err != nil { return err } - key := append(append(blockPrefix, header.Hash().Bytes()...), headerSuffix...) + hash := header.Hash().Bytes() + num := header.Number.Uint64() + encNum := encodeBlockNumber(num) + key := append(blockHashPrefix, hash...) + if err := db.Put(key, encNum); err != nil { + glog.Fatalf("failed to store hash to number mapping into database: %v", err) + } + key = append(append(headerPrefix, encNum...), hash...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store header into database: %v", err) - return err } - glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, header.Hash().Bytes()[:4]) + glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, hash[:4]) return nil } // WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error { +func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err } - key := append(append(blockPrefix, hash.Bytes()...), bodySuffix...) + key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store block body into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4]) return nil } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { +func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err } - key := append(append(blockPrefix, hash.Bytes()...), tdSuffix...) + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store block total difficulty into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block total difficulty [%x…]: %v", hash.Bytes()[:4], td) return nil @@ -318,7 +368,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { // WriteBlock serializes a block into the database, header and body separately. func WriteBlock(db ethdb.Database, block *types.Block) error { // Store the body first to retain database consistency - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err } // Store the header too, signaling full block ownership @@ -331,7 +381,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -342,9 +392,9 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Rece return err } // Store the flattened receipt slice - if err := db.Put(append(blockReceiptsPrefix, hash.Bytes()...), bytes); err != nil { + key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, bytes); err != nil { glog.Fatalf("failed to store block receipts into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block receipts [%x…]", hash.Bytes()[:4]) return nil @@ -388,7 +438,6 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error { // Write the scheduled data into the database if err := batch.Write(); err != nil { glog.Fatalf("failed to store transactions into database: %v", err) - return err } return nil } @@ -411,42 +460,42 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error { // Write the scheduled data into the database if err := batch.Write(); err != nil { glog.Fatalf("failed to store receipts into database: %v", err) - return err } return nil } // DeleteCanonicalHash removes the number to hash canonical mapping. func DeleteCanonicalHash(db ethdb.Database, number uint64) { - db.Delete(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), headerSuffix...)) +func DeleteHeader(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(blockHashPrefix, hash.Bytes()...)) + db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), bodySuffix...)) +func DeleteBody(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +func DeleteTd(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...)) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db ethdb.Database, hash common.Hash) { - DeleteBlockReceipts(db, hash) - DeleteHeader(db, hash) - DeleteBody(db, hash) - DeleteTd(db, hash) +func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { + DeleteBlockReceipts(db, hash, number) + DeleteHeader(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) } // DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db ethdb.Database, hash common.Hash) { - db.Delete(append(blockReceiptsPrefix, hash.Bytes()...)) +func DeleteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTransaction removes all transaction data associated with a hash. @@ -466,7 +515,7 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) { // access the old combined block representation. It will be dropped after the // network transitions to eth/63. func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block { - data, _ := db.Get(append(blockHashPrefix, hash[:]...)) + data, _ := db.Get(append(oldBlockHashPrefix, hash[:]...)) if len(data) == 0 { return nil } -- cgit v1.2.3