From ca376ead88a5a26626a90abdb62f4de7f6313822 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= <zsfelfoldi@gmail.com>
Date: Tue, 24 Oct 2017 15:19:09 +0200
Subject: les, light: LES/2 protocol version (#14970)
This PR implements the new LES protocol version extensions:
* new and more efficient Merkle proofs reply format (when replying to
a multiple Merkle proofs request, we just send a single set of trie
nodes containing all necessary nodes)
* BBT (BloomBitsTrie) works similarly to the existing CHT and contains
the bloombits search data to speed up log searches
* GetTxStatusMsg returns the inclusion position or the
pending/queued/unknown state of a transaction referenced by hash
* an optional signature of new block data (number/hash/td) can be
included in AnnounceMsg to provide an option for "very light
clients" (mobile/embedded devices) to skip expensive Ethash check
and accept multiple signatures of somewhat trusted servers (still a
lot better than trusting a single server completely and retrieving
everything through RPC). The new client mode is not implemented in
this PR, just the protocol extension.
---
light/lightchain.go | 36 ++++---
light/nodeset.go | 141 ++++++++++++++++++++++++
light/odr.go | 48 +++++----
light/odr_test.go | 4 +-
light/odr_util.go | 121 +++++++++++++--------
light/postprocess.go | 295 +++++++++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 571 insertions(+), 74 deletions(-)
create mode 100644 light/nodeset.go
create mode 100644 light/postprocess.go
(limited to 'light')
diff --git a/light/lightchain.go b/light/lightchain.go
index 4c877a771..30baeaccb 100644
--- a/light/lightchain.go
+++ b/light/lightchain.go
@@ -95,15 +95,8 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.
if bc.genesisBlock == nil {
return nil, core.ErrNoGenesis
}
- if bc.genesisBlock.Hash() == params.MainnetGenesisHash {
- // add trusted CHT
- WriteTrustedCht(bc.chainDb, TrustedCht{Number: 1040, Root: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca")})
- log.Info("Added trusted CHT for mainnet")
- }
- if bc.genesisBlock.Hash() == params.TestnetGenesisHash {
- // add trusted CHT
- WriteTrustedCht(bc.chainDb, TrustedCht{Number: 400, Root: common.HexToHash("2a4befa19e4675d939c3dc22dca8c6ae9fcd642be1f04b06bd6e4203cc304660")})
- log.Info("Added trusted CHT for ropsten testnet")
+ if cp, ok := trustedCheckpoints[bc.genesisBlock.Hash()]; ok {
+ bc.addTrustedCheckpoint(cp)
}
if err := bc.loadLastState(); err != nil {
@@ -120,6 +113,22 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.
return bc, nil
}
+// addTrustedCheckpoint adds a trusted checkpoint to the blockchain
+func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) {
+ if self.odr.ChtIndexer() != nil {
+ StoreChtRoot(self.chainDb, cp.sectionIdx, cp.sectionHead, cp.chtRoot)
+ self.odr.ChtIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead)
+ }
+ if self.odr.BloomTrieIndexer() != nil {
+ StoreBloomTrieRoot(self.chainDb, cp.sectionIdx, cp.sectionHead, cp.bloomTrieRoot)
+ self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead)
+ }
+ if self.odr.BloomIndexer() != nil {
+ self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead)
+ }
+ log.Info("Added trusted checkpoint", "chain name", cp.name)
+}
+
func (self *LightChain) getProcInterrupt() bool {
return atomic.LoadInt32(&self.procInterrupt) == 1
}
@@ -449,10 +458,13 @@ func (self *LightChain) GetHeaderByNumberOdr(ctx context.Context, number uint64)
}
func (self *LightChain) SyncCht(ctx context.Context) bool {
+ if self.odr.ChtIndexer() == nil {
+ return false
+ }
headNum := self.CurrentHeader().Number.Uint64()
- cht := GetTrustedCht(self.chainDb)
- if headNum+1 < cht.Number*ChtFrequency {
- num := cht.Number*ChtFrequency - 1
+ chtCount, _, _ := self.odr.ChtIndexer().Sections()
+ if headNum+1 < chtCount*ChtFrequency {
+ num := chtCount*ChtFrequency - 1
header, err := GetHeaderByNumber(ctx, self.odr, num)
if header != nil && err == nil {
self.mu.Lock()
diff --git a/light/nodeset.go b/light/nodeset.go
new file mode 100644
index 000000000..c530a4fbe
--- /dev/null
+++ b/light/nodeset.go
@@ -0,0 +1,141 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package light
+
+import (
+ "errors"
+ "sync"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// NodeSet stores a set of trie nodes. It implements trie.Database and can also
+// act as a cache for another trie.Database.
+type NodeSet struct {
+ db map[string][]byte
+ dataSize int
+ lock sync.RWMutex
+}
+
+// NewNodeSet creates an empty node set
+func NewNodeSet() *NodeSet {
+ return &NodeSet{
+ db: make(map[string][]byte),
+ }
+}
+
+// Put stores a new node in the set
+func (db *NodeSet) Put(key []byte, value []byte) error {
+ db.lock.Lock()
+ defer db.lock.Unlock()
+
+ if _, ok := db.db[string(key)]; !ok {
+ db.db[string(key)] = common.CopyBytes(value)
+ db.dataSize += len(value)
+ }
+ return nil
+}
+
+// Get returns a stored node
+func (db *NodeSet) Get(key []byte) ([]byte, error) {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
+ if entry, ok := db.db[string(key)]; ok {
+ return entry, nil
+ }
+ return nil, errors.New("not found")
+}
+
+// Has returns true if the node set contains the given key
+func (db *NodeSet) Has(key []byte) (bool, error) {
+ _, err := db.Get(key)
+ return err == nil, nil
+}
+
+// KeyCount returns the number of nodes in the set
+func (db *NodeSet) KeyCount() int {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
+ return len(db.db)
+}
+
+// DataSize returns the aggregated data size of nodes in the set
+func (db *NodeSet) DataSize() int {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
+ return db.dataSize
+}
+
+// NodeList converts the node set to a NodeList
+func (db *NodeSet) NodeList() NodeList {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
+ var values NodeList
+ for _, value := range db.db {
+ values = append(values, value)
+ }
+ return values
+}
+
+// Store writes the contents of the set to the given database
+func (db *NodeSet) Store(target trie.Database) {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
+ for key, value := range db.db {
+ target.Put([]byte(key), value)
+ }
+}
+
+// NodeList stores an ordered list of trie nodes. It implements trie.DatabaseWriter.
+type NodeList []rlp.RawValue
+
+// Store writes the contents of the list to the given database
+func (n NodeList) Store(db trie.Database) {
+ for _, node := range n {
+ db.Put(crypto.Keccak256(node), node)
+ }
+}
+
+// NodeSet converts the node list to a NodeSet
+func (n NodeList) NodeSet() *NodeSet {
+ db := NewNodeSet()
+ n.Store(db)
+ return db
+}
+
+// Put stores a new node at the end of the list
+func (n *NodeList) Put(key []byte, value []byte) error {
+ *n = append(*n, value)
+ return nil
+}
+
+// DataSize returns the aggregated data size of nodes in the list
+func (n NodeList) DataSize() int {
+ var size int
+ for _, node := range n {
+ size += len(node)
+ }
+ return size
+}
diff --git a/light/odr.go b/light/odr.go
index d19a488f6..e2c3d9c5a 100644
--- a/light/odr.go
+++ b/light/odr.go
@@ -25,9 +25,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
- "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
- "github.com/ethereum/go-ethereum/rlp"
)
// NoOdr is the default context passed to an ODR capable function when the ODR
@@ -37,6 +35,9 @@ var NoOdr = context.Background()
// OdrBackend is an interface to a backend service that handles ODR retrievals type
type OdrBackend interface {
Database() ethdb.Database
+ ChtIndexer() *core.ChainIndexer
+ BloomTrieIndexer() *core.ChainIndexer
+ BloomIndexer() *core.ChainIndexer
Retrieve(ctx context.Context, req OdrRequest) error
}
@@ -80,23 +81,12 @@ type TrieRequest struct {
OdrRequest
Id *TrieID
Key []byte
- Proof []rlp.RawValue
+ Proof *NodeSet
}
// StoreResult stores the retrieved data in local database
func (req *TrieRequest) StoreResult(db ethdb.Database) {
- storeProof(db, req.Proof)
-}
-
-// storeProof stores the new trie nodes obtained from a merkle proof in the database
-func storeProof(db ethdb.Database, proof []rlp.RawValue) {
- for _, buf := range proof {
- hash := crypto.Keccak256(buf)
- val, _ := db.Get(hash)
- if val == nil {
- db.Put(hash, buf)
- }
- }
+ req.Proof.Store(db)
}
// CodeRequest is the ODR request type for retrieving contract code
@@ -138,14 +128,14 @@ func (req *ReceiptsRequest) StoreResult(db ethdb.Database) {
core.WriteBlockReceipts(db, req.Hash, req.Number, req.Receipts)
}
-// TrieRequest is the ODR request type for state/storage trie entries
+// ChtRequest is the ODR request type for state/storage trie entries
type ChtRequest struct {
OdrRequest
ChtNum, BlockNum uint64
ChtRoot common.Hash
Header *types.Header
Td *big.Int
- Proof []rlp.RawValue
+ Proof *NodeSet
}
// StoreResult stores the retrieved data in local database
@@ -155,5 +145,27 @@ func (req *ChtRequest) StoreResult(db ethdb.Database) {
hash, num := req.Header.Hash(), req.Header.Number.Uint64()
core.WriteTd(db, hash, num, req.Td)
core.WriteCanonicalHash(db, hash, num)
- //storeProof(db, req.Proof)
+}
+
+// BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure
+type BloomRequest struct {
+ OdrRequest
+ BloomTrieNum uint64
+ BitIdx uint
+ SectionIdxList []uint64
+ BloomTrieRoot common.Hash
+ BloomBits [][]byte
+ Proofs *NodeSet
+}
+
+// StoreResult stores the retrieved data in local database
+func (req *BloomRequest) StoreResult(db ethdb.Database) {
+ for i, sectionIdx := range req.SectionIdxList {
+ sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1)
+ // if we don't have the canonical hash stored for this section head number, we'll still store it under
+ // a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical
+ // hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the
+ // bit vector again from the network.
+ core.WriteBloomBits(db, req.BitIdx, sectionIdx, sectionHead, req.BloomBits[i])
+ }
}
diff --git a/light/odr_test.go b/light/odr_test.go
index c0c5438fd..e6afb1a48 100644
--- a/light/odr_test.go
+++ b/light/odr_test.go
@@ -77,7 +77,9 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
req.Receipts = core.GetBlockReceipts(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash))
case *TrieRequest:
t, _ := trie.New(req.Id.Root, odr.sdb)
- req.Proof = t.Prove(req.Key)
+ nodes := NewNodeSet()
+ t.Prove(req.Key, 0, nodes)
+ req.Proof = nodes
case *CodeRequest:
req.Data, _ = odr.sdb.Get(req.Hash[:])
}
diff --git a/light/odr_util.go b/light/odr_util.go
index fcdfdb82c..a0eb6303d 100644
--- a/light/odr_util.go
+++ b/light/odr_util.go
@@ -19,56 +19,16 @@ package light
import (
"bytes"
"context"
- "errors"
- "math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
- "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
)
var sha3_nil = crypto.Keccak256Hash(nil)
-var (
- ErrNoTrustedCht = errors.New("No trusted canonical hash trie")
- ErrNoHeader = errors.New("Header not found")
-
- ChtFrequency = uint64(4096)
- ChtConfirmations = uint64(2048)
- trustedChtKey = []byte("TrustedCHT")
-)
-
-type ChtNode struct {
- Hash common.Hash
- Td *big.Int
-}
-
-type TrustedCht struct {
- Number uint64
- Root common.Hash
-}
-
-func GetTrustedCht(db ethdb.Database) TrustedCht {
- data, _ := db.Get(trustedChtKey)
- var res TrustedCht
- if err := rlp.DecodeBytes(data, &res); err != nil {
- return TrustedCht{0, common.Hash{}}
- }
- return res
-}
-
-func WriteTrustedCht(db ethdb.Database, cht TrustedCht) {
- data, _ := rlp.EncodeToBytes(cht)
- db.Put(trustedChtKey, data)
-}
-
-func DeleteTrustedCht(db ethdb.Database) {
- db.Delete(trustedChtKey)
-}
-
func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) {
db := odr.Database()
hash := core.GetCanonicalHash(db, number)
@@ -81,12 +41,29 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ
return header, nil
}
- cht := GetTrustedCht(db)
- if number >= cht.Number*ChtFrequency {
+ var (
+ chtCount, sectionHeadNum uint64
+ sectionHead common.Hash
+ )
+ if odr.ChtIndexer() != nil {
+ chtCount, sectionHeadNum, sectionHead = odr.ChtIndexer().Sections()
+ canonicalHash := core.GetCanonicalHash(db, sectionHeadNum)
+ // if the CHT was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too
+ for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
+ chtCount--
+ if chtCount > 0 {
+ sectionHeadNum = chtCount*ChtFrequency - 1
+ sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1)
+ canonicalHash = core.GetCanonicalHash(db, sectionHeadNum)
+ }
+ }
+ }
+
+ if number >= chtCount*ChtFrequency {
return nil, ErrNoTrustedCht
}
- r := &ChtRequest{ChtRoot: cht.Root, ChtNum: cht.Number, BlockNum: number}
+ r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number}
if err := odr.Retrieve(ctx, r); err != nil {
return nil, err
} else {
@@ -162,3 +139,61 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num
}
return r.Receipts, nil
}
+
+// GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes
+func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxList []uint64) ([][]byte, error) {
+ db := odr.Database()
+ result := make([][]byte, len(sectionIdxList))
+ var (
+ reqList []uint64
+ reqIdx []int
+ )
+
+ var (
+ bloomTrieCount, sectionHeadNum uint64
+ sectionHead common.Hash
+ )
+ if odr.BloomTrieIndexer() != nil {
+ bloomTrieCount, sectionHeadNum, sectionHead = odr.BloomTrieIndexer().Sections()
+ canonicalHash := core.GetCanonicalHash(db, sectionHeadNum)
+ // if the BloomTrie was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too
+ for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
+ bloomTrieCount--
+ if bloomTrieCount > 0 {
+ sectionHeadNum = bloomTrieCount*BloomTrieFrequency - 1
+ sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1)
+ canonicalHash = core.GetCanonicalHash(db, sectionHeadNum)
+ }
+ }
+ }
+
+ for i, sectionIdx := range sectionIdxList {
+ sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1)
+ // if we don't have the canonical hash stored for this section head number, we'll still look for
+ // an entry with a zero sectionHead (we store it with zero section head too if we don't know it
+ // at the time of the retrieval)
+ bloomBits, err := core.GetBloomBits(db, bitIdx, sectionIdx, sectionHead)
+ if err == nil {
+ result[i] = bloomBits
+ } else {
+ if sectionIdx >= bloomTrieCount {
+ return nil, ErrNoTrustedBloomTrie
+ }
+ reqList = append(reqList, sectionIdx)
+ reqIdx = append(reqIdx, i)
+ }
+ }
+ if reqList == nil {
+ return result, nil
+ }
+
+ r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1, BitIdx: bitIdx, SectionIdxList: reqList}
+ if err := odr.Retrieve(ctx, r); err != nil {
+ return nil, err
+ } else {
+ for i, idx := range reqIdx {
+ result[idx] = r.BloomBits[i]
+ }
+ return result, nil
+ }
+}
diff --git a/light/postprocess.go b/light/postprocess.go
new file mode 100644
index 000000000..e7e513880
--- /dev/null
+++ b/light/postprocess.go
@@ -0,0 +1,295 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package light
+
+import (
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "math/big"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/bitutil"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+const (
+ ChtFrequency = 32768
+ ChtV1Frequency = 4096 // as long as we want to retain LES/1 compatibility, servers generate CHTs with the old, higher frequency
+ HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available
+ HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated
+)
+
+// trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with
+// the appropriate section index and head hash. It is used to start light syncing from this checkpoint
+// and avoid downloading the entire header chain while still being able to securely access old headers/logs.
+type trustedCheckpoint struct {
+ name string
+ sectionIdx uint64
+ sectionHead, chtRoot, bloomTrieRoot common.Hash
+}
+
+var (
+ mainnetCheckpoint = trustedCheckpoint{
+ name: "ETH mainnet",
+ sectionIdx: 129,
+ sectionHead: common.HexToHash("64100587c8ec9a76870056d07cb0f58622552d16de6253a59cac4b580c899501"),
+ chtRoot: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca"),
+ bloomTrieRoot: common.HexToHash("0db524b2c4a2a9520a42fd842b02d2e8fb58ff37c75cf57bd0eb82daeace6716"),
+ }
+
+ ropstenCheckpoint = trustedCheckpoint{
+ name: "Ropsten testnet",
+ sectionIdx: 50,
+ sectionHead: common.HexToHash("00bd65923a1aa67f85e6b4ae67835784dd54be165c37f056691723c55bf016bd"),
+ chtRoot: common.HexToHash("6f56dc61936752cc1f8c84b4addabdbe6a1c19693de3f21cb818362df2117f03"),
+ bloomTrieRoot: common.HexToHash("aca7d7c504d22737242effc3fdc604a762a0af9ced898036b5986c3a15220208"),
+ }
+)
+
+// trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to
+var trustedCheckpoints = map[common.Hash]trustedCheckpoint{
+ params.MainnetGenesisHash: mainnetCheckpoint,
+ params.TestnetGenesisHash: ropstenCheckpoint,
+}
+
+var (
+ ErrNoTrustedCht = errors.New("No trusted canonical hash trie")
+ ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie")
+ ErrNoHeader = errors.New("Header not found")
+ chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash
+ ChtTablePrefix = "cht-"
+)
+
+// ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format
+type ChtNode struct {
+ Hash common.Hash
+ Td *big.Int
+}
+
+// GetChtRoot reads the CHT root assoctiated to the given section from the database
+// Note that sectionIdx is specified according to LES/1 CHT section size
+func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash {
+ var encNumber [8]byte
+ binary.BigEndian.PutUint64(encNumber[:], sectionIdx)
+ data, _ := db.Get(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...))
+ return common.BytesToHash(data)
+}
+
+// GetChtV2Root reads the CHT root assoctiated to the given section from the database
+// Note that sectionIdx is specified according to LES/2 CHT section size
+func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash {
+ return GetChtRoot(db, (sectionIdx+1)*(ChtFrequency/ChtV1Frequency)-1, sectionHead)
+}
+
+// StoreChtRoot writes the CHT root assoctiated to the given section into the database
+// Note that sectionIdx is specified according to LES/1 CHT section size
+func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) {
+ var encNumber [8]byte
+ binary.BigEndian.PutUint64(encNumber[:], sectionIdx)
+ db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes())
+}
+
+// ChtIndexerBackend implements core.ChainIndexerBackend
+type ChtIndexerBackend struct {
+ db, cdb ethdb.Database
+ section, sectionSize uint64
+ lastHash common.Hash
+ trie *trie.Trie
+}
+
+// NewBloomTrieIndexer creates a BloomTrie chain indexer
+func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer {
+ cdb := ethdb.NewTable(db, ChtTablePrefix)
+ idb := ethdb.NewTable(db, "chtIndex-")
+ var sectionSize, confirmReq uint64
+ if clientMode {
+ sectionSize = ChtFrequency
+ confirmReq = HelperTrieConfirmations
+ } else {
+ sectionSize = ChtV1Frequency
+ confirmReq = HelperTrieProcessConfirmations
+ }
+ return core.NewChainIndexer(db, idb, &ChtIndexerBackend{db: db, cdb: cdb, sectionSize: sectionSize}, sectionSize, confirmReq, time.Millisecond*100, "cht")
+}
+
+// Reset implements core.ChainIndexerBackend
+func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error {
+ var root common.Hash
+ if section > 0 {
+ root = GetChtRoot(c.db, section-1, lastSectionHead)
+ }
+ var err error
+ c.trie, err = trie.New(root, c.cdb)
+ c.section = section
+ return err
+}
+
+// Process implements core.ChainIndexerBackend
+func (c *ChtIndexerBackend) Process(header *types.Header) {
+ hash, num := header.Hash(), header.Number.Uint64()
+ c.lastHash = hash
+
+ td := core.GetTd(c.db, hash, num)
+ if td == nil {
+ panic(nil)
+ }
+ var encNumber [8]byte
+ binary.BigEndian.PutUint64(encNumber[:], num)
+ data, _ := rlp.EncodeToBytes(ChtNode{hash, td})
+ c.trie.Update(encNumber[:], data)
+}
+
+// Commit implements core.ChainIndexerBackend
+func (c *ChtIndexerBackend) Commit() error {
+ batch := c.cdb.NewBatch()
+ root, err := c.trie.CommitTo(batch)
+ if err != nil {
+ return err
+ } else {
+ batch.Write()
+ if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 {
+ log.Info("Storing CHT", "idx", c.section*c.sectionSize/ChtFrequency, "sectionHead", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root))
+ }
+ StoreChtRoot(c.db, c.section, c.lastHash, root)
+ }
+ return nil
+}
+
+const (
+ BloomTrieFrequency = 32768
+ ethBloomBitsSection = 4096
+ ethBloomBitsConfirmations = 256
+)
+
+var (
+ bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash
+ BloomTrieTablePrefix = "blt-"
+)
+
+// GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database
+func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash {
+ var encNumber [8]byte
+ binary.BigEndian.PutUint64(encNumber[:], sectionIdx)
+ data, _ := db.Get(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...))
+ return common.BytesToHash(data)
+}
+
+// StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database
+func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) {
+ var encNumber [8]byte
+ binary.BigEndian.PutUint64(encNumber[:], sectionIdx)
+ db.Put(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes())
+}
+
+// BloomTrieIndexerBackend implements core.ChainIndexerBackend
+type BloomTrieIndexerBackend struct {
+ db, cdb ethdb.Database
+ section, parentSectionSize, bloomTrieRatio uint64
+ trie *trie.Trie
+ sectionHeads []common.Hash
+}
+
+// NewBloomTrieIndexer creates a BloomTrie chain indexer
+func NewBloomTrieIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer {
+ cdb := ethdb.NewTable(db, BloomTrieTablePrefix)
+ idb := ethdb.NewTable(db, "bltIndex-")
+ backend := &BloomTrieIndexerBackend{db: db, cdb: cdb}
+ var confirmReq uint64
+ if clientMode {
+ backend.parentSectionSize = BloomTrieFrequency
+ confirmReq = HelperTrieConfirmations
+ } else {
+ backend.parentSectionSize = ethBloomBitsSection
+ confirmReq = HelperTrieProcessConfirmations
+ }
+ backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize
+ backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio)
+ return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie")
+}
+
+// Reset implements core.ChainIndexerBackend
+func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error {
+ var root common.Hash
+ if section > 0 {
+ root = GetBloomTrieRoot(b.db, section-1, lastSectionHead)
+ }
+ var err error
+ b.trie, err = trie.New(root, b.cdb)
+ b.section = section
+ return err
+}
+
+// Process implements core.ChainIndexerBackend
+func (b *BloomTrieIndexerBackend) Process(header *types.Header) {
+ num := header.Number.Uint64() - b.section*BloomTrieFrequency
+ if (num+1)%b.parentSectionSize == 0 {
+ b.sectionHeads[num/b.parentSectionSize] = header.Hash()
+ }
+}
+
+// Commit implements core.ChainIndexerBackend
+func (b *BloomTrieIndexerBackend) Commit() error {
+ var compSize, decompSize uint64
+
+ for i := uint(0); i < types.BloomBitLength; i++ {
+ var encKey [10]byte
+ binary.BigEndian.PutUint16(encKey[0:2], uint16(i))
+ binary.BigEndian.PutUint64(encKey[2:10], b.section)
+ var decomp []byte
+ for j := uint64(0); j < b.bloomTrieRatio; j++ {
+ data, err := core.GetBloomBits(b.db, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j])
+ if err != nil {
+ return err
+ }
+ decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8))
+ if err2 != nil {
+ return err2
+ }
+ decomp = append(decomp, decompData...)
+ }
+ comp := bitutil.CompressBytes(decomp)
+
+ decompSize += uint64(len(decomp))
+ compSize += uint64(len(comp))
+ if len(comp) > 0 {
+ b.trie.Update(encKey[:], comp)
+ } else {
+ b.trie.Delete(encKey[:])
+ }
+ }
+
+ batch := b.cdb.NewBatch()
+ root, err := b.trie.CommitTo(batch)
+ if err != nil {
+ return err
+ } else {
+ batch.Write()
+ sectionHead := b.sectionHeads[b.bloomTrieRatio-1]
+ log.Info("Storing BloomTrie", "section", b.section, "sectionHead", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression ratio", float64(compSize)/float64(decompSize))
+ StoreBloomTrieRoot(b.db, b.section, sectionHead, root)
+ }
+
+ return nil
+}
--
cgit v1.2.3