From 054412e33528e53f6deae940c870217b614707b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 24 Sep 2018 15:57:49 +0300 Subject: all: clean up and proerly abstract database access --- trie/database.go | 17 ++++------------- trie/iterator_test.go | 17 ++++++++++++----- trie/proof.go | 6 +++--- trie/proof_test.go | 31 ++++++++++++++++++------------- trie/secure_trie_test.go | 6 +++--- trie/sync.go | 6 +++--- trie/sync_test.go | 22 +++++++++++----------- trie/trie_test.go | 20 +++++++++++--------- 8 files changed, 65 insertions(+), 60 deletions(-) (limited to 'trie') diff --git a/trie/database.go b/trie/database.go index 958823eb8..73ba2e761 100644 --- a/trie/database.go +++ b/trie/database.go @@ -55,20 +55,11 @@ var secureKeyPrefix = []byte("secure-key-") // secureKeyLength is the length of the above prefix + 32byte hash. const secureKeyLength = 11 + 32 -// DatabaseReader wraps the Get and Has method of a backing store for the trie. -type DatabaseReader interface { - // Get retrieves the value associated with key from the database. - Get(key []byte) (value []byte, err error) - - // Has retrieves whether a key is present in the database. - Has(key []byte) (bool, error) -} - // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. type Database struct { - diskdb ethdb.Database // Persistent storage for matured trie nodes + diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes @@ -271,14 +262,14 @@ func expandNode(hash hashNode, n node, cachegen uint16) node { // NewDatabase creates a new trie database to store ephemeral trie content before // its written out to disk or garbage collected. No read cache is created, so all // data retrievals will hit the underlying disk database. -func NewDatabase(diskdb ethdb.Database) *Database { +func NewDatabase(diskdb ethdb.KeyValueStore) *Database { return NewDatabaseWithCache(diskdb, 0) } // NewDatabaseWithCache creates a new trie database to store ephemeral trie content // before its written out to disk or garbage collected. It also acts as a read cache // for nodes loaded from disk. -func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { +func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database { var cleans *bigcache.BigCache if cache > 0 { cleans, _ = bigcache.NewBigCache(bigcache.Config{ @@ -298,7 +289,7 @@ func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { } // DiskDB retrieves the persistent storage backing the trie database. -func (db *Database) DiskDB() DatabaseReader { +func (db *Database) DiskDB() ethdb.Reader { return db.diskdb } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 4f633b195..88b8103fb 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -23,7 +23,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func TestIterator(t *testing.T) { @@ -120,11 +120,14 @@ func TestNodeIteratorCoverage(t *testing.T) { } } } - for _, key := range db.diskdb.(*ethdb.MemDatabase).Keys() { + it := db.diskdb.NewIterator() + for it.Next() { + key := it.Key() if _, ok := hashes[common.BytesToHash(key)]; !ok { t.Errorf("state entry not reported %x", key) } } + it.Release() } type kvs struct{ k, v string } @@ -289,7 +292,7 @@ func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueA func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) } func testIteratorContinueAfterError(t *testing.T, memonly bool) { - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) tr, _ := New(common.Hash{}, triedb) @@ -309,7 +312,11 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { if memonly { memKeys = triedb.Nodes() } else { - diskKeys = diskdb.Keys() + it := diskdb.NewIterator() + for it.Next() { + diskKeys = append(diskKeys, it.Key()) + } + it.Release() } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. @@ -376,7 +383,7 @@ func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) { func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { // Commit test trie to db, then remove the node containing "bars". - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) ctr, _ := New(common.Hash{}, triedb) diff --git a/trie/proof.go b/trie/proof.go index 1334bde97..0f18dd26b 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -34,7 +34,7 @@ import ( // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { +func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error { // Collect all nodes on the path to key. key = keybytesToHex(key) var nodes []node @@ -97,14 +97,14 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { +func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error { return t.trie.Prove(key, fromLevel, proofDb) } // VerifyProof checks merkle proofs. The given proof must contain the value for // key in a trie with the given root hash. VerifyProof returns an error if the // proof contains invalid trie nodes or the wrong value. -func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, nodes int, err error) { +func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.Reader) (value []byte, nodes int, err error) { key = keybytesToHex(key) wantHash := rootHash for i := 0; ; i++ { diff --git a/trie/proof_test.go b/trie/proof_test.go index 996f87478..bcb241bd7 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func init() { @@ -34,18 +34,18 @@ func init() { // makeProvers creates Merkle trie provers based on different implementations to // test all variations. -func makeProvers(trie *Trie) []func(key []byte) *ethdb.MemDatabase { - var provers []func(key []byte) *ethdb.MemDatabase +func makeProvers(trie *Trie) []func(key []byte) *memorydb.MemoryDatabase { + var provers []func(key []byte) *memorydb.MemoryDatabase // Create a direct trie based Merkle prover - provers = append(provers, func(key []byte) *ethdb.MemDatabase { - proof := ethdb.NewMemDatabase() + provers = append(provers, func(key []byte) *memorydb.MemoryDatabase { + proof := memorydb.New() trie.Prove(key, 0, proof) return proof }) // Create a leaf iterator based Merkle prover - provers = append(provers, func(key []byte) *ethdb.MemDatabase { - proof := ethdb.NewMemDatabase() + provers = append(provers, func(key []byte) *memorydb.MemoryDatabase { + proof := memorydb.New() if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { for _, p := range it.Prove() { proof.Put(crypto.Keccak256(p), p) @@ -106,9 +106,14 @@ func TestBadProof(t *testing.T) { if proof == nil { t.Fatalf("prover %d: nil proof", i) } - key := proof.Keys()[mrand.Intn(proof.Len())] + it := proof.NewIterator() + for i, d := 0, mrand.Intn(proof.Len()); i <= d; i++ { + it.Next() + } + key := it.Key() val, _ := proof.Get(key) proof.Delete(key) + it.Release() mutateByte(val) proof.Put(crypto.Keccak256(val), val) @@ -127,7 +132,7 @@ func TestMissingKeyProof(t *testing.T) { updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { - proof := ethdb.NewMemDatabase() + proof := memorydb.New() trie.Prove([]byte(key), 0, proof) if proof.Len() != 1 { @@ -164,8 +169,8 @@ func BenchmarkProve(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { kv := vals[keys[i%len(keys)]] - proofs := ethdb.NewMemDatabase() - if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 { + proofs := memorydb.New() + if trie.Prove(kv.k, 0, proofs); proofs.Len() == 0 { b.Fatalf("zero length proof for %x", kv.k) } } @@ -175,10 +180,10 @@ func BenchmarkVerifyProof(b *testing.B) { trie, vals := randomTrie(100) root := trie.Hash() var keys []string - var proofs []*ethdb.MemDatabase + var proofs []*memorydb.MemoryDatabase for k := range vals { keys = append(keys, k) - proof := ethdb.NewMemDatabase() + proof := memorydb.New() trie.Prove([]byte(k), 0, proof) proofs = append(proofs, proof) } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index d16d99968..f0ca6c800 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -24,18 +24,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0) + trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New()), 0) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(ethdb.NewMemDatabase()) + triedb := NewDatabase(memorydb.New()) trie, _ := NewSecure(common.Hash{}, triedb, 0) diff --git a/trie/sync.go b/trie/sync.go index 44f5087b9..ef931f633 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -72,14 +72,14 @@ func newSyncMemBatch() *syncMemBatch { // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. type Sync struct { - database DatabaseReader // Persistent database to check for existing entries + database ethdb.Reader // Persistent database to check for existing entries membatch *syncMemBatch // Memory buffer to avoid frequent database writes requests map[common.Hash]*request // Pending requests pertaining to a key hash queue *prque.Prque // Priority queue with the pending requests } // NewSync creates a new trie data download scheduler. -func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) *Sync { +func NewSync(root common.Hash, database ethdb.Reader, callback LeafCallback) *Sync { ts := &Sync{ database: database, membatch: newSyncMemBatch(), @@ -213,7 +213,7 @@ func (s *Sync) Process(results []SyncResult) (bool, int, error) { // Commit flushes the data stored in the internal membatch out to persistent // storage, returning the number of items written and any occurred error. -func (s *Sync) Commit(dbw ethdb.Putter) (int, error) { +func (s *Sync) Commit(dbw ethdb.Writer) (int, error) { // Dump the membatch into a database dbw for i, key := range s.membatch.order { if err := dbw.Put(key[:], s.membatch.batch[key]); err != nil { diff --git a/trie/sync_test.go b/trie/sync_test.go index ff15baa52..d80070f3e 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -21,13 +21,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) // makeTestTrie create a sample test trie to test node-wise reconstruction. func makeTestTrie() (*Database, *Trie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(ethdb.NewMemDatabase()) + triedb := NewDatabase(memorydb.New()) trie, _ := New(common.Hash{}, triedb) // Fill it with some arbitrary data @@ -88,13 +88,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error { // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { - dbA := NewDatabase(ethdb.NewMemDatabase()) - dbB := NewDatabase(ethdb.NewMemDatabase()) + dbA := NewDatabase(memorydb.New()) + dbB := NewDatabase(memorydb.New()) emptyA, _ := New(common.Hash{}, dbA) emptyB, _ := New(emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { - if req := NewSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { + if req := NewSync(trie.Hash(), memorydb.New(), nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -110,7 +110,7 @@ func testIterativeSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -143,7 +143,7 @@ func TestIterativeDelayedSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -181,7 +181,7 @@ func testIterativeRandomSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -222,7 +222,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -269,7 +269,7 @@ func TestDuplicateAvoidanceSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -309,7 +309,7 @@ func TestIncompleteSync(t *testing.T) { srcDb, srcTrie, _ := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) diff --git a/trie/trie_test.go b/trie/trie_test.go index 4d84aa96c..cf133706f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -33,6 +33,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,7 +45,7 @@ func init() { // Used for testing func newEmpty() *Trie { - trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + trie, _ := New(common.Hash{}, NewDatabase(memorydb.New())) return trie } @@ -67,7 +69,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase())) + trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -80,7 +82,7 @@ func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) } func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) } func testMissingNode(t *testing.T, memonly bool) { - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) trie, _ := New(common.Hash{}, triedb) @@ -317,13 +319,13 @@ func TestLargeValue(t *testing.T) { } type countingDB struct { - ethdb.Database + ethdb.KeyValueStore gets map[string]int } func (db *countingDB) Get(key []byte) ([]byte, error) { db.gets[string(key)]++ - return db.Database.Get(key) + return db.KeyValueStore.Get(key) } // TestCacheUnload checks that decoded nodes are unloaded after a @@ -342,7 +344,7 @@ func TestCacheUnload(t *testing.T) { // Commit the trie repeatedly and access key1. // The branch containing it is loaded from DB exactly two times: // in the 0th and 6th iteration. - diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} + diskdb := &countingDB{KeyValueStore: trie.db.diskdb, gets: make(map[string]int)} triedb := NewDatabase(diskdb) trie, _ = New(root, triedb) trie.SetCacheLimit(5) @@ -412,7 +414,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { } func runRandTest(rt randTest) bool { - triedb := NewDatabase(ethdb.NewMemDatabase()) + triedb := NewDatabase(memorydb.New()) tr, _ := New(common.Hash{}, triedb) values := make(map[string]string) // tracks content of the trie @@ -540,7 +542,7 @@ func benchGet(b *testing.B, commit bool) { b.StopTimer() if commit { - ldb := trie.db.diskdb.(*ethdb.LDBDatabase) + ldb := trie.db.diskdb.(*leveldb.LevelDBDatabase) ldb.Close() os.RemoveAll(ldb.Path()) } @@ -596,7 +598,7 @@ func tempDB() (string, *Database) { if err != nil { panic(fmt.Sprintf("can't create temporary directory: %v", err)) } - diskdb, err := ethdb.NewLDBDatabase(dir, 256, 0) + diskdb, err := leveldb.New(dir, 256, 0, "") if err != nil { panic(fmt.Sprintf("can't create temporary database: %v", err)) } -- cgit v1.2.3