diff options
94 files changed, 1586 insertions, 1394 deletions
diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index de81db134..7371dfb1f 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -66,7 +66,7 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { - database := ethdb.NewMemDatabase() + database := rawdb.NewMemoryDatabase() genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index c732c8b57..bc5d00cfb 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -31,10 +31,10 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/runtime" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" cli "gopkg.in/urfave/cli.v1" @@ -99,12 +99,12 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalString(GenesisFlag.Name) != "" { gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) genesisConfig = gen - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := gen.ToBlock(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db)) chainConfig = gen.Config } else { - statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) genesisConfig = new(core.Genesis) } if ctx.GlobalString(SenderFlag.Name) != "" { diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6d41261f8..f0e5d2619 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -29,14 +29,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" - "github.com/syndtr/goleveldb/leveldb/util" "gopkg.in/urfave/cli.v1" ) @@ -193,7 +192,7 @@ func initGenesis(ctx *cli.Context) error { defer stack.Close() for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabase(name, 0, 0) + chaindb, err := stack.OpenDatabase(name, 0, 0, "") if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -201,6 +200,7 @@ func initGenesis(ctx *cli.Context) error { if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } + chaindb.Close() log.Info("Successfully wrote genesis state", "database", name, "hash", hash) } return nil @@ -213,8 +213,8 @@ func importChain(ctx *cli.Context) error { stack := makeFullNode(ctx) defer stack.Close() - chain, chainDb := utils.MakeChain(ctx, stack) - defer chainDb.Close() + chain, db := utils.MakeChain(ctx, stack) + defer db.Close() // Start periodically gathering memory profiles var peakMemAlloc, peakMemSys uint64 @@ -249,15 +249,13 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Import done in %v.\n\n", time.Since(start)) // Output pre-compaction stats mostly to see the import trashing - db := chainDb.(*ethdb.LDBDatabase) - - stats, err := db.LDB().GetProperty("leveldb.stats") + stats, err := db.Stat("leveldb.stats") if err != nil { utils.Fatalf("Failed to read database stats: %v", err) } fmt.Println(stats) - ioStats, err := db.LDB().GetProperty("leveldb.iostats") + ioStats, err := db.Stat("leveldb.iostats") if err != nil { utils.Fatalf("Failed to read database iostats: %v", err) } @@ -282,23 +280,22 @@ func importChain(ctx *cli.Context) error { // Compact the entire database to more accurately measure disk io and print the stats start = time.Now() fmt.Println("Compacting entire database...") - if err = db.LDB().CompactRange(util.Range{}); err != nil { + if err = db.Compact(nil, nil); err != nil { utils.Fatalf("Compaction failed: %v", err) } fmt.Printf("Compaction done in %v.\n\n", time.Since(start)) - stats, err = db.LDB().GetProperty("leveldb.stats") + stats, err = db.Stat("leveldb.stats") if err != nil { utils.Fatalf("Failed to read database stats: %v", err) } fmt.Println(stats) - ioStats, err = db.LDB().GetProperty("leveldb.iostats") + ioStats, err = db.Stat("leveldb.iostats") if err != nil { utils.Fatalf("Failed to read database iostats: %v", err) } fmt.Println(ioStats) - return nil } @@ -344,10 +341,10 @@ func importPreimages(ctx *cli.Context) error { stack := makeFullNode(ctx) defer stack.Close() - diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase) + db := utils.MakeChainDatabase(ctx, stack) start := time.Now() - if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil { + if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil { utils.Fatalf("Import error: %v\n", err) } fmt.Printf("Import done in %v\n", time.Since(start)) @@ -362,10 +359,10 @@ func exportPreimages(ctx *cli.Context) error { stack := makeFullNode(ctx) defer stack.Close() - diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase) + db := utils.MakeChainDatabase(ctx, stack) start := time.Now() - if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil { + if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil { utils.Fatalf("Export error: %v\n", err) } fmt.Printf("Export done in %v\n", time.Since(start)) @@ -386,7 +383,7 @@ func copyDb(ctx *cli.Context) error { dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil) // Create a source peer to satisfy downloader requests from - db, err := ethdb.NewLDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256) + db, err := rawdb.NewLevelDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256, "") if err != nil { return err } @@ -413,11 +410,10 @@ func copyDb(ctx *cli.Context) error { // Compact the entire database to remove any sync overhead start = time.Now() fmt.Println("Compacting entire database...") - if err = chainDb.(*ethdb.LDBDatabase).LDB().CompactRange(util.Range{}); err != nil { + if err = db.Compact(nil, nil); err != nil { utils.Fatalf("Compaction failed: %v", err) } fmt.Printf("Compaction done in %v.\n\n", time.Since(start)) - return nil } diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 52983ff2a..cb06038ec 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -121,7 +120,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc } // Retrieve the DAO config flag from the database path := filepath.Join(datadir, "geth", "chaindata") - db, err := ethdb.NewLDBDatabase(path, 0, 0) + db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "") if err != nil { t.Fatalf("test %d: failed to open test database: %v", test, err) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index f23aa5775..74a8c7f39 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -238,7 +238,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las } // ImportPreimages imports a batch of exported hash preimages into the database. -func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { +func ImportPreimages(db ethdb.Database, fn string) error { log.Info("Importing preimages", "file", fn) // Open the file handle and potentially unwrap the gzip stream @@ -285,7 +285,7 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { // ExportPreimages exports all known hash preimages into the specified file, // truncating any data already present in the file. -func ExportPreimages(db *ethdb.LDBDatabase, fn string) error { +func ExportPreimages(db ethdb.Database, fn string) error { log.Info("Exporting preimages", "file", fn) // Open the file handle and potentially wrap with a gzip stream diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4db59097d..deb6df364 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1548,7 +1548,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { if ctx.GlobalString(SyncModeFlag.Name) == "light" { name = "lightchaindata" } - chainDb, err := stack.OpenDatabase(name, cache, handles) + chainDb, err := stack.OpenDatabase(name, cache, handles, "") if err != nil { Fatalf("Could not open database: %v", err) } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index b920312a8..47c25aa04 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -24,10 +24,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -400,7 +400,7 @@ func TestClique(t *testing.T) { copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:]) } // Create a pristine blockchain with the genesis injected - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis.Commit(db) // Assemble a chain of headers from the cast votes diff --git a/core/bench_test.go b/core/bench_test.go index 53cba0517..e0ccef788 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -150,14 +150,14 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Create the database in memory or in a temporary directory. var db ethdb.Database if !disk { - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() } else { dir, err := ioutil.TempDir("", "eth-core-bench") if err != nil { b.Fatalf("cannot create temporary directory: %v", err) } defer os.RemoveAll(dir) - db, err = ethdb.NewLDBDatabase(dir, 128, 128) + db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "") if err != nil { b.Fatalf("cannot create temporary database: %v", err) } @@ -255,7 +255,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("cannot create temporary directory: %v", err) } - db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } @@ -272,7 +272,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { } defer os.RemoveAll(dir) - db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } @@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { b.ResetTimer() for i := 0; i < b.N; i++ { - db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 9319a7835..06e2ba1a4 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -22,9 +22,9 @@ import ( "time" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -32,7 +32,7 @@ import ( func TestHeaderVerification(t *testing.T) { // Create a simple chain to verify var ( - testdb = ethdb.NewMemDatabase() + testdb = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -84,7 +84,7 @@ func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVeri func testHeaderConcurrentVerification(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb = ethdb.NewMemDatabase() + testdb = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -156,7 +156,7 @@ func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion func testHeaderConcurrentAbortion(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb = ethdb.NewMemDatabase() + testdb = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil) diff --git a/core/blockchain.go b/core/blockchain.go index 7b4f4b303..71e806e6e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -43,7 +43,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" ) var ( @@ -291,7 +291,7 @@ func (bc *BlockChain) SetHead(head uint64) error { defer bc.chainmu.Unlock() // Rewind the header chain, deleting all block bodies until then - delFn := func(db rawdb.DatabaseDeleter, hash common.Hash, num uint64) { + delFn := func(db ethdb.Deleter, hash common.Hash, num uint64) { rawdb.DeleteBody(db, hash, num) } bc.hc.SetHead(head, delFn) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 1ac7b03fc..c9e999cc9 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -46,7 +46,7 @@ var ( // header only chain. func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { var ( - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() genesis = new(Genesis).MustCommit(db) ) @@ -586,7 +586,7 @@ func testInsertNonceError(t *testing.T, full bool) { func TestFastVsFullChains(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = ethdb.NewMemDatabase() + gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -616,7 +616,7 @@ func TestFastVsFullChains(t *testing.T) { } }) // Import the chain as an archive node for the comparison baseline - archiveDb := ethdb.NewMemDatabase() + archiveDb := rawdb.NewMemoryDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil) defer archive.Stop() @@ -625,7 +625,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to process block %d: %v", n, err) } // Fast import the chain as a non-archive node to test - fastDb := ethdb.NewMemDatabase() + fastDb := rawdb.NewMemoryDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil) defer fast.Stop() @@ -674,7 +674,7 @@ func TestFastVsFullChains(t *testing.T) { func TestLightVsFastVsFullChainHeads(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = ethdb.NewMemDatabase() + gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -702,7 +702,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { } } // Import the chain as an archive node and ensure all pointers are updated - archiveDb := ethdb.NewMemDatabase() + archiveDb := rawdb.NewMemoryDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil) @@ -716,7 +716,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "archive", archive, height/2, height/2, height/2) // Import the chain as a non-archive node and ensure all pointers are updated - fastDb := ethdb.NewMemDatabase() + fastDb := rawdb.NewMemoryDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil) defer fast.Stop() @@ -736,7 +736,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "fast", fast, height/2, height/2, 0) // Import the chain as a light node and ensure all pointers are updated - lightDb := ethdb.NewMemDatabase() + lightDb := rawdb.NewMemoryDatabase() gspec.MustCommit(lightDb) light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil) @@ -759,7 +759,7 @@ func TestChainTxReorgs(t *testing.T) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, @@ -871,7 +871,7 @@ func TestLogReorgs(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} @@ -915,7 +915,7 @@ func TestLogReorgs(t *testing.T) { func TestReorgSideEvent(t *testing.T) { var ( - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ @@ -1043,7 +1043,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { func TestEIP155Transition(t *testing.T) { // Configure and generate a sample block chain var ( - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1146,7 +1146,7 @@ func TestEIP155Transition(t *testing.T) { func TestEIP161AccountRemoval(t *testing.T) { // Configure and generate a sample block chain var ( - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1218,7 +1218,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1234,7 +1234,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - diskdb := ethdb.NewMemDatabase() + diskdb := rawdb.NewMemoryDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) @@ -1263,7 +1263,7 @@ func TestTrieForkGC(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1278,7 +1278,7 @@ func TestTrieForkGC(t *testing.T) { forks[i] = fork[0] } // Import the canonical and fork chain side by side, forcing the trie cache to cache both - diskdb := ethdb.NewMemDatabase() + diskdb := rawdb.NewMemoryDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) @@ -1309,7 +1309,7 @@ func TestLargeReorgTrieGC(t *testing.T) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := new(Genesis).MustCommit(db) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1317,7 +1317,7 @@ func TestLargeReorgTrieGC(t *testing.T) { competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - diskdb := ethdb.NewMemDatabase() + diskdb := rawdb.NewMemoryDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) @@ -1377,7 +1377,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in ) // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(db) blockGenerator := func(i int, block *BlockGen) { @@ -1399,7 +1399,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.ResetTimer() for i := 0; i < b.N; i++ { // Import the shared chain and the original canonical one - diskdb := ethdb.NewMemDatabase() + diskdb := rawdb.NewMemoryDatabase() gspec.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) @@ -1477,7 +1477,7 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { func TestLowDiffLongChain(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := new(Genesis).MustCommit(db) // We must use a pretty long chain to ensure that the fork doesn't overtake us @@ -1488,7 +1488,7 @@ func TestLowDiffLongChain(t *testing.T) { }) // Import the canonical chain - diskdb := ethdb.NewMemDatabase() + diskdb := rawdb.NewMemoryDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) @@ -1531,12 +1531,12 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() genesis := new(Genesis).MustCommit(db) // Generate and import the canonical chain blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, nil) - diskdb := ethdb.NewMemDatabase() + diskdb := rawdb.NewMemoryDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) if err != nil { diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 1adde1fcb..26538260c 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -97,7 +97,7 @@ type ChainIndexer struct { // NewChainIndexer creates a new chain indexer to do background processing on // chain segments of a given size after certain number of confirmations passed. // The throttling parameter might be used to prevent database thrashing. -func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer { +func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer { c := &ChainIndexer{ chainDb: chainDb, indexDb: indexDb, diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index a029dec62..abf5b3cc1 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" ) // Runs multiple tests with randomized parameters. @@ -49,7 +48,7 @@ func TestChainIndexerWithChildren(t *testing.T) { // multiple backends. The section size and required confirmation count parameters // are randomized. func testChainIndexer(t *testing.T, count int) { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() defer db.Close() // Create a chain of indexers and ensure they all report empty @@ -60,7 +59,7 @@ func testChainIndexer(t *testing.T, count int) { confirmsReq = uint64(rand.Intn(10)) ) backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)} - backends[i].indexer = NewChainIndexer(db, ethdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i)) + backends[i].indexer = NewChainIndexer(db, rawdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i)) if sections, _, _ := backends[i].indexer.Sections(); sections != 0 { t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0) diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 64b64fd6a..32e3888d5 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -21,10 +21,10 @@ import ( "math/big" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -36,7 +36,7 @@ func ExampleGenerateChain() { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() ) // Ensure that key1 has some funds in the genesis block. diff --git a/core/dao_test.go b/core/dao_test.go index 966139bce..4e8dba9e8 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -21,8 +21,8 @@ import ( "testing" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -32,13 +32,13 @@ func TestDAOForkRangeExtradata(t *testing.T) { forkBlock := big.NewInt(32) // Generate a common prefix for both pro-forkers and non-forkers - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() gspec := new(Genesis) genesis := gspec.MustCommit(db) prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) // Create the concurrent, conflicting two nodes - proDb := ethdb.NewMemDatabase() + proDb := rawdb.NewMemoryDatabase() gspec.MustCommit(proDb) proConf := *params.TestChainConfig @@ -48,7 +48,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil) defer proBc.Stop() - conDb := ethdb.NewMemDatabase() + conDb := rawdb.NewMemoryDatabase() gspec.MustCommit(conDb) conConf := *params.TestChainConfig @@ -67,7 +67,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ { // Create a pro-fork block, and try to feed into the no-fork chain - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil) defer bc.Stop() @@ -92,7 +92,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil) defer bc.Stop() @@ -118,7 +118,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { } } // Verify that contra-forkers accept pro-fork extra-datas after forking finishes - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil) defer bc.Stop() @@ -138,7 +138,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil) defer bc.Stop() diff --git a/core/genesis.go b/core/genesis.go index cbb6eecd2..4aa129966 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -134,7 +134,7 @@ type GenesisMismatchError struct { } func (e *GenesisMismatchError) Error() string { - return fmt.Sprintf("database already contains an incompatible genesis block (have %x, new %x)", e.Stored[:8], e.New[:8]) + return fmt.Sprintf("database contains incompatible genesis (have %x, new %x)", e.Stored, e.New) } // SetupGenesisBlock writes or updates the genesis block in db. @@ -228,7 +228,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // to the given database (or discards it if nil). func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() } statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) for addr, account := range g.Alloc { diff --git a/core/genesis_test.go b/core/genesis_test.go index c7d54f205..c6bcd0aa5 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -141,7 +141,7 @@ func TestSetupGenesis(t *testing.T) { } for _, test := range tests { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() config, hash, err := test.fn(db) // Check the return values. if !reflect.DeepEqual(err, test.wantErr) { diff --git a/core/headerchain.go b/core/headerchain.go index 8904dd887..027cb798f 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" ) const ( @@ -455,7 +455,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { // DeleteCallback is a callback function that is called by SetHead before // each header is deleted. -type DeleteCallback func(rawdb.DatabaseDeleter, common.Hash, uint64) +type DeleteCallback func(ethdb.Deleter, common.Hash, uint64) // SetHead rewinds the local chain to a new head. Everything above the new head // will be deleted and the new one set. diff --git a/core/helper_test.go b/core/helper_test.go index 051384d85..e61c92dcd 100644 --- a/core/helper_test.go +++ b/core/helper_test.go @@ -19,6 +19,7 @@ package core import ( "container/list" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -78,7 +79,7 @@ func (tm *TestManager) Db() ethdb.Database { func NewTestManager() *TestManager { testManager := &TestManager{} testManager.eventMux = new(event.TypeMux) - testManager.db = ethdb.NewMemDatabase() + testManager.db = rawdb.NewMemoryDatabase() // testManager.txPool = NewTxPool(testManager) // testManager.blockChain = NewBlockChain(testManager) // testManager.stateManager = NewStateManager(testManager) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 8a95dafe9..ea923f9d1 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -23,12 +23,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) // ReadCanonicalHash retrieves the hash assigned to a canonical block number. -func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { +func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { data, _ := db.Get(headerHashKey(number)) if len(data) == 0 { return common.Hash{} @@ -37,21 +38,21 @@ func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { } // WriteCanonicalHash stores the hash assigned to a canonical block number. -func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) { +func WriteCanonicalHash(db ethdb.Writer, hash common.Hash, number uint64) { if err := db.Put(headerHashKey(number), hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) } } // DeleteCanonicalHash removes the number to hash canonical mapping. -func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { +func DeleteCanonicalHash(db ethdb.Deleter, number uint64) { if err := db.Delete(headerHashKey(number)); err != nil { log.Crit("Failed to delete number to hash mapping", "err", err) } } // ReadHeaderNumber returns the header number assigned to a hash. -func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { +func ReadHeaderNumber(db ethdb.Reader, hash common.Hash) *uint64 { data, _ := db.Get(headerNumberKey(hash)) if len(data) != 8 { return nil @@ -61,7 +62,7 @@ func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { } // ReadHeadHeaderHash retrieves the hash of the current canonical head header. -func ReadHeadHeaderHash(db DatabaseReader) common.Hash { +func ReadHeadHeaderHash(db ethdb.Reader) common.Hash { data, _ := db.Get(headHeaderKey) if len(data) == 0 { return common.Hash{} @@ -70,14 +71,14 @@ func ReadHeadHeaderHash(db DatabaseReader) common.Hash { } // WriteHeadHeaderHash stores the hash of the current canonical head header. -func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) { +func WriteHeadHeaderHash(db ethdb.Writer, hash common.Hash) { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { log.Crit("Failed to store last header's hash", "err", err) } } // ReadHeadBlockHash retrieves the hash of the current canonical head block. -func ReadHeadBlockHash(db DatabaseReader) common.Hash { +func ReadHeadBlockHash(db ethdb.Reader) common.Hash { data, _ := db.Get(headBlockKey) if len(data) == 0 { return common.Hash{} @@ -86,14 +87,14 @@ func ReadHeadBlockHash(db DatabaseReader) common.Hash { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) { +func WriteHeadBlockHash(db ethdb.Writer, hash common.Hash) { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last block's hash", "err", err) } } // ReadHeadFastBlockHash retrieves the hash of the current fast-sync head block. -func ReadHeadFastBlockHash(db DatabaseReader) common.Hash { +func ReadHeadFastBlockHash(db ethdb.Reader) common.Hash { data, _ := db.Get(headFastBlockKey) if len(data) == 0 { return common.Hash{} @@ -102,7 +103,7 @@ func ReadHeadFastBlockHash(db DatabaseReader) common.Hash { } // WriteHeadFastBlockHash stores the hash of the current fast-sync head block. -func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) { +func WriteHeadFastBlockHash(db ethdb.Writer, hash common.Hash) { if err := db.Put(headFastBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last fast block's hash", "err", err) } @@ -110,7 +111,7 @@ func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) { // ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow // reporting correct numbers across restarts. -func ReadFastTrieProgress(db DatabaseReader) uint64 { +func ReadFastTrieProgress(db ethdb.Reader) uint64 { data, _ := db.Get(fastTrieProgressKey) if len(data) == 0 { return 0 @@ -120,20 +121,20 @@ func ReadFastTrieProgress(db DatabaseReader) uint64 { // WriteFastTrieProgress stores the fast sync trie process counter to support // retrieving it across restarts. -func WriteFastTrieProgress(db DatabaseWriter, count uint64) { +func WriteFastTrieProgress(db ethdb.Writer, count uint64) { if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil { log.Crit("Failed to store fast sync trie progress", "err", err) } } // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. -func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { +func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(headerKey(number, hash)) return data } // HasHeader verifies the existence of a block header corresponding to the hash. -func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { +func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Has(headerKey(number, hash)); !has || err != nil { return false } @@ -141,7 +142,7 @@ func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { } // ReadHeader retrieves the block header corresponding to the hash. -func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header { +func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header { data := ReadHeaderRLP(db, hash, number) if len(data) == 0 { return nil @@ -156,7 +157,7 @@ func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Heade // WriteHeader stores a block header into the database and also stores the hash- // to-number mapping. -func WriteHeader(db DatabaseWriter, header *types.Header) { +func WriteHeader(db ethdb.Writer, header *types.Header) { // Write the hash -> number mapping var ( hash = header.Hash() @@ -179,30 +180,36 @@ func WriteHeader(db DatabaseWriter, header *types.Header) { } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(headerKey(number, hash)); err != nil { - log.Crit("Failed to delete header", "err", err) - } +func DeleteHeader(db ethdb.Deleter, hash common.Hash, number uint64) { + deleteHeaderWithoutNumber(db, hash, number) if err := db.Delete(headerNumberKey(hash)); err != nil { log.Crit("Failed to delete hash to number mapping", "err", err) } } +// deleteHeaderWithoutNumber removes only the block header but does not remove +// the hash to number mapping. +func deleteHeaderWithoutNumber(db ethdb.Deleter, hash common.Hash, number uint64) { + if err := db.Delete(headerKey(number, hash)); err != nil { + log.Crit("Failed to delete header", "err", err) + } +} + // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { +func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(blockBodyKey(number, hash)) return data } // WriteBodyRLP stores an RLP encoded block body into the database. -func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) { +func WriteBodyRLP(db ethdb.Writer, hash common.Hash, number uint64, rlp rlp.RawValue) { if err := db.Put(blockBodyKey(number, hash), rlp); err != nil { log.Crit("Failed to store block body", "err", err) } } // HasBody verifies the existence of a block body corresponding to the hash. -func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { +func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false } @@ -210,7 +217,7 @@ func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { } // ReadBody retrieves the block body corresponding to the hash. -func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { +func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body { data := ReadBodyRLP(db, hash, number) if len(data) == 0 { return nil @@ -224,7 +231,7 @@ func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { } // WriteBody storea a block body into the database. -func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) { +func WriteBody(db ethdb.Writer, hash common.Hash, number uint64, body *types.Body) { data, err := rlp.EncodeToBytes(body) if err != nil { log.Crit("Failed to RLP encode body", "err", err) @@ -233,15 +240,21 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { +func DeleteBody(db ethdb.Deleter, hash common.Hash, number uint64) { if err := db.Delete(blockBodyKey(number, hash)); err != nil { log.Crit("Failed to delete block body", "err", err) } } -// ReadTd retrieves a block's total difficulty corresponding to the hash. -func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { +// ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding. +func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(headerTDKey(number, hash)) + return data +} + +// ReadTd retrieves a block's total difficulty corresponding to the hash. +func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int { + data := ReadTdRLP(db, hash, number) if len(data) == 0 { return nil } @@ -254,7 +267,7 @@ func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { } // WriteTd stores the total difficulty of a block into the database. -func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) { +func WriteTd(db ethdb.Writer, hash common.Hash, number uint64, td *big.Int) { data, err := rlp.EncodeToBytes(td) if err != nil { log.Crit("Failed to RLP encode block total difficulty", "err", err) @@ -265,7 +278,7 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) { } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { +func DeleteTd(db ethdb.Deleter, hash common.Hash, number uint64) { if err := db.Delete(headerTDKey(number, hash)); err != nil { log.Crit("Failed to delete block total difficulty", "err", err) } @@ -273,17 +286,23 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { // HasReceipts verifies the existence of all the transaction receipts belonging // to a block. -func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool { +func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { return false } return true } +// ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. +func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(blockReceiptsKey(number, hash)) + return data +} + // ReadReceipts retrieves all the transaction receipts belonging to a block. -func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { +func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice - data, _ := db.Get(blockReceiptsKey(number, hash)) + data := ReadReceiptsRLP(db, hash, number) if len(data) == 0 { return nil } @@ -311,7 +330,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece } // WriteReceipts stores all the transaction receipts belonging to a block. -func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) { +func WriteReceipts(db ethdb.Writer, hash common.Hash, number uint64, receipts types.Receipts) { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -328,7 +347,7 @@ func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts } // DeleteReceipts removes all receipt data associated with a block hash. -func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { +func DeleteReceipts(db ethdb.Deleter, hash common.Hash, number uint64) { if err := db.Delete(blockReceiptsKey(number, hash)); err != nil { log.Crit("Failed to delete block receipts", "err", err) } @@ -340,7 +359,7 @@ func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { // // Note, due to concurrent download of header and block body the header and thus // canonical hash can be stored in the database but the body data not (yet). -func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { +func ReadBlock(db ethdb.Reader, hash common.Hash, number uint64) *types.Block { header := ReadHeader(db, hash, number) if header == nil { return nil @@ -353,21 +372,30 @@ func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db DatabaseWriter, block *types.Block) { +func WriteBlock(db ethdb.Writer, block *types.Block) { WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) WriteHeader(db, block.Header()) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) { +func DeleteBlock(db ethdb.Deleter, hash common.Hash, number uint64) { DeleteReceipts(db, hash, number) DeleteHeader(db, hash, number) DeleteBody(db, hash, number) DeleteTd(db, hash, number) } +// deleteBlockWithoutNumber removes all block data associated with a hash, except +// the hash to number mapping. +func deleteBlockWithoutNumber(db ethdb.Deleter, hash common.Hash, number uint64) { + DeleteReceipts(db, hash, number) + deleteHeaderWithoutNumber(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) +} + // FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header { +func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header { for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) if a == nil { diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 37e0d4fda..9f6e9cdb3 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -23,14 +23,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) // Tests block header storage and retrieval operations. func TestHeaderStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() // Create a test header to move around the database and make sure it's really new header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} @@ -63,7 +62,7 @@ func TestHeaderStorage(t *testing.T) { // Tests block body storage and retrieval operations. func TestBodyStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() // Create a test body to move around the database and make sure it's really new body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} @@ -101,7 +100,7 @@ func TestBodyStorage(t *testing.T) { // Tests block storage and retrieval operations. func TestBlockStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(&types.Header{ @@ -151,7 +150,7 @@ func TestBlockStorage(t *testing.T) { // Tests that partial block contents don't get reassembled into full blocks. func TestPartialBlockStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() block := types.NewBlockWithHeader(&types.Header{ Extra: []byte("test block"), UncleHash: types.EmptyUncleHash, @@ -185,7 +184,7 @@ func TestPartialBlockStorage(t *testing.T) { // Tests block total difficulty storage and retrieval operations. func TestTdStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) @@ -208,7 +207,7 @@ func TestTdStorage(t *testing.T) { // Tests that canonical numbers can be mapped to hashes and retrieved. func TestCanonicalMappingStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() // Create a test canonical number and assinged hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) @@ -231,7 +230,7 @@ func TestCanonicalMappingStorage(t *testing.T) { // Tests that head headers and head blocks can be assigned, individually. func TestHeadStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")}) blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")}) @@ -266,7 +265,7 @@ func TestHeadStorage(t *testing.T) { // Tests that receipts associated with a single block can be stored and retrieved. func TestBlockReceiptStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() receipt1 := &types.Receipt{ Status: types.ReceiptStatusFailed, diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index e6f7782a1..d90a43012 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -19,13 +19,14 @@ package rawdb import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) // ReadTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. -func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) common.Hash { +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) common.Hash { data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { return common.Hash{} @@ -44,7 +45,7 @@ func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) common.Hash { // WriteTxLookupEntries stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. -func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { +func WriteTxLookupEntries(db ethdb.Writer, block *types.Block) { for _, tx := range block.Transactions() { if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil { log.Crit("Failed to store transaction lookup entry", "err", err) @@ -53,13 +54,13 @@ func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { } // DeleteTxLookupEntry removes all transaction data associated with a hash. -func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { +func DeleteTxLookupEntry(db ethdb.Deleter, hash common.Hash) { db.Delete(txLookupKey(hash)) } // ReadTransaction retrieves a specific transaction from the database, along with // its added positional metadata. -func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { +func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { blockHash := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 @@ -84,7 +85,7 @@ func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, c // ReadReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. -func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { +func ReadReceipt(db ethdb.Reader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { blockHash := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 @@ -105,13 +106,13 @@ func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Ha // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given // section and bit index from the. -func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { +func ReadBloomBits(db ethdb.Reader, bit uint, section uint64, head common.Hash) ([]byte, error) { return db.Get(bloomBitsKey(bit, section, head)) } // WriteBloomBits stores the compressed bloom bits vector belonging to the given // section and bit index. -func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { +func WriteBloomBits(db ethdb.Writer, bit uint, section uint64, head common.Hash, bits []byte) { if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil { log.Crit("Failed to store bloom bits", "err", err) } diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index bed03a5e6..ca74ba6af 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -22,13 +22,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" ) // Tests that positional lookup metadata can be stored and retrieved. func TestLookupStorage(t *testing.T) { - db := ethdb.NewMemDatabase() + db := NewMemoryDatabase() tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 82e4bf045..1361b0d73 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -20,13 +20,14 @@ import ( "encoding/json" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) // ReadDatabaseVersion retrieves the version number of the database. -func ReadDatabaseVersion(db DatabaseReader) *uint64 { +func ReadDatabaseVersion(db ethdb.Reader) *uint64 { var version uint64 enc, _ := db.Get(databaseVerisionKey) @@ -41,7 +42,7 @@ func ReadDatabaseVersion(db DatabaseReader) *uint64 { } // WriteDatabaseVersion stores the version number of the database -func WriteDatabaseVersion(db DatabaseWriter, version uint64) { +func WriteDatabaseVersion(db ethdb.Writer, version uint64) { enc, err := rlp.EncodeToBytes(version) if err != nil { log.Crit("Failed to encode database version", "err", err) @@ -52,7 +53,7 @@ func WriteDatabaseVersion(db DatabaseWriter, version uint64) { } // ReadChainConfig retrieves the consensus settings based on the given genesis hash. -func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { +func ReadChainConfig(db ethdb.Reader, hash common.Hash) *params.ChainConfig { data, _ := db.Get(configKey(hash)) if len(data) == 0 { return nil @@ -66,7 +67,7 @@ func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) { +func WriteChainConfig(db ethdb.Writer, hash common.Hash, cfg *params.ChainConfig) { if cfg == nil { return } @@ -80,13 +81,13 @@ func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConf } // ReadPreimage retrieves a single preimage of the provided hash. -func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { +func ReadPreimage(db ethdb.Reader, hash common.Hash) []byte { data, _ := db.Get(preimageKey(hash)) return data } // WritePreimages writes the provided set of preimages to the database. -func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) { +func WritePreimages(db ethdb.Writer, preimages map[common.Hash][]byte) { for hash, preimage := range preimages { if err := db.Put(preimageKey(hash), preimage); err != nil { log.Crit("Failed to store trie preimage", "err", err) diff --git a/core/rawdb/database.go b/core/rawdb/database.go new file mode 100644 index 000000000..b4c5dea70 --- /dev/null +++ b/core/rawdb/database.go @@ -0,0 +1,52 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +// NewDatabase creates a high level database on top of a given key-value data +// store without a freezer moving immutable chain segments into cold storage. +func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { + return db +} + +// NewMemoryDatabase creates an ephemeral in-memory key-value database without a +// freezer moving immutable chain segments into cold storage. +func NewMemoryDatabase() ethdb.Database { + return NewDatabase(memorydb.New()) +} + +// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database with +// an initial starting capacity, but without a freezer moving immutable chain +// segments into cold storage. +func NewMemoryDatabaseWithCap(size int) ethdb.Database { + return NewDatabase(memorydb.NewWithCap(size)) +} + +// NewLevelDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) { + db, err := leveldb.New(file, cache, handles, namespace) + if err != nil { + return nil, err + } + return NewDatabase(db), nil +} diff --git a/core/rawdb/interfaces.go b/core/rawdb/interfaces.go deleted file mode 100644 index 3bdf55124..000000000 --- a/core/rawdb/interfaces.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package rawdb - -// DatabaseReader wraps the Has and Get method of a backing data store. -type DatabaseReader interface { - Has(key []byte) (bool, error) - Get(key []byte) ([]byte, error) -} - -// DatabaseWriter wraps the Put method of a backing data store. -type DatabaseWriter interface { - Put(key []byte, value []byte) error -} - -// DatabaseDeleter wraps the Delete method of a backing data store. -type DatabaseDeleter interface { - Delete(key []byte) error -} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3bb86e7ff..87dbf94fc 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -78,6 +78,11 @@ func encodeBlockNumber(number uint64) []byte { return enc } +// headerKeyPrefix = headerPrefix + num (uint64 big endian) +func headerKeyPrefix(number uint64) []byte { + return append(headerPrefix, encodeBlockNumber(number)...) +} + // headerKey = headerPrefix + num (uint64 big endian) + hash func headerKey(number uint64, hash common.Hash) []byte { return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) diff --git a/core/rawdb/table.go b/core/rawdb/table.go new file mode 100644 index 000000000..974df681b --- /dev/null +++ b/core/rawdb/table.go @@ -0,0 +1,150 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/ethdb" +) + +// table is a wrapper around a database that prefixes each key access with a pre- +// configured string. +type table struct { + db ethdb.Database + prefix string +} + +// NewTable returns a database object that prefixes all keys with a given string. +func NewTable(db ethdb.Database, prefix string) ethdb.Database { + return &table{ + db: db, + prefix: prefix, + } +} + +// Close is a noop to implement the Database interface. +func (t *table) Close() error { + return nil +} + +// Has retrieves if a prefixed version of a key is present in the database. +func (t *table) Has(key []byte) (bool, error) { + return t.db.Has(append([]byte(t.prefix), key...)) +} + +// Get retrieves the given prefixed key if it's present in the database. +func (t *table) Get(key []byte) ([]byte, error) { + return t.db.Get(append([]byte(t.prefix), key...)) +} + +// Put inserts the given value into the database at a prefixed version of the +// provided key. +func (t *table) Put(key []byte, value []byte) error { + return t.db.Put(append([]byte(t.prefix), key...), value) +} + +// Delete removes the given prefixed key from the database. +func (t *table) Delete(key []byte) error { + return t.db.Delete(append([]byte(t.prefix), key...)) +} + +// NewIterator creates a binary-alphabetical iterator over the entire keyspace +// contained within the database. +func (t *table) NewIterator() ethdb.Iterator { + return t.NewIteratorWithPrefix(nil) +} + +// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix. +func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { + return t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...)) +} + +// Stat returns a particular internal stat of the database. +func (t *table) Stat(property string) (string, error) { + return t.db.Stat(property) +} + +// Compact flattens the underlying data store for the given key range. In essence, +// deleted and overwritten versions are discarded, and the data is rearranged to +// reduce the cost of operations needed to access them. +// +// A nil start is treated as a key before all keys in the data store; a nil limit +// is treated as a key after all keys in the data store. If both is nil then it +// will compact entire data store. +func (t *table) Compact(start []byte, limit []byte) error { + // If no start was specified, use the table prefix as the first value + if start == nil { + start = []byte(t.prefix) + } + // If no limit was specified, use the first element not matching the prefix + // as the limit + if limit == nil { + limit = []byte(t.prefix) + for i := len(limit) - 1; i >= 0; i-- { + // Bump the current character, stopping if it doesn't overflow + limit[i]++ + if limit[i] > 0 { + break + } + // Character overflown, proceed to the next or nil if the last + if i == 0 { + limit = nil + } + } + } + // Range correctly calculated based on table prefix, delegate down + return t.db.Compact(start, limit) +} + +// NewBatch creates a write-only database that buffers changes to its host db +// until a final write is called, each operation prefixing all keys with the +// pre-configured string. +func (t *table) NewBatch() ethdb.Batch { + return &tableBatch{t.db.NewBatch(), t.prefix} +} + +// tableBatch is a wrapper around a database batch that prefixes each key access +// with a pre-configured string. +type tableBatch struct { + batch ethdb.Batch + prefix string +} + +// Put inserts the given value into the batch for later committing. +func (b *tableBatch) Put(key, value []byte) error { + return b.batch.Put(append([]byte(b.prefix), key...), value) +} + +// Delete inserts the a key removal into the batch for later committing. +func (b *tableBatch) Delete(key []byte) error { + return b.batch.Delete(append([]byte(b.prefix), key...)) +} + +// ValueSize retrieves the amount of data queued up for writing. +func (b *tableBatch) ValueSize() int { + return b.batch.ValueSize() +} + +// Write flushes any accumulated data to disk. +func (b *tableBatch) Write() error { + return b.batch.Write() +} + +// Reset resets the batch for reuse. +func (b *tableBatch) Reset() { + b.batch.Reset() +} diff --git a/core/state/database.go b/core/state/database.go index f6ea144b9..ce085747a 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -68,7 +68,7 @@ type Trie interface { Hash() common.Hash NodeIterator(startKey []byte) trie.NodeIterator GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed - Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error + Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error } // NewDatabase creates a backing store for state. The returned database is safe for @@ -179,6 +179,6 @@ func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) { return root, err } -func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { +func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error { return m.SecureTrie.Prove(key, fromLevel, proofDb) } diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 9e46c851c..69f51c4c7 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -51,7 +51,9 @@ func TestNodeIteratorCoverage(t *testing.T) { t.Errorf("state entry not reported %x", hash) } } - for _, key := range db.TrieDB().DiskDB().(*ethdb.MemDatabase).Keys() { + it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator() + for it.Next() { + key := it.Key() if bytes.HasPrefix(key, []byte("secure-key-")) { continue } @@ -59,4 +61,5 @@ func TestNodeIteratorCoverage(t *testing.T) { t.Errorf("state entry not reported %x", key) } } + it.Release() } diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index 3d9c4e867..fdfde96ad 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -20,13 +20,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core/rawdb" ) var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { - statedb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) ms := ManageState(statedb) ms.StateDB.SetNonce(addr, 100) ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr)) diff --git a/core/state/state_test.go b/core/state/state_test.go index a09273f3b..606f2a6f6 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -22,13 +22,14 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" checker "gopkg.in/check.v1" ) type StateSuite struct { - db *ethdb.MemDatabase + db ethdb.Database state *StateDB } @@ -87,7 +88,7 @@ func (s *StateSuite) TestDump(c *checker.C) { } func (s *StateSuite) SetUpTest(c *checker.C) { - s.db = ethdb.NewMemDatabase() + s.db = rawdb.NewMemoryDatabase() s.state, _ = New(common.Hash{}, NewDatabase(s.db)) } @@ -141,7 +142,7 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) { // use testing instead of checker because checker does not support // printing/logging in tests (-check.vv does not work) func TestSnapshot2(t *testing.T) { - state, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 69392d972..c2d2b2f69 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -31,15 +31,15 @@ import ( check "gopkg.in/check.v1" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" ) // Tests that updating a state trie does not leak any database writes prior to // actually committing the state. func TestUpdateLeaks(t *testing.T) { // Create an empty state database - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() state, _ := New(common.Hash{}, NewDatabase(db)) // Update it with some accounts @@ -56,18 +56,19 @@ func TestUpdateLeaks(t *testing.T) { state.IntermediateRoot(false) } // Ensure that no data was leaked into the database - for _, key := range db.Keys() { - value, _ := db.Get(key) - t.Errorf("State leaked into database: %x -> %x", key, value) + it := db.NewIterator() + for it.Next() { + t.Errorf("State leaked into database: %x -> %x", it.Key(), it.Value()) } + it.Release() } // Tests that no intermediate state of an object is stored into the database, // only the one right before the commit. func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning - transDb := ethdb.NewMemDatabase() - finalDb := ethdb.NewMemDatabase() + transDb := rawdb.NewMemoryDatabase() + finalDb := rawdb.NewMemoryDatabase() transState, _ := New(common.Hash{}, NewDatabase(transDb)) finalState, _ := New(common.Hash{}, NewDatabase(finalDb)) @@ -103,16 +104,20 @@ func TestIntermediateLeaks(t *testing.T) { if _, err := finalState.Commit(false); err != nil { t.Fatalf("failed to commit final state: %v", err) } - for _, key := range finalDb.Keys() { + it := finalDb.NewIterator() + for it.Next() { + key := it.Key() if _, err := transDb.Get(key); err != nil { - val, _ := finalDb.Get(key) - t.Errorf("entry missing from the transition database: %x -> %x", key, val) + t.Errorf("entry missing from the transition database: %x -> %x", key, it.Value()) } } - for _, key := range transDb.Keys() { + it.Release() + + it = transDb.NewIterator() + for it.Next() { + key := it.Key() if _, err := finalDb.Get(key); err != nil { - val, _ := transDb.Get(key) - t.Errorf("extra entry in the transition database: %x -> %x", key, val) + t.Errorf("extra entry in the transition database: %x -> %x", key, it.Value()) } } } @@ -122,7 +127,7 @@ func TestIntermediateLeaks(t *testing.T) { // https://github.com/ethereum/go-ethereum/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - orig, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + orig, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) for i := byte(0); i < 255; i++ { obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) @@ -342,7 +347,7 @@ func (test *snapshotTest) String() string { func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( - state, _ = New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + state, _ = New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 ) @@ -433,7 +438,7 @@ func (s *StateSuite) TestTouchDelete(c *check.C) { // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy. // See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512 func TestCopyOfCopy(t *testing.T) { - sdb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + sdb, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) addr := common.HexToAddress("aaaa") sdb.SetBalance(addr, big.NewInt(42)) diff --git a/core/state/sync.go b/core/state/sync.go index c566e7907..5290411a3 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -20,12 +20,13 @@ import ( "bytes" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.Sync { +func NewStateSync(root common.Hash, database ethdb.Reader) *trie.Sync { var syncer *trie.Sync callback := func(leaf []byte, parent common.Hash) error { var obj Account diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 317740160..ab4718b04 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" @@ -38,7 +39,7 @@ type testAccount struct { // makeTestState create a sample test state to test node-wise reconstruction. func makeTestState() (Database, common.Hash, []*testAccount) { // Create an empty state - db := NewDatabase(ethdb.NewMemDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase()) state, _ := New(common.Hash{}, db) // Fill it with some arbitrary data @@ -124,7 +125,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - if req := NewStateSync(empty, ethdb.NewMemDatabase()).Missing(1); len(req) != 0 { + if req := NewStateSync(empty, rawdb.NewMemoryDatabase()).Missing(1); len(req) != 0 { t.Errorf("content requested for empty state: %v", req) } } @@ -139,7 +140,7 @@ func testIterativeStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb := ethdb.NewMemDatabase() + dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(batch)...) @@ -171,7 +172,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb := ethdb.NewMemDatabase() + dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(0)...) @@ -208,7 +209,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb := ethdb.NewMemDatabase() + dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -248,7 +249,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb := ethdb.NewMemDatabase() + dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -295,7 +296,7 @@ func TestIncompleteStateSync(t *testing.T) { checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot) // Create a destination state and sync with the scheduler - dstDb := ethdb.NewMemDatabase() + dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb) added := []common.Hash{} diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index e1c2e0669..50c73cf53 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -27,10 +27,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" ) @@ -78,7 +78,7 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec } func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} key, _ := crypto.GenerateKey() @@ -163,7 +163,7 @@ func (c *testChain) State() (*state.StateDB, error) { // a state change between those fetches. stdb := c.statedb if *c.trigger { - c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether)) @@ -181,7 +181,7 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) { var ( key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) - statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) trigger = false ) @@ -335,7 +335,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -364,7 +364,7 @@ func TestTransactionDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -554,7 +554,7 @@ func TestTransactionPostponing(t *testing.T) { t.Parallel() // Create the pool to test the postponing with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -769,7 +769,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -857,7 +857,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { evictionInterval = time.Second // Create the pool to test the non-expiration enforcement - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1011,7 +1011,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1057,7 +1057,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1091,7 +1091,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1139,7 +1139,7 @@ func TestTransactionPoolRepricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1260,7 +1260,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1322,7 +1322,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1428,7 +1428,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1494,7 +1494,7 @@ func TestTransactionReplacement(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1588,7 +1588,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { os.Remove(journal) // Create the original pool to inject transaction into the journal - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1686,7 +1686,7 @@ func TestTransactionStatusCheck(t *testing.T) { t.Parallel() // Create the pool to test the status retrievals with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index cda49a34b..db1f6f382 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -22,10 +22,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -99,7 +99,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { setDefaults(cfg) if cfg.State == nil { - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) } var ( address = common.BytesToAddress([]byte("contract")) @@ -129,7 +129,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { setDefaults(cfg) if cfg.State == nil { - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) } var ( vmenv = NewEnv(cfg) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index bac06e524..15f545ddc 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -23,9 +23,9 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -95,7 +95,7 @@ func TestExecute(t *testing.T) { } func TestCall(t *testing.T) { - state, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) address := common.HexToAddress("0x0a") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, @@ -151,7 +151,7 @@ func BenchmarkCall(b *testing.B) { } func benchmarkEVM_Create(bench *testing.B, code string) { var ( - statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) sender = common.BytesToAddress([]byte("sender")) receiver = common.BytesToAddress([]byte("receiver")) ) diff --git a/eth/api_test.go b/eth/api_test.go index 47b062a40..cdd5bb8e3 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -22,8 +22,8 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/ethdb" ) var dumper = spew.ConfigState{Indent: " "} @@ -31,7 +31,7 @@ var dumper = spew.ConfigState{Indent: " "} func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - state, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + state, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), diff --git a/eth/backend.go b/eth/backend.go index cccb5993f..07c14163d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -121,7 +121,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object - chainDb, err := CreateDB(ctx, config, "chaindata") + chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/") if err != nil { return nil, err } @@ -220,18 +220,6 @@ func makeExtraData(extra []byte) []byte { return extra } -// CreateDB creates the chain database. -func CreateDB(ctx *node.ServiceContext, config *Config, name string) (ethdb.Database, error) { - db, err := ctx.OpenDatabase(name, config.DatabaseCache, config.DatabaseHandles) - if err != nil { - return nil, err - } - if db, ok := db.(*ethdb.LDBDatabase); ok { - db.Meter("eth/db/chaindata/") - } - return db, nil -} - // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up diff --git a/eth/bloombits.go b/eth/bloombits.go index c7bb56140..9a31997d6 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -102,7 +102,7 @@ func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *core.ChainIndexe db: db, size: size, } - table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) + table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) return core.NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") } diff --git a/eth/config.go b/eth/config.go index 740e6825b..1bbe326da 100644 --- a/eth/config.go +++ b/eth/config.go @@ -111,9 +111,10 @@ type Config struct { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCleanCache int - TrieDirtyCache int - TrieTimeout time.Duration + + TrieCleanCache int + TrieDirtyCache int + TrieTimeout time.Duration // Mining-related options Etherbase common.Address `toml:",omitempty"` diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 1a42965d3..405d52a71 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -28,6 +28,7 @@ import ( ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -71,8 +72,9 @@ func newTester() *downloadTester { ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, } - tester.stateDb = ethdb.NewMemDatabase() + tester.stateDb = rawdb.NewMemoryDatabase() tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) + tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) return tester } diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 0b5a21425..4ae342dc6 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -24,9 +24,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -34,7 +34,7 @@ import ( var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - testDB = ethdb.NewMemDatabase() + testDB = rawdb.NewMemoryDatabase() testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000)) ) diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index a86e773e3..83172c534 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -27,14 +27,14 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) var ( - testdb = ethdb.NewMemDatabase() + testdb = rawdb.NewMemoryDatabase() testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index c5f681e02..434e6a44c 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -17,7 +17,6 @@ package filters import ( - "bytes" "context" "fmt" "testing" @@ -67,7 +66,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" fmt.Println("Running bloombits benchmark section size:", sectionSize) - db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } @@ -129,7 +128,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() - db, _ = ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") backend = &testBackend{mux, db, cnt, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} } var addr common.Address @@ -146,37 +145,21 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { db.Close() } -func forEachKey(db ethdb.Database, startPrefix, endPrefix []byte, fn func(key []byte)) { - it := db.(*ethdb.LDBDatabase).NewIterator() - it.Seek(startPrefix) - for it.Valid() { - key := it.Key() - cmpLen := len(key) - if len(endPrefix) < cmpLen { - cmpLen = len(endPrefix) - } - if bytes.Compare(key[:cmpLen], endPrefix) == 1 { - break - } - fn(common.CopyBytes(key)) - it.Next() - } - it.Release() -} - var bloomBitsPrefix = []byte("bloomBits-") func clearBloomBits(db ethdb.Database) { fmt.Println("Clearing bloombits data...") - forEachKey(db, bloomBitsPrefix, bloomBitsPrefix, func(key []byte) { - db.Delete(key) - }) + it := db.NewIteratorWithPrefix(bloomBitsPrefix) + for it.Next() { + db.Delete(it.Key()) + } + it.Release() } func BenchmarkNoBloomBits(b *testing.B) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" fmt.Println("Running benchmark without bloombits") - db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index e71080b1a..e0c2a6a95 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -161,7 +161,7 @@ func TestBlockSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -218,7 +218,7 @@ func TestPendingTxFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -278,7 +278,7 @@ func TestPendingTxFilter(t *testing.T) { func TestLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -327,7 +327,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -354,7 +354,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { func TestInvalidGetLogsRequest(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -384,7 +384,7 @@ func TestLogFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -503,7 +503,7 @@ func TestPendingLogsSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 396a03d61..96d27bcf8 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" ) @@ -51,7 +50,7 @@ func BenchmarkFilters(b *testing.B) { defer os.RemoveAll(dir) var ( - db, _ = ethdb.NewLDBDatabase(dir, 0, 0) + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") mux = new(event.TypeMux) txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -110,7 +109,7 @@ func TestFilters(t *testing.T) { defer os.RemoveAll(dir) var ( - db, _ = ethdb.NewLDBDatabase(dir, 0, 0) + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") mux = new(event.TypeMux) txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) diff --git a/eth/handler_test.go b/eth/handler_test.go index 9fffd9581..7028d8d08 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -27,12 +27,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -344,11 +344,15 @@ func testGetNodeData(t *testing.T, protocol int) { // Fetch for now the entire chain db hashes := []common.Hash{} - for _, key := range db.Keys() { - if len(key) == len(common.Hash{}) { + + it := db.NewIterator() + for it.Next() { + if key := it.Key(); len(key) == common.HashLength { hashes = append(hashes, common.BytesToHash(key)) } } + it.Release() + p2p.Send(peer.app, 0x0d, hashes) msg, err := peer.app.ReadMsg() if err != nil { @@ -367,7 +371,7 @@ func testGetNodeData(t *testing.T, protocol int) { t.Errorf("data hash mismatch: have %x, want %x", hash, want) } } - statedb := ethdb.NewMemDatabase() + statedb := rawdb.NewMemoryDatabase() for i := 0; i < len(data); i++ { statedb.Put(hashes[i].Bytes(), data[i]) } @@ -469,7 +473,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool var ( evmux = new(event.TypeMux) pow = ethash.NewFaker() - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} gspec = &core.Genesis{Config: config} genesis = gspec.MustCommit(db) @@ -550,7 +554,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { var ( evmux = new(event.TypeMux) pow = ethash.NewFaker() - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() config = ¶ms.ChainConfig{} gspec = &core.Genesis{Config: config} genesis = gspec.MustCommit(db) diff --git a/eth/helper_test.go b/eth/helper_test.go index b18a02baf..e91429b8c 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -49,11 +50,11 @@ var ( // newTestProtocolManager creates a new protocol manager for testing purposes, // with the given number of blocks already known, and potential notification // channels for different events. -func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, *ethdb.MemDatabase, error) { +func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, ethdb.Database, error) { var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec = &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}}, @@ -78,7 +79,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func // with the given number of blocks already known, and potential notification // channels for different events. In case of an error, the constructor force- // fails the test. -func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, *ethdb.MemDatabase) { +func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, ethdb.Database) { pm, db, err := newTestProtocolManager(mode, blocks, generator, newtx) if err != nil { t.Fatalf("Failed to create protocol manager: %v", err) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 8b12b5b65..69eb80a5c 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -31,10 +31,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" @@ -155,6 +155,7 @@ func TestPrestateTracerCreate2(t *testing.T) { GasPrice: big.NewInt(1), } alloc := core.GenesisAlloc{} + // The code pushes 'deadbeef' into memory, then the other params, and calls CREATE2, then returns // the address alloc[common.HexToAddress("0x00000000000000000000000000000000deadbeef")] = core.GenesisAccount{ @@ -167,7 +168,8 @@ func TestPrestateTracerCreate2(t *testing.T) { Code: []byte{}, Balance: big.NewInt(500000000000000), } - statedb := tests.MakePreState(ethdb.NewMemDatabase(), alloc) + statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc) + // Create the tracer, the EVM environment and run it tracer, err := New("prestateTracer") if err != nil { @@ -240,7 +242,7 @@ func TestCallTracer(t *testing.T) { GasLimit: uint64(test.Context.GasLimit), GasPrice: tx.GasPrice(), } - statedb := tests.MakePreState(ethdb.NewMemDatabase(), test.Genesis.Alloc) + statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc) // Create the tracer, the EVM environment and run it tracer, err := New("callTracer") diff --git a/ethdb/.gitignore b/ethdb/.gitignore deleted file mode 100644 index f725d58d1..000000000 --- a/ethdb/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -# See http://help.github.com/ignore-files/ for more about ignoring files. -# -# If you find yourself ignoring temporary files generated by your text editor -# or operating system, you probably want to add a global ignore instead: -# git config --global core.excludesfile ~/.gitignore_global - -/tmp -*/**/*un~ -*un~ -.DS_Store -*/**/.DS_Store - diff --git a/ethdb/interface.go b/ethdb/batch.go index af1355779..a6f015821 100644 --- a/ethdb/interface.go +++ b/ethdb/batch.go @@ -1,4 +1,4 @@ -// Copyright 2014 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -16,37 +16,29 @@ package ethdb -// Code using batches should try to add this much data to the batch. -// The value was determined empirically. +// IdealBatchSize defines the size of the data batches should ideally add in one +// write. const IdealBatchSize = 100 * 1024 -// Putter wraps the database write operation supported by both batches and regular databases. -type Putter interface { - Put(key []byte, value []byte) error -} - -// Deleter wraps the database delete operation supported by both batches and regular databases. -type Deleter interface { - Delete(key []byte) error -} - -// Database wraps all database operations. All methods are safe for concurrent use. -type Database interface { - Putter - Deleter - Get(key []byte) ([]byte, error) - Has(key []byte) (bool, error) - Close() - NewBatch() Batch -} - // Batch is a write-only database that commits changes to its host database -// when Write is called. Batch cannot be used concurrently. +// when Write is called. A batch cannot be used concurrently. type Batch interface { - Putter + Writer Deleter - ValueSize() int // amount of data in the batch + + // ValueSize retrieves the amount of data queued up for writing. + ValueSize() int + + // Write flushes any accumulated data to disk. Write() error + // Reset resets the batch for reuse Reset() } + +// Batcher wraps the NewBatch method of a backing data store. +type Batcher interface { + // NewBatch creates a write-only database that buffers changes to its host db + // until a final write is called. + NewBatch() Batch +} diff --git a/ethdb/database.go b/ethdb/database.go index 17f1478e5..30208e146 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -1,4 +1,4 @@ -// Copyright 2014 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,372 +14,72 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -// +build !js - +// Package database defines the interfaces for an Ethereum data store. package ethdb -import ( - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -const ( - writePauseWarningThrottler = 1 * time.Minute -) - -var OpenFileLimit = 64 - -type LDBDatabase struct { - fn string // filename for reporting - db *leveldb.DB // LevelDB instance - - compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter metrics.Meter // Meter for measuring the data read during compaction - compWriteMeter metrics.Meter // Meter for measuring the data written during compaction - writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction - writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction - diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read - diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written - - quitLock sync.Mutex // Mutex protecting the quit channel access - quitChan chan chan error // Quit channel to stop the metrics collection before closing the database - - log log.Logger // Contextual logger tracking the database path -} - -// NewLDBDatabase returns a LevelDB wrapped object. -func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { - logger := log.New("database", file) - - // Ensure we have some minimal caching and file guarantees - if cache < 16 { - cache = 16 - } - if handles < 16 { - handles = 16 - } - logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) - - // Open the db and recover any potential corruptions - db, err := leveldb.OpenFile(file, &opt.Options{ - OpenFilesCacheCapacity: handles, - BlockCacheCapacity: cache / 2 * opt.MiB, - WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally - Filter: filter.NewBloomFilter(10), - }) - if _, corrupted := err.(*errors.ErrCorrupted); corrupted { - db, err = leveldb.RecoverFile(file, nil) - } - // (Re)check for errors and abort if opening of the db failed - if err != nil { - return nil, err - } - return &LDBDatabase{ - fn: file, - db: db, - log: logger, - }, nil -} - -// Path returns the path to the database directory. -func (db *LDBDatabase) Path() string { - return db.fn -} - -// Put puts the given key / value to the queue -func (db *LDBDatabase) Put(key []byte, value []byte) error { - return db.db.Put(key, value, nil) -} - -func (db *LDBDatabase) Has(key []byte) (bool, error) { - return db.db.Has(key, nil) -} - -// Get returns the given key if it's present. -func (db *LDBDatabase) Get(key []byte) ([]byte, error) { - dat, err := db.db.Get(key, nil) - if err != nil { - return nil, err - } - return dat, nil -} - -// Delete deletes the key from the queue and database -func (db *LDBDatabase) Delete(key []byte) error { - return db.db.Delete(key, nil) -} - -func (db *LDBDatabase) NewIterator() iterator.Iterator { - return db.db.NewIterator(nil, nil) -} - -// NewIteratorWithPrefix returns a iterator to iterate over subset of database content with a particular prefix. -func (db *LDBDatabase) NewIteratorWithPrefix(prefix []byte) iterator.Iterator { - return db.db.NewIterator(util.BytesPrefix(prefix), nil) -} - -func (db *LDBDatabase) Close() { - // Stop the metrics collection to avoid internal database races - db.quitLock.Lock() - defer db.quitLock.Unlock() - - if db.quitChan != nil { - errc := make(chan error) - db.quitChan <- errc - if err := <-errc; err != nil { - db.log.Error("Metrics collection failed", "err", err) - } - db.quitChan = nil - } - err := db.db.Close() - if err == nil { - db.log.Info("Database closed") - } else { - db.log.Error("Failed to close database", "err", err) - } -} - -func (db *LDBDatabase) LDB() *leveldb.DB { - return db.db -} - -// Meter configures the database metrics collectors and -func (db *LDBDatabase) Meter(prefix string) { - // Initialize all the metrics collector at the requested prefix - db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) - db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) - db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) - db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) - db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) - db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil) - db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil) - - // Create a quit channel for the periodic collector and run it - db.quitLock.Lock() - db.quitChan = make(chan chan error) - db.quitLock.Unlock() - - go db.meter(3 * time.Second) -} - -// meter periodically retrieves internal leveldb counters and reports them to -// the metrics subsystem. -// -// This is how a stats table look like (currently): -// Compactions -// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) -// -------+------------+---------------+---------------+---------------+--------------- -// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 -// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 -// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 -// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 -// -// This is how the write delay look like (currently): -// DelayN:5 Delay:406.604657ms Paused: false -// -// This is how the iostats look like (currently): -// Read(MB):3895.04860 Write(MB):3654.64712 -func (db *LDBDatabase) meter(refresh time.Duration) { - // Create the counters to store current and previous compaction values - compactions := make([][]float64, 2) - for i := 0; i < 2; i++ { - compactions[i] = make([]float64, 3) - } - // Create storage for iostats. - var iostats [2]float64 - - // Create storage and warning log tracer for write delay. - var ( - delaystats [2]int64 - lastWritePaused time.Time - ) - - var ( - errc chan error - merr error - ) - - // Iterate ad infinitum and collect the stats - for i := 1; errc == nil && merr == nil; i++ { - // Retrieve the database stats - stats, err := db.db.GetProperty("leveldb.stats") - if err != nil { - db.log.Error("Failed to read database stats", "err", err) - merr = err - continue - } - // Find the compaction table, skip the header - lines := strings.Split(stats, "\n") - for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { - lines = lines[1:] - } - if len(lines) <= 3 { - db.log.Error("Compaction table not found") - merr = errors.New("compaction table not found") - continue - } - lines = lines[3:] - - // Iterate over all the table rows, and accumulate the entries - for j := 0; j < len(compactions[i%2]); j++ { - compactions[i%2][j] = 0 - } - for _, line := range lines { - parts := strings.Split(line, "|") - if len(parts) != 6 { - break - } - for idx, counter := range parts[3:] { - value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) - if err != nil { - db.log.Error("Compaction entry parsing failed", "err", err) - merr = err - continue - } - compactions[i%2][idx] += value - } - } - // Update all the requested meters - if db.compTimeMeter != nil { - db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) - } - if db.compReadMeter != nil { - db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) - } - if db.compWriteMeter != nil { - db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) - } - - // Retrieve the write delay statistic - writedelay, err := db.db.GetProperty("leveldb.writedelay") - if err != nil { - db.log.Error("Failed to read database write delay statistic", "err", err) - merr = err - continue - } - var ( - delayN int64 - delayDuration string - duration time.Duration - paused bool - ) - if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { - db.log.Error("Write delay statistic not found") - merr = err - continue - } - duration, err = time.ParseDuration(delayDuration) - if err != nil { - db.log.Error("Failed to parse delay duration", "err", err) - merr = err - continue - } - if db.writeDelayNMeter != nil { - db.writeDelayNMeter.Mark(delayN - delaystats[0]) - } - if db.writeDelayMeter != nil { - db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) - } - // If a warning that db is performing compaction has been displayed, any subsequent - // warnings will be withheld for one minute not to overwhelm the user. - if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && - time.Now().After(lastWritePaused.Add(writePauseWarningThrottler)) { - db.log.Warn("Database compacting, degraded performance") - lastWritePaused = time.Now() - } - delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() - - // Retrieve the database iostats. - ioStats, err := db.db.GetProperty("leveldb.iostats") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - merr = err - continue - } - var nRead, nWrite float64 - parts := strings.Split(ioStats, " ") - if len(parts) < 2 { - db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) - continue - } - if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { - db.log.Error("Bad syntax of read entry", "entry", parts[0]) - merr = err - continue - } - if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { - db.log.Error("Bad syntax of write entry", "entry", parts[1]) - merr = err - continue - } - if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) - } - if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) - } - iostats[0], iostats[1] = nRead, nWrite - - // Sleep a bit, then repeat the stats collection - select { - case errc = <-db.quitChan: - // Quit requesting, stop hammering the database - case <-time.After(refresh): - // Timeout, gather a new set of stats - } - } - - if errc == nil { - errc = <-db.quitChan - } - errc <- merr -} - -func (db *LDBDatabase) NewBatch() Batch { - return &ldbBatch{db: db.db, b: new(leveldb.Batch)} -} - -type ldbBatch struct { - db *leveldb.DB - b *leveldb.Batch - size int -} - -func (b *ldbBatch) Put(key, value []byte) error { - b.b.Put(key, value) - b.size += len(value) - return nil -} - -func (b *ldbBatch) Delete(key []byte) error { - b.b.Delete(key) - b.size += 1 - return nil -} - -func (b *ldbBatch) Write() error { - return b.db.Write(b.b, nil) -} - -func (b *ldbBatch) ValueSize() int { - return b.size -} - -func (b *ldbBatch) Reset() { - b.b.Reset() - b.size = 0 +import "io" + +// Reader wraps the Has and Get method of a backing data store. +type Reader interface { + // Has retrieves if a key is present in the key-value data store. + Has(key []byte) (bool, error) + + // Get retrieves the given key if it's present in the key-value data store. + Get(key []byte) ([]byte, error) +} + +// Writer wraps the Put method of a backing data store. +type Writer interface { + // Put inserts the given value into the key-value data store. + Put(key []byte, value []byte) error +} + +// Deleter wraps the Delete method of a backing data store. +type Deleter interface { + // Delete removes the key from the key-value data store. + Delete(key []byte) error +} + +// Stater wraps the Stat method of a backing data store. +type Stater interface { + // Stat returns a particular internal stat of the database. + Stat(property string) (string, error) +} + +// Compacter wraps the Compact method of a backing data store. +type Compacter interface { + // Compact flattens the underlying data store for the given key range. In essence, + // deleted and overwritten versions are discarded, and the data is rearranged to + // reduce the cost of operations needed to access them. + // + // A nil start is treated as a key before all keys in the data store; a nil limit + // is treated as a key after all keys in the data store. If both is nil then it + // will compact entire data store. + Compact(start []byte, limit []byte) error +} + +// KeyValueStore contains all the methods required to allow handling different +// key-value data stores backing the high level database. +type KeyValueStore interface { + Reader + Writer + Deleter + Batcher + Iteratee + Stater + Compacter + io.Closer +} + +// Database contains all the methods required by the high level database to not +// only access the key-value data store but also the chain freezer. +type Database interface { + Reader + Writer + Deleter + Batcher + Iteratee + Stater + Compacter + io.Closer } diff --git a/ethdb/database_js.go b/ethdb/database_js.go deleted file mode 100644 index ba6eeb5a2..000000000 --- a/ethdb/database_js.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build js - -package ethdb - -import ( - "errors" -) - -var errNotSupported = errors.New("ethdb: not supported") - -type LDBDatabase struct { -} - -// NewLDBDatabase returns a LevelDB wrapped object. -func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { - return nil, errNotSupported -} - -// Path returns the path to the database directory. -func (db *LDBDatabase) Path() string { - return "" -} - -// Put puts the given key / value to the queue -func (db *LDBDatabase) Put(key []byte, value []byte) error { - return errNotSupported -} - -func (db *LDBDatabase) Has(key []byte) (bool, error) { - return false, errNotSupported -} - -// Get returns the given key if it's present. -func (db *LDBDatabase) Get(key []byte) ([]byte, error) { - return nil, errNotSupported -} - -// Delete deletes the key from the queue and database -func (db *LDBDatabase) Delete(key []byte) error { - return errNotSupported -} - -func (db *LDBDatabase) Close() { -} - -// Meter configures the database metrics collectors and -func (db *LDBDatabase) Meter(prefix string) { -} - -func (db *LDBDatabase) NewBatch() Batch { - return nil -} diff --git a/ethdb/database_js_test.go b/ethdb/database_js_test.go deleted file mode 100644 index b4c12ae0b..000000000 --- a/ethdb/database_js_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build js - -package ethdb_test - -import ( - "github.com/ethereum/go-ethereum/ethdb" -) - -var _ ethdb.Database = ðdb.LDBDatabase{} diff --git a/ethdb/database_test.go b/ethdb/database_test.go deleted file mode 100644 index 382fedbf9..000000000 --- a/ethdb/database_test.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -// +build !js - -package ethdb_test - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "strconv" - "sync" - "testing" - - "github.com/ethereum/go-ethereum/ethdb" -) - -func newTestLDB() (*ethdb.LDBDatabase, func()) { - dirname, err := ioutil.TempDir(os.TempDir(), "ethdb_test_") - if err != nil { - panic("failed to create test file: " + err.Error()) - } - db, err := ethdb.NewLDBDatabase(dirname, 0, 0) - if err != nil { - panic("failed to create test database: " + err.Error()) - } - - return db, func() { - db.Close() - os.RemoveAll(dirname) - } -} - -var test_values = []string{"", "a", "1251", "\x00123\x00"} - -func TestLDB_PutGet(t *testing.T) { - db, remove := newTestLDB() - defer remove() - testPutGet(db, t) -} - -func TestMemoryDB_PutGet(t *testing.T) { - testPutGet(ethdb.NewMemDatabase(), t) -} - -func testPutGet(db ethdb.Database, t *testing.T) { - t.Parallel() - - for _, k := range test_values { - err := db.Put([]byte(k), nil) - if err != nil { - t.Fatalf("put failed: %v", err) - } - } - - for _, k := range test_values { - data, err := db.Get([]byte(k)) - if err != nil { - t.Fatalf("get failed: %v", err) - } - if len(data) != 0 { - t.Fatalf("get returned wrong result, got %q expected nil", string(data)) - } - } - - _, err := db.Get([]byte("non-exist-key")) - if err == nil { - t.Fatalf("expect to return a not found error") - } - - for _, v := range test_values { - err := db.Put([]byte(v), []byte(v)) - if err != nil { - t.Fatalf("put failed: %v", err) - } - } - - for _, v := range test_values { - data, err := db.Get([]byte(v)) - if err != nil { - t.Fatalf("get failed: %v", err) - } - if !bytes.Equal(data, []byte(v)) { - t.Fatalf("get returned wrong result, got %q expected %q", string(data), v) - } - } - - for _, v := range test_values { - err := db.Put([]byte(v), []byte("?")) - if err != nil { - t.Fatalf("put override failed: %v", err) - } - } - - for _, v := range test_values { - data, err := db.Get([]byte(v)) - if err != nil { - t.Fatalf("get failed: %v", err) - } - if !bytes.Equal(data, []byte("?")) { - t.Fatalf("get returned wrong result, got %q expected ?", string(data)) - } - } - - for _, v := range test_values { - orig, err := db.Get([]byte(v)) - if err != nil { - t.Fatalf("get failed: %v", err) - } - orig[0] = byte(0xff) - data, err := db.Get([]byte(v)) - if err != nil { - t.Fatalf("get failed: %v", err) - } - if !bytes.Equal(data, []byte("?")) { - t.Fatalf("get returned wrong result, got %q expected ?", string(data)) - } - } - - for _, v := range test_values { - err := db.Delete([]byte(v)) - if err != nil { - t.Fatalf("delete %q failed: %v", v, err) - } - } - - for _, v := range test_values { - _, err := db.Get([]byte(v)) - if err == nil { - t.Fatalf("got deleted value %q", v) - } - } -} - -func TestLDB_ParallelPutGet(t *testing.T) { - db, remove := newTestLDB() - defer remove() - testParallelPutGet(db, t) -} - -func TestMemoryDB_ParallelPutGet(t *testing.T) { - testParallelPutGet(ethdb.NewMemDatabase(), t) -} - -func testParallelPutGet(db ethdb.Database, t *testing.T) { - const n = 8 - var pending sync.WaitGroup - - pending.Add(n) - for i := 0; i < n; i++ { - go func(key string) { - defer pending.Done() - err := db.Put([]byte(key), []byte("v"+key)) - if err != nil { - panic("put failed: " + err.Error()) - } - }(strconv.Itoa(i)) - } - pending.Wait() - - pending.Add(n) - for i := 0; i < n; i++ { - go func(key string) { - defer pending.Done() - data, err := db.Get([]byte(key)) - if err != nil { - panic("get failed: " + err.Error()) - } - if !bytes.Equal(data, []byte("v"+key)) { - panic(fmt.Sprintf("get failed, got %q expected %q", []byte(data), []byte("v"+key))) - } - }(strconv.Itoa(i)) - } - pending.Wait() - - pending.Add(n) - for i := 0; i < n; i++ { - go func(key string) { - defer pending.Done() - err := db.Delete([]byte(key)) - if err != nil { - panic("delete failed: " + err.Error()) - } - }(strconv.Itoa(i)) - } - pending.Wait() - - pending.Add(n) - for i := 0; i < n; i++ { - go func(key string) { - defer pending.Done() - _, err := db.Get([]byte(key)) - if err == nil { - panic("get succeeded") - } - }(strconv.Itoa(i)) - } - pending.Wait() -} diff --git a/ethdb/iterator.go b/ethdb/iterator.go new file mode 100644 index 000000000..f3cee7ec9 --- /dev/null +++ b/ethdb/iterator.go @@ -0,0 +1,61 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package ethdb + +// Iterator iterates over a database's key/value pairs in ascending key order. +// +// When it encounters an error any seek will return false and will yield no key/ +// value pairs. The error can be queried by calling the Error method. Calling +// Release is still necessary. +// +// An iterator must be released after use, but it is not necessary to read an +// iterator until exhaustion. An iterator is not safe for concurrent use, but it +// is safe to use multiple iterators concurrently. +type Iterator interface { + // Next moves the iterator to the next key/value pair. It returns whether the + // iterator is exhausted. + Next() bool + + // Error returns any accumulated error. Exhausting all the key/value pairs + // is not considered to be an error. + Error() error + + // Key returns the key of the current key/value pair, or nil if done. The caller + // should not modify the contents of the returned slice, and its contents may + // change on the next call to Next. + Key() []byte + + // Value returns the value of the current key/value pair, or nil if done. The + // caller should not modify the contents of the returned slice, and its contents + // may change on the next call to Next. + Value() []byte + + // Release releases associated resources. Release should always succeed and can + // be called multiple times without causing error. + Release() +} + +// Iteratee wraps the NewIterator methods of a backing data store. +type Iteratee interface { + // NewIterator creates a binary-alphabetical iterator over the entire keyspace + // contained within the key-value database. + NewIterator() Iterator + + // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset + // of database content with a particular key prefix. + NewIteratorWithPrefix(prefix []byte) Iterator +} diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go new file mode 100644 index 000000000..11bba6b08 --- /dev/null +++ b/ethdb/leveldb/leveldb.go @@ -0,0 +1,418 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build !js + +// Package leveldb implements the key-value database layer based on LevelDB. +package leveldb + +import ( + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" +) + +const ( + // leveldbDegradationWarnInterval specifies how often warning should be printed + // if the leveldb database cannot keep up with requested writes. + leveldbDegradationWarnInterval = time.Minute + + // leveldbMinCache is the minimum amount of memory in megabytes to allocate to + // leveldb read and write caching, split half and half. + leveldbMinCache = 16 + + // leveldbMinHandles is the minimum number of files handles to allocate to the + // open database files. + leveldbMinHandles = 16 + + // metricsGatheringInterval specifies the interval to retrieve leveldb database + // compaction, io and pause stats to report to the user. + metricsGatheringInterval = 3 * time.Second +) + +// LevelDBDatabase is a persistent key-value store. Apart from basic data storage +// functionality it also supports batch writes and iterating over the keyspace in +// binary-alphabetical order. +type LevelDBDatabase struct { + fn string // filename for reporting + db *leveldb.DB // LevelDB instance + + compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction + compReadMeter metrics.Meter // Meter for measuring the data read during compaction + compWriteMeter metrics.Meter // Meter for measuring the data written during compaction + writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction + writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction + diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read + diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written + + quitLock sync.Mutex // Mutex protecting the quit channel access + quitChan chan chan error // Quit channel to stop the metrics collection before closing the database + + log log.Logger // Contextual logger tracking the database path +} + +// New returns a wrapped LevelDB object. The namespace is the prefix that the +// metrics reporting should use for surfacing internal stats. +func New(file string, cache int, handles int, namespace string) (*LevelDBDatabase, error) { + // Ensure we have some minimal caching and file guarantees + if cache < leveldbMinCache { + cache = leveldbMinCache + } + if handles < leveldbMinHandles { + handles = leveldbMinHandles + } + logger := log.New("database", file) + logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) + + // Open the db and recover any potential corruptions + db, err := leveldb.OpenFile(file, &opt.Options{ + OpenFilesCacheCapacity: handles, + BlockCacheCapacity: cache / 2 * opt.MiB, + WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally + Filter: filter.NewBloomFilter(10), + }) + if _, corrupted := err.(*errors.ErrCorrupted); corrupted { + db, err = leveldb.RecoverFile(file, nil) + } + if err != nil { + return nil, err + } + // Assemble the wrapper with all the registered metrics + ldb := &LevelDBDatabase{ + fn: file, + db: db, + log: logger, + quitChan: make(chan chan error), + } + ldb.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil) + ldb.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil) + ldb.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil) + ldb.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil) + ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil) + ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil) + ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil) + + // Start up the metrics gathering and return + go ldb.meter(metricsGatheringInterval) + return ldb, nil +} + +// Close stops the metrics collection, flushes any pending data to disk and closes +// all io accesses to the underlying key-value store. +func (db *LevelDBDatabase) Close() error { + db.quitLock.Lock() + defer db.quitLock.Unlock() + + if db.quitChan != nil { + errc := make(chan error) + db.quitChan <- errc + if err := <-errc; err != nil { + db.log.Error("Metrics collection failed", "err", err) + } + db.quitChan = nil + } + return db.db.Close() +} + +// Has retrieves if a key is present in the key-value store. +func (db *LevelDBDatabase) Has(key []byte) (bool, error) { + return db.db.Has(key, nil) +} + +// Get retrieves the given key if it's present in the key-value store. +func (db *LevelDBDatabase) Get(key []byte) ([]byte, error) { + dat, err := db.db.Get(key, nil) + if err != nil { + return nil, err + } + return dat, nil +} + +// Put inserts the given value into the key-value store. +func (db *LevelDBDatabase) Put(key []byte, value []byte) error { + return db.db.Put(key, value, nil) +} + +// Delete removes the key from the key-value store. +func (db *LevelDBDatabase) Delete(key []byte) error { + return db.db.Delete(key, nil) +} + +// NewBatch creates a write-only key-value store that buffers changes to its host +// database until a final write is called. +func (db *LevelDBDatabase) NewBatch() ethdb.Batch { + return &levelDBBatch{ + db: db.db, + b: new(leveldb.Batch), + } +} + +// NewIterator creates a binary-alphabetical iterator over the entire keyspace +// contained within the leveldb database. +func (db *LevelDBDatabase) NewIterator() ethdb.Iterator { + return db.NewIteratorWithPrefix(nil) +} + +// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix. +func (db *LevelDBDatabase) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { + return db.db.NewIterator(util.BytesPrefix(prefix), nil) +} + +// Stat returns a particular internal stat of the database. +func (db *LevelDBDatabase) Stat(property string) (string, error) { + return db.db.GetProperty(property) +} + +// Compact flattens the underlying data store for the given key range. In essence, +// deleted and overwritten versions are discarded, and the data is rearranged to +// reduce the cost of operations needed to access them. +// +// A nil start is treated as a key before all keys in the data store; a nil limit +// is treated as a key after all keys in the data store. If both is nil then it +// will compact entire data store. +func (db *LevelDBDatabase) Compact(start []byte, limit []byte) error { + return db.db.CompactRange(util.Range{Start: start, Limit: limit}) +} + +// Path returns the path to the database directory. +func (db *LevelDBDatabase) Path() string { + return db.fn +} + +// meter periodically retrieves internal leveldb counters and reports them to +// the metrics subsystem. +// +// This is how a LevelDB stats table looks like (currently): +// Compactions +// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +// -------+------------+---------------+---------------+---------------+--------------- +// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 +// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 +// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 +// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 +// +// This is how the write delay look like (currently): +// DelayN:5 Delay:406.604657ms Paused: false +// +// This is how the iostats look like (currently): +// Read(MB):3895.04860 Write(MB):3654.64712 +func (db *LevelDBDatabase) meter(refresh time.Duration) { + // Create the counters to store current and previous compaction values + compactions := make([][]float64, 2) + for i := 0; i < 2; i++ { + compactions[i] = make([]float64, 3) + } + // Create storage for iostats. + var iostats [2]float64 + + // Create storage and warning log tracer for write delay. + var ( + delaystats [2]int64 + lastWritePaused time.Time + ) + + var ( + errc chan error + merr error + ) + + // Iterate ad infinitum and collect the stats + for i := 1; errc == nil && merr == nil; i++ { + // Retrieve the database stats + stats, err := db.db.GetProperty("leveldb.stats") + if err != nil { + db.log.Error("Failed to read database stats", "err", err) + merr = err + continue + } + // Find the compaction table, skip the header + lines := strings.Split(stats, "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { + lines = lines[1:] + } + if len(lines) <= 3 { + db.log.Error("Compaction leveldbTable not found") + merr = errors.New("compaction leveldbTable not found") + continue + } + lines = lines[3:] + + // Iterate over all the leveldbTable rows, and accumulate the entries + for j := 0; j < len(compactions[i%2]); j++ { + compactions[i%2][j] = 0 + } + for _, line := range lines { + parts := strings.Split(line, "|") + if len(parts) != 6 { + break + } + for idx, counter := range parts[3:] { + value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) + if err != nil { + db.log.Error("Compaction entry parsing failed", "err", err) + merr = err + continue + } + compactions[i%2][idx] += value + } + } + // Update all the requested meters + if db.compTimeMeter != nil { + db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) + } + if db.compReadMeter != nil { + db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) + } + if db.compWriteMeter != nil { + db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) + } + + // Retrieve the write delay statistic + writedelay, err := db.db.GetProperty("leveldb.writedelay") + if err != nil { + db.log.Error("Failed to read database write delay statistic", "err", err) + merr = err + continue + } + var ( + delayN int64 + delayDuration string + duration time.Duration + paused bool + ) + if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { + db.log.Error("Write delay statistic not found") + merr = err + continue + } + duration, err = time.ParseDuration(delayDuration) + if err != nil { + db.log.Error("Failed to parse delay duration", "err", err) + merr = err + continue + } + if db.writeDelayNMeter != nil { + db.writeDelayNMeter.Mark(delayN - delaystats[0]) + } + if db.writeDelayMeter != nil { + db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) + } + // If a warning that db is performing compaction has been displayed, any subsequent + // warnings will be withheld for one minute not to overwhelm the user. + if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && + time.Now().After(lastWritePaused.Add(leveldbDegradationWarnInterval)) { + db.log.Warn("Database compacting, degraded performance") + lastWritePaused = time.Now() + } + delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() + + // Retrieve the database iostats. + ioStats, err := db.db.GetProperty("leveldb.iostats") + if err != nil { + db.log.Error("Failed to read database iostats", "err", err) + merr = err + continue + } + var nRead, nWrite float64 + parts := strings.Split(ioStats, " ") + if len(parts) < 2 { + db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) + merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) + continue + } + if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { + db.log.Error("Bad syntax of read entry", "entry", parts[0]) + merr = err + continue + } + if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { + db.log.Error("Bad syntax of write entry", "entry", parts[1]) + merr = err + continue + } + if db.diskReadMeter != nil { + db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) + } + if db.diskWriteMeter != nil { + db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) + } + iostats[0], iostats[1] = nRead, nWrite + + // Sleep a bit, then repeat the stats collection + select { + case errc = <-db.quitChan: + // Quit requesting, stop hammering the database + case <-time.After(refresh): + // Timeout, gather a new set of stats + } + } + + if errc == nil { + errc = <-db.quitChan + } + errc <- merr +} + +// levelDBBatch is a write-only leveldb batch that commits changes to its host +// database when Write is called. A batch cannot be used concurrently. +type levelDBBatch struct { + db *leveldb.DB + b *leveldb.Batch + size int +} + +// Put inserts the given value into the batch for later committing. +func (b *levelDBBatch) Put(key, value []byte) error { + b.b.Put(key, value) + b.size += len(value) + return nil +} + +// Delete inserts the a key removal into the batch for later committing. +func (b *levelDBBatch) Delete(key []byte) error { + b.b.Delete(key) + b.size += 1 + return nil +} + +// ValueSize retrieves the amount of data queued up for writing. +func (b *levelDBBatch) ValueSize() int { + return b.size +} + +// Write flushes any accumulated data to disk. +func (b *levelDBBatch) Write() error { + return b.db.Write(b.b, nil) +} + +// Reset resets the batch for reuse. +func (b *levelDBBatch) Reset() { + b.b.Reset() + b.size = 0 +} diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go deleted file mode 100644 index 727f2f7ca..000000000 --- a/ethdb/memory_database.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package ethdb - -import ( - "errors" - "sync" - - "github.com/ethereum/go-ethereum/common" -) - -/* - * This is a test memory database. Do not use for any production it does not get persisted - */ -type MemDatabase struct { - db map[string][]byte - lock sync.RWMutex -} - -func NewMemDatabase() *MemDatabase { - return &MemDatabase{ - db: make(map[string][]byte), - } -} - -func NewMemDatabaseWithCap(size int) *MemDatabase { - return &MemDatabase{ - db: make(map[string][]byte, size), - } -} - -func (db *MemDatabase) Put(key []byte, value []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - db.db[string(key)] = common.CopyBytes(value) - return nil -} - -func (db *MemDatabase) Has(key []byte) (bool, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - _, ok := db.db[string(key)] - return ok, nil -} - -func (db *MemDatabase) Get(key []byte) ([]byte, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - if entry, ok := db.db[string(key)]; ok { - return common.CopyBytes(entry), nil - } - return nil, errors.New("not found") -} - -func (db *MemDatabase) Keys() [][]byte { - db.lock.RLock() - defer db.lock.RUnlock() - - keys := [][]byte{} - for key := range db.db { - keys = append(keys, []byte(key)) - } - return keys -} - -func (db *MemDatabase) Delete(key []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - delete(db.db, string(key)) - return nil -} - -func (db *MemDatabase) Close() {} - -func (db *MemDatabase) NewBatch() Batch { - return &memBatch{db: db} -} - -func (db *MemDatabase) Len() int { return len(db.db) } - -type kv struct { - k, v []byte - del bool -} - -type memBatch struct { - db *MemDatabase - writes []kv - size int -} - -func (b *memBatch) Put(key, value []byte) error { - b.writes = append(b.writes, kv{common.CopyBytes(key), common.CopyBytes(value), false}) - b.size += len(value) - return nil -} - -func (b *memBatch) Delete(key []byte) error { - b.writes = append(b.writes, kv{common.CopyBytes(key), nil, true}) - b.size += 1 - return nil -} - -func (b *memBatch) Write() error { - b.db.lock.Lock() - defer b.db.lock.Unlock() - - for _, kv := range b.writes { - if kv.del { - delete(b.db.db, string(kv.k)) - continue - } - b.db.db[string(kv.k)] = kv.v - } - return nil -} - -func (b *memBatch) ValueSize() int { - return b.size -} - -func (b *memBatch) Reset() { - b.writes = b.writes[:0] - b.size = 0 -} diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go new file mode 100644 index 000000000..9c6bd48be --- /dev/null +++ b/ethdb/memorydb/memorydb.go @@ -0,0 +1,298 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// Package memorydb implements the key-value database layer based on memory maps. +package memorydb + +import ( + "errors" + "sort" + "strings" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +var ( + // errMemorydbClosed is returned if a memory database was already closed at the + // invocation of a data access operation. + errMemorydbClosed = errors.New("database closed") + + // errMemorydbNotFound is returned if a key is requested that is not found in + // the provided memory database. + errMemorydbNotFound = errors.New("not found") +) + +// MemoryDatabase is an ephemeral key-value store. Apart from basic data storage +// functionality it also supports batch writes and iterating over the keyspace in +// binary-alphabetical order. +type MemoryDatabase struct { + db map[string][]byte + lock sync.RWMutex +} + +// New returns a wrapped map with all the required database interface methods +// implemented. +func New() *MemoryDatabase { + return &MemoryDatabase{ + db: make(map[string][]byte), + } +} + +// NewWithCap returns a wrapped map pre-allocated to the provided capcity with +// all the required database interface methods implemented. +func NewWithCap(size int) *MemoryDatabase { + return &MemoryDatabase{ + db: make(map[string][]byte, size), + } +} + +// Close deallocates the internal map and ensures any consecutive data access op +// failes with an error. +func (db *MemoryDatabase) Close() error { + db.lock.Lock() + defer db.lock.Unlock() + + db.db = nil + return nil +} + +// Has retrieves if a key is present in the key-value store. +func (db *MemoryDatabase) Has(key []byte) (bool, error) { + db.lock.RLock() + defer db.lock.RUnlock() + + if db.db == nil { + return false, errMemorydbClosed + } + _, ok := db.db[string(key)] + return ok, nil +} + +// Get retrieves the given key if it's present in the key-value store. +func (db *MemoryDatabase) Get(key []byte) ([]byte, error) { + db.lock.RLock() + defer db.lock.RUnlock() + + if db.db == nil { + return nil, errMemorydbClosed + } + if entry, ok := db.db[string(key)]; ok { + return common.CopyBytes(entry), nil + } + return nil, errMemorydbNotFound +} + +// Put inserts the given value into the key-value store. +func (db *MemoryDatabase) Put(key []byte, value []byte) error { + db.lock.Lock() + defer db.lock.Unlock() + + if db.db == nil { + return errMemorydbClosed + } + db.db[string(key)] = common.CopyBytes(value) + return nil +} + +// Delete removes the key from the key-value store. +func (db *MemoryDatabase) Delete(key []byte) error { + db.lock.Lock() + defer db.lock.Unlock() + + if db.db == nil { + return errMemorydbClosed + } + delete(db.db, string(key)) + return nil +} + +// NewBatch creates a write-only key-value store that buffers changes to its host +// database until a final write is called. +func (db *MemoryDatabase) NewBatch() ethdb.Batch { + return &memoryBatch{ + db: db, + } +} + +// NewIterator creates a binary-alphabetical iterator over the entire keyspace +// contained within the memory database. +func (db *MemoryDatabase) NewIterator() ethdb.Iterator { + return db.NewIteratorWithPrefix(nil) +} + +// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix. +func (db *MemoryDatabase) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { + db.lock.RLock() + defer db.lock.RUnlock() + + var ( + pr = string(prefix) + keys = make([]string, 0, len(db.db)) + values = make([][]byte, 0, len(db.db)) + ) + // Collect the keys from the memory database corresponding to the given prefix + for key := range db.db { + if strings.HasPrefix(key, pr) { + keys = append(keys, key) + } + } + // Sort the items and retrieve the associated values + sort.Strings(keys) + for _, key := range keys { + values = append(values, db.db[key]) + } + return &memoryIterator{ + keys: keys, + values: values, + } +} + +// Stat returns a particular internal stat of the database. +func (db *MemoryDatabase) Stat(property string) (string, error) { + return "", errors.New("unknown property") +} + +// Compact is not supported on a memory database. +func (db *MemoryDatabase) Compact(start []byte, limit []byte) error { + return errors.New("unsupported operation") +} + +// Len returns the number of entries currently present in the memory database. +// +// Note, this method is only used for testing (i.e. not public in general) and +// does not have explicit checks for closed-ness to allow simpler testing code. +func (db *MemoryDatabase) Len() int { + db.lock.RLock() + defer db.lock.RUnlock() + + return len(db.db) +} + +// keyvalue is a key-value tuple tagged with a deletion field to allow creating +// memory-database write batches. +type keyvalue struct { + key []byte + value []byte + delete bool +} + +// memoryBatch is a write-only memory batch that commits changes to its host +// database when Write is called. A batch cannot be used concurrently. +type memoryBatch struct { + db *MemoryDatabase + writes []keyvalue + size int +} + +// Put inserts the given value into the batch for later committing. +func (b *memoryBatch) Put(key, value []byte) error { + b.writes = append(b.writes, keyvalue{common.CopyBytes(key), common.CopyBytes(value), false}) + b.size += len(value) + return nil +} + +// Delete inserts the a key removal into the batch for later committing. +func (b *memoryBatch) Delete(key []byte) error { + b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true}) + b.size += 1 + return nil +} + +// ValueSize retrieves the amount of data queued up for writing. +func (b *memoryBatch) ValueSize() int { + return b.size +} + +// Write flushes any accumulated data to the memory database. +func (b *memoryBatch) Write() error { + b.db.lock.Lock() + defer b.db.lock.Unlock() + + for _, keyvalue := range b.writes { + if keyvalue.delete { + delete(b.db.db, string(keyvalue.key)) + continue + } + b.db.db[string(keyvalue.key)] = keyvalue.value + } + return nil +} + +// Reset resets the batch for reuse. +func (b *memoryBatch) Reset() { + b.writes = b.writes[:0] + b.size = 0 +} + +// memoryIterator can walk over the (potentially partial) keyspace of a memory +// key value store. Internally it is a deep copy of the entire iterated state, +// sorted by keys. +type memoryIterator struct { + inited bool + keys []string + values [][]byte +} + +// Next moves the iterator to the next key/value pair. It returns whether the +// iterator is exhausted. +func (it *memoryIterator) Next() bool { + // If the iterator was not yet initialized, do it now + if !it.inited { + it.inited = true + return len(it.keys) > 0 + } + // Iterator already initialize, advance it + if len(it.keys) > 0 { + it.keys = it.keys[1:] + it.values = it.values[1:] + } + return len(it.keys) > 0 +} + +// Error returns any accumulated error. Exhausting all the key/value pairs +// is not considered to be an error. A memory iterator cannot encounter errors. +func (it *memoryIterator) Error() error { + return nil +} + +// Key returns the key of the current key/value pair, or nil if done. The caller +// should not modify the contents of the returned slice, and its contents may +// change on the next call to Next. +func (it *memoryIterator) Key() []byte { + if len(it.keys) > 0 { + return []byte(it.keys[0]) + } + return nil +} + +// Value returns the value of the current key/value pair, or nil if done. The +// caller should not modify the contents of the returned slice, and its contents +// may change on the next call to Next. +func (it *memoryIterator) Value() []byte { + if len(it.values) > 0 { + return it.values[0] + } + return nil +} + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (it *memoryIterator) Release() { + it.keys, it.values = nil, nil +} diff --git a/ethdb/memorydb/memorydb_test.go b/ethdb/memorydb/memorydb_test.go new file mode 100644 index 000000000..4210a0f7c --- /dev/null +++ b/ethdb/memorydb/memorydb_test.go @@ -0,0 +1,100 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package memorydb + +import ( + "bytes" + "testing" +) + +// Tests that key-value iteration on top of a memory database works. +func TestMemoryDBIterator(t *testing.T) { + tests := []struct { + content map[string]string + prefix string + order []string + }{ + // Empty databases should be iterable + {map[string]string{}, "", nil}, + {map[string]string{}, "non-existent-prefix", nil}, + + // Single-item databases should be iterable + {map[string]string{"key": "val"}, "", []string{"key"}}, + {map[string]string{"key": "val"}, "k", []string{"key"}}, + {map[string]string{"key": "val"}, "l", nil}, + + // Multi-item databases should be fully iterable + { + map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, + "", + []string{"k1", "k2", "k3", "k4", "k5"}, + }, + { + map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, + "k", + []string{"k1", "k2", "k3", "k4", "k5"}, + }, + { + map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, + "l", + nil, + }, + // Multi-item databases should be prefix-iterable + { + map[string]string{ + "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", + "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", + }, + "ka", + []string{"ka1", "ka2", "ka3", "ka4", "ka5"}, + }, + { + map[string]string{ + "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", + "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", + }, + "kc", + nil, + }, + } + for i, tt := range tests { + // Create the key-value data store + db := New() + for key, val := range tt.content { + if err := db.Put([]byte(key), []byte(val)); err != nil { + t.Fatalf("test %d: failed to insert item %s:%s into database: %v", i, key, val, err) + } + } + // Iterate over the database with the given configs and verify the results + it, idx := db.NewIteratorWithPrefix([]byte(tt.prefix)), 0 + for it.Next() { + if !bytes.Equal(it.Key(), []byte(tt.order[idx])) { + t.Errorf("test %d: item %d: key mismatch: have %s, want %s", i, idx, string(it.Key()), tt.order[idx]) + } + if !bytes.Equal(it.Value(), []byte(tt.content[tt.order[idx]])) { + t.Errorf("test %d: item %d: value mismatch: have %s, want %s", i, idx, string(it.Value()), tt.content[tt.order[idx]]) + } + idx++ + } + if err := it.Error(); err != nil { + t.Errorf("test %d: iteration failed: %v", i, err) + } + if idx != len(tt.order) { + t.Errorf("test %d: iteration terminated prematurely: have %d, want %d", i, idx, len(tt.order)) + } + } +} diff --git a/ethdb/table.go b/ethdb/table.go deleted file mode 100644 index 28069c078..000000000 --- a/ethdb/table.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package ethdb - -type table struct { - db Database - prefix string -} - -// NewTable returns a Database object that prefixes all keys with a given -// string. -func NewTable(db Database, prefix string) Database { - return &table{ - db: db, - prefix: prefix, - } -} - -func (dt *table) Put(key []byte, value []byte) error { - return dt.db.Put(append([]byte(dt.prefix), key...), value) -} - -func (dt *table) Has(key []byte) (bool, error) { - return dt.db.Has(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Get(key []byte) ([]byte, error) { - return dt.db.Get(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Delete(key []byte) error { - return dt.db.Delete(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Close() { - // Do nothing; don't close the underlying DB. -} diff --git a/ethdb/table_batch.go b/ethdb/table_batch.go deleted file mode 100644 index ae83e79ce..000000000 --- a/ethdb/table_batch.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package ethdb - -type tableBatch struct { - batch Batch - prefix string -} - -// NewTableBatch returns a Batch object which prefixes all keys with a given string. -func NewTableBatch(db Database, prefix string) Batch { - return &tableBatch{db.NewBatch(), prefix} -} - -func (dt *table) NewBatch() Batch { - return &tableBatch{dt.db.NewBatch(), dt.prefix} -} - -func (tb *tableBatch) Put(key, value []byte) error { - return tb.batch.Put(append([]byte(tb.prefix), key...), value) -} - -func (tb *tableBatch) Delete(key []byte) error { - return tb.batch.Delete(append([]byte(tb.prefix), key...)) -} - -func (tb *tableBatch) Write() error { - return tb.batch.Write() -} - -func (tb *tableBatch) ValueSize() int { - return tb.batch.ValueSize() -} - -func (tb *tableBatch) Reset() { - tb.batch.Reset() -} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 4d5ef27da..e85e6d20a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/util" ) const ( @@ -1556,16 +1555,9 @@ func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) { } func (api *PrivateDebugAPI) ChaindbCompact() error { - ldb, ok := api.b.ChainDb().(interface { - LDB() *leveldb.DB - }) - if !ok { - return fmt.Errorf("chaindbCompact does not work for memory databases") - } for b := byte(0); b < 255; b++ { log.Info("Compacting chain database", "range", fmt.Sprintf("0x%0.2X-0x%0.2X", b, b+1)) - err := ldb.LDB().CompactRange(util.Range{Start: []byte{b}, Limit: []byte{b + 1}}) - if err != nil { + if err := api.b.ChainDb().Compact([]byte{b}, []byte{b + 1}); err != nil { log.Error("Database compaction failed", "err", err) return err } diff --git a/les/backend.go b/les/backend.go index 67ddf17e4..bc4d0f21d 100644 --- a/les/backend.go +++ b/les/backend.go @@ -79,7 +79,7 @@ type LightEthereum struct { } func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { - chainDb, err := eth.CreateDB(ctx, config, "lightchaindata") + chainDb, err := ctx.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/") if err != nil { return nil, err } diff --git a/les/freeclient_test.go b/les/freeclient_test.go index dc8b51a8d..191822264 100644 --- a/les/freeclient_test.go +++ b/les/freeclient_test.go @@ -24,7 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core/rawdb" ) func TestFreeClientPoolL10C100(t *testing.T) { @@ -44,7 +44,7 @@ const testFreeClientPoolTicks = 500000 func testFreeClientPool(t *testing.T, connLimit, clientCount int) { var ( clock mclock.Simulated - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() connected = make([]bool, clientCount) connTicks = make([]int, clientCount) disconnCh = make(chan int, clientCount) diff --git a/les/handler.go b/les/handler.go index 0352f5b03..9efe7d9e1 100644 --- a/les/handler.go +++ b/les/handler.go @@ -938,7 +938,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrRequestRejected, "") } go func() { - trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) + trieDb := trie.NewDatabase(rawdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for i, req := range req.Reqs { if i != 0 && !task.waitOrStop() { return @@ -1003,7 +1003,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { var prefix string if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { - auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) + auxTrie, _ = trie.New(root, trie.NewDatabase(rawdb.NewTable(pm.chainDb, prefix))) } } if req.AuxReq == auxRoot { diff --git a/les/handler_test.go b/les/handler_test.go index e9033729e..5cf31b8f5 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -405,7 +404,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { switch protocol { case 1: root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) var proof light.NodeList trie.Prove(key, 0, &proof) @@ -413,7 +412,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { case 2: root := light.GetChtRoot(server.db, (frequency/config.ChtSize)-1, bc.GetHeaderByNumber(frequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) } // Assemble the requests for the different protocols @@ -480,7 +479,7 @@ func TestGetBloombitsProofs(t *testing.T) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response @@ -493,7 +492,7 @@ func TestGetBloombitsProofs(t *testing.T) { } func TestTransactionStatusLes2(t *testing.T) { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db, nil) chain := pm.blockchain.(*core.BlockChain) config := core.DefaultTxPoolConfig diff --git a/les/helper_test.go b/les/helper_test.go index a8097708e..020c69e17 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -331,7 +332,7 @@ type TestEntity struct { // newServerEnv creates a server testing environment with a connected test peer for testing purpose. func newServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer)) (*TestEntity, func()) { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() cIndexer, bIndexer, btIndexer := testIndexers(db, nil, light.TestServerIndexerConfig) pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, nil, db, nil) @@ -363,7 +364,7 @@ func newServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*cor // newClientServerEnv creates a client/server arch environment with a connected les server and light client pair // for testing purpose. func newClientServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer), newPeer bool) (*TestEntity, *TestEntity, func()) { - db, ldb := ethdb.NewMemDatabase(), ethdb.NewMemDatabase() + db, ldb := rawdb.NewMemoryDatabase(), rawdb.NewMemoryDatabase() peers, lPeers := newPeerSet(), newPeerSet() dist := newRequestDistributor(lPeers, make(chan struct{}), &mclock.System{}) diff --git a/les/odr_requests.go b/les/odr_requests.go index 7b7876762..6bd4a2931 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -558,7 +558,7 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { // readTraceDB stores the keys of database reads. We use this to check that received node // sets contain only the trie nodes necessary to make proofs pass. type readTraceDB struct { - db trie.DatabaseReader + db ethdb.Reader reads map[string]struct{} } diff --git a/les/ulc_test.go b/les/ulc_test.go index 69ea62059..81986fa1e 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -1,21 +1,19 @@ package les import ( + "crypto/ecdsa" "fmt" + "math/big" + "net" "reflect" "testing" "time" - "net" - - "crypto/ecdsa" - "math/big" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -198,7 +196,7 @@ func connectPeers(full, light pairPeer, version int) (*peer, *peer, error) { // newFullPeerPair creates node with full sync mode func newFullPeerPair(t *testing.T, index int, numberOfblocks int, chainGen func(int, *core.BlockGen)) pairPeer { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() pmFull := newTestProtocolManagerMust(t, false, numberOfblocks, chainGen, nil, nil, db, nil) @@ -220,7 +218,7 @@ func newLightPeer(t *testing.T, ulcConfig *eth.ULCConfig) pairPeer { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{}), &mclock.System{}) rm := newRetrieveManager(peers, dist, nil) - ldb := ethdb.NewMemDatabase() + ldb := rawdb.NewMemoryDatabase() odr := NewLesOdr(ldb, light.DefaultClientIndexerConfig, rm) diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 374cb5319..58ea93044 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -52,7 +52,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [ // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(n int) (ethdb.Database, *LightChain, error) { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker()) @@ -69,7 +69,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { // newTestLightChain creates a LightChain that doesn't validate anything. func newTestLightChain() *LightChain { - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() gspec := &core.Genesis{ Difficulty: big.NewInt(1), Config: params.TestChainConfig, diff --git a/light/nodeset.go b/light/nodeset.go index 6f25219c1..3b556108a 100644 --- a/light/nodeset.go +++ b/light/nodeset.go @@ -106,7 +106,7 @@ func (db *NodeSet) NodeList() NodeList { } // Store writes the contents of the set to the given database -func (db *NodeSet) Store(target ethdb.Putter) { +func (db *NodeSet) Store(target ethdb.Writer) { db.lock.RLock() defer db.lock.RUnlock() @@ -115,11 +115,11 @@ func (db *NodeSet) Store(target ethdb.Putter) { } } -// NodeList stores an ordered list of trie nodes. It implements ethdb.Putter. +// NodeList stores an ordered list of trie nodes. It implements ethdb.Writer. type NodeList []rlp.RawValue // Store writes the contents of the list to the given database -func (n NodeList) Store(db ethdb.Putter) { +func (n NodeList) Store(db ethdb.Writer) { for _, node := range n { db.Put(crypto.Keccak256(node), node) } diff --git a/light/odr_test.go b/light/odr_test.go index 3da7b3055..c1762c43e 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -250,8 +250,8 @@ func testChainGen(i int, block *core.BlockGen) { func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( - sdb = ethdb.NewMemDatabase() - ldb = ethdb.NewMemDatabase() + sdb = rawdb.NewMemoryDatabase() + ldb = rawdb.NewMemoryDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) diff --git a/light/postprocess.go b/light/postprocess.go index dd1b74a7b..2030782b1 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -154,7 +154,7 @@ type ChtIndexerBackend struct { // NewChtIndexer creates a Cht chain indexer func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *core.ChainIndexer { - trieTable := ethdb.NewTable(db, ChtTablePrefix) + trieTable := rawdb.NewTable(db, ChtTablePrefix) backend := &ChtIndexerBackend{ diskdb: db, odr: odr, @@ -162,7 +162,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down sectionSize: size, } - return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") + return core.NewChainIndexer(db, rawdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") } // fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the @@ -276,7 +276,7 @@ type BloomTrieIndexerBackend struct { // NewBloomTrieIndexer creates a BloomTrie chain indexer func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uint64) *core.ChainIndexer { - trieTable := ethdb.NewTable(db, BloomTrieTablePrefix) + trieTable := rawdb.NewTable(db, BloomTrieTablePrefix) backend := &BloomTrieIndexerBackend{ diskdb: db, odr: odr, @@ -287,7 +287,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin } backend.bloomTrieRatio = size / parentSize backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) - return core.NewChainIndexer(db, ethdb.NewTable(db, "bltIndex-"), backend, size, 0, time.Millisecond*100, "bloomtrie") + return core.NewChainIndexer(db, rawdb.NewTable(db, "bltIndex-"), backend, size, 0, time.Millisecond*100, "bloomtrie") } // fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the diff --git a/light/trie.go b/light/trie.go index 4d4abe137..27abb1dc2 100644 --- a/light/trie.go +++ b/light/trie.go @@ -141,7 +141,7 @@ func (t *odrTrie) GetKey(sha []byte) []byte { return nil } -func (t *odrTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { +func (t *odrTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error { return errors.New("not implemented, needs client/server interface split") } diff --git a/light/trie_test.go b/light/trie_test.go index 5b5fce31d..4919f8964 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -25,17 +25,17 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) func TestNodeIterator(t *testing.T) { var ( - fulldb = ethdb.NewMemDatabase() - lightdb = ethdb.NewMemDatabase() + fulldb = rawdb.NewMemoryDatabase() + lightdb = rawdb.NewMemoryDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(fulldb) ) diff --git a/light/txpool_test.go b/light/txpool_test.go index ce77573ef..4f446c6ca 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -26,9 +26,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -81,8 +81,8 @@ func TestTxPool(t *testing.T) { } var ( - sdb = ethdb.NewMemDatabase() - ldb = ethdb.NewMemDatabase() + sdb = rawdb.NewMemoryDatabase() + ldb = rawdb.NewMemoryDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) diff --git a/miner/worker_test.go b/miner/worker_test.go index bfa9cbd21..99a671ae3 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -79,7 +80,7 @@ type testWorkerBackend struct { func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, n int) *testWorkerBackend { var ( - db = ethdb.NewMemDatabase() + db = rawdb.NewMemoryDatabase() gspec = core.Genesis{ Config: chainConfig, Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, @@ -142,7 +143,7 @@ func TestPendingStateAndBlockEthash(t *testing.T) { testPendingStateAndBlock(t, ethashChainConfig, ethash.NewFaker()) } func TestPendingStateAndBlockClique(t *testing.T) { - testPendingStateAndBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) + testPendingStateAndBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) } func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { @@ -174,7 +175,7 @@ func TestEmptyWorkEthash(t *testing.T) { testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) } func TestEmptyWorkClique(t *testing.T) { - testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) + testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) } func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { @@ -291,7 +292,7 @@ func TestRegenerateMiningBlockEthash(t *testing.T) { } func TestRegenerateMiningBlockClique(t *testing.T) { - testRegenerateMiningBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) + testRegenerateMiningBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) } func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { @@ -356,7 +357,7 @@ func TestAdjustIntervalEthash(t *testing.T) { } func TestAdjustIntervalClique(t *testing.T) { - testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) + testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) } func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { diff --git a/node/node.go b/node/node.go index 9227bf6bb..bd031bd0f 100644 --- a/node/node.go +++ b/node/node.go @@ -27,6 +27,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/debug" @@ -601,11 +602,11 @@ func (n *Node) EventMux() *event.TypeMux { // OpenDatabase opens an existing database with the given name (or creates one if no // previous can be found) from within the node's instance directory. If the node is // ephemeral, a memory database is returned. -func (n *Node) OpenDatabase(name string, cache, handles int) (ethdb.Database, error) { +func (n *Node) OpenDatabase(name string, cache, handles int, namespace string) (ethdb.Database, error) { if n.config.DataDir == "" { - return ethdb.NewMemDatabase(), nil + return rawdb.NewMemoryDatabase(), nil } - return ethdb.NewLDBDatabase(n.config.ResolvePath(name), cache, handles) + return rawdb.NewLevelDBDatabase(n.config.ResolvePath(name), cache, handles, namespace) } // ResolvePath returns the absolute path of a resource in the instance directory. diff --git a/node/service.go b/node/service.go index 6a96d9b1e..4f6cb6676 100644 --- a/node/service.go +++ b/node/service.go @@ -20,6 +20,7 @@ import ( "reflect" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/p2p" @@ -39,11 +40,11 @@ type ServiceContext struct { // OpenDatabase opens an existing database with the given name (or creates one // if no previous can be found) from within the node's data directory. If the // node is an ephemeral one, a memory database is returned. -func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) { +func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, namespace string) (ethdb.Database, error) { if ctx.config.DataDir == "" { - return ethdb.NewMemDatabase(), nil + return rawdb.NewMemoryDatabase(), nil } - db, err := ethdb.NewLDBDatabase(ctx.config.ResolvePath(name), cache, handles) + db, err := rawdb.NewLevelDBDatabase(ctx.config.ResolvePath(name), cache, handles, namespace) if err != nil { return nil, err } diff --git a/node/service_test.go b/node/service_test.go index 73c51eccb..63004a51a 100644 --- a/node/service_test.go +++ b/node/service_test.go @@ -39,7 +39,7 @@ func TestContextDatabases(t *testing.T) { } // Request the opening/creation of a database and ensure it persists to disk ctx := &ServiceContext{config: &Config{Name: "unit-test", DataDir: dir}} - db, err := ctx.OpenDatabase("persistent", 0, 0) + db, err := ctx.OpenDatabase("persistent", 0, 0, "") if err != nil { t.Fatalf("failed to open persistent database: %v", err) } @@ -50,7 +50,7 @@ func TestContextDatabases(t *testing.T) { } // Request th opening/creation of an ephemeral database and ensure it's not persisted ctx = &ServiceContext{config: &Config{DataDir: ""}} - db, err = ctx.OpenDatabase("ephemeral", 0, 0) + db, err = ctx.OpenDatabase("ephemeral", 0, 0, "") if err != nil { t.Fatalf("failed to open ephemeral database: %v", err) } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 9fa69bf4e..d40a3726c 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -30,10 +30,10 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -101,7 +101,7 @@ func (t *BlockTest) Run() error { } // import pre accounts & construct test genesis block & state root - db := ethdb.NewMemDatabase() + db := rawdb.NewMemoryDatabase() gblock, err := t.genesis(config).Commit(db) if err != nil { return err diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 436284196..0b78f26ed 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -126,7 +127,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD return nil, UnsupportedForkError{subtest.Fork} } block := t.genesis(config).ToBlock(nil) - statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) + statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre) post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index cb81c5b94..91566c47e 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -26,10 +26,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -79,7 +79,7 @@ type vmExecMarshaling struct { } func (t *VMTest) Run(vmconfig vm.Config) error { - statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) + statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre) ret, gasRemaining, err := t.exec(statedb, vmconfig) if t.json.GasRemaining == nil { diff --git a/trie/database.go b/trie/database.go index 958823eb8..73ba2e761 100644 --- a/trie/database.go +++ b/trie/database.go @@ -55,20 +55,11 @@ var secureKeyPrefix = []byte("secure-key-") // secureKeyLength is the length of the above prefix + 32byte hash. const secureKeyLength = 11 + 32 -// DatabaseReader wraps the Get and Has method of a backing store for the trie. -type DatabaseReader interface { - // Get retrieves the value associated with key from the database. - Get(key []byte) (value []byte, err error) - - // Has retrieves whether a key is present in the database. - Has(key []byte) (bool, error) -} - // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. type Database struct { - diskdb ethdb.Database // Persistent storage for matured trie nodes + diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes @@ -271,14 +262,14 @@ func expandNode(hash hashNode, n node, cachegen uint16) node { // NewDatabase creates a new trie database to store ephemeral trie content before // its written out to disk or garbage collected. No read cache is created, so all // data retrievals will hit the underlying disk database. -func NewDatabase(diskdb ethdb.Database) *Database { +func NewDatabase(diskdb ethdb.KeyValueStore) *Database { return NewDatabaseWithCache(diskdb, 0) } // NewDatabaseWithCache creates a new trie database to store ephemeral trie content // before its written out to disk or garbage collected. It also acts as a read cache // for nodes loaded from disk. -func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { +func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database { var cleans *bigcache.BigCache if cache > 0 { cleans, _ = bigcache.NewBigCache(bigcache.Config{ @@ -298,7 +289,7 @@ func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { } // DiskDB retrieves the persistent storage backing the trie database. -func (db *Database) DiskDB() DatabaseReader { +func (db *Database) DiskDB() ethdb.Reader { return db.diskdb } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 4f633b195..88b8103fb 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -23,7 +23,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func TestIterator(t *testing.T) { @@ -120,11 +120,14 @@ func TestNodeIteratorCoverage(t *testing.T) { } } } - for _, key := range db.diskdb.(*ethdb.MemDatabase).Keys() { + it := db.diskdb.NewIterator() + for it.Next() { + key := it.Key() if _, ok := hashes[common.BytesToHash(key)]; !ok { t.Errorf("state entry not reported %x", key) } } + it.Release() } type kvs struct{ k, v string } @@ -289,7 +292,7 @@ func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueA func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) } func testIteratorContinueAfterError(t *testing.T, memonly bool) { - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) tr, _ := New(common.Hash{}, triedb) @@ -309,7 +312,11 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { if memonly { memKeys = triedb.Nodes() } else { - diskKeys = diskdb.Keys() + it := diskdb.NewIterator() + for it.Next() { + diskKeys = append(diskKeys, it.Key()) + } + it.Release() } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. @@ -376,7 +383,7 @@ func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) { func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { // Commit test trie to db, then remove the node containing "bars". - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) ctr, _ := New(common.Hash{}, triedb) diff --git a/trie/proof.go b/trie/proof.go index 1334bde97..0f18dd26b 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -34,7 +34,7 @@ import ( // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { +func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error { // Collect all nodes on the path to key. key = keybytesToHex(key) var nodes []node @@ -97,14 +97,14 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { +func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error { return t.trie.Prove(key, fromLevel, proofDb) } // VerifyProof checks merkle proofs. The given proof must contain the value for // key in a trie with the given root hash. VerifyProof returns an error if the // proof contains invalid trie nodes or the wrong value. -func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, nodes int, err error) { +func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.Reader) (value []byte, nodes int, err error) { key = keybytesToHex(key) wantHash := rootHash for i := 0; ; i++ { diff --git a/trie/proof_test.go b/trie/proof_test.go index 996f87478..bcb241bd7 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func init() { @@ -34,18 +34,18 @@ func init() { // makeProvers creates Merkle trie provers based on different implementations to // test all variations. -func makeProvers(trie *Trie) []func(key []byte) *ethdb.MemDatabase { - var provers []func(key []byte) *ethdb.MemDatabase +func makeProvers(trie *Trie) []func(key []byte) *memorydb.MemoryDatabase { + var provers []func(key []byte) *memorydb.MemoryDatabase // Create a direct trie based Merkle prover - provers = append(provers, func(key []byte) *ethdb.MemDatabase { - proof := ethdb.NewMemDatabase() + provers = append(provers, func(key []byte) *memorydb.MemoryDatabase { + proof := memorydb.New() trie.Prove(key, 0, proof) return proof }) // Create a leaf iterator based Merkle prover - provers = append(provers, func(key []byte) *ethdb.MemDatabase { - proof := ethdb.NewMemDatabase() + provers = append(provers, func(key []byte) *memorydb.MemoryDatabase { + proof := memorydb.New() if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { for _, p := range it.Prove() { proof.Put(crypto.Keccak256(p), p) @@ -106,9 +106,14 @@ func TestBadProof(t *testing.T) { if proof == nil { t.Fatalf("prover %d: nil proof", i) } - key := proof.Keys()[mrand.Intn(proof.Len())] + it := proof.NewIterator() + for i, d := 0, mrand.Intn(proof.Len()); i <= d; i++ { + it.Next() + } + key := it.Key() val, _ := proof.Get(key) proof.Delete(key) + it.Release() mutateByte(val) proof.Put(crypto.Keccak256(val), val) @@ -127,7 +132,7 @@ func TestMissingKeyProof(t *testing.T) { updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { - proof := ethdb.NewMemDatabase() + proof := memorydb.New() trie.Prove([]byte(key), 0, proof) if proof.Len() != 1 { @@ -164,8 +169,8 @@ func BenchmarkProve(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { kv := vals[keys[i%len(keys)]] - proofs := ethdb.NewMemDatabase() - if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 { + proofs := memorydb.New() + if trie.Prove(kv.k, 0, proofs); proofs.Len() == 0 { b.Fatalf("zero length proof for %x", kv.k) } } @@ -175,10 +180,10 @@ func BenchmarkVerifyProof(b *testing.B) { trie, vals := randomTrie(100) root := trie.Hash() var keys []string - var proofs []*ethdb.MemDatabase + var proofs []*memorydb.MemoryDatabase for k := range vals { keys = append(keys, k) - proof := ethdb.NewMemDatabase() + proof := memorydb.New() trie.Prove([]byte(k), 0, proof) proofs = append(proofs, proof) } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index d16d99968..f0ca6c800 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -24,18 +24,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0) + trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New()), 0) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(ethdb.NewMemDatabase()) + triedb := NewDatabase(memorydb.New()) trie, _ := NewSecure(common.Hash{}, triedb, 0) diff --git a/trie/sync.go b/trie/sync.go index 44f5087b9..ef931f633 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -72,14 +72,14 @@ func newSyncMemBatch() *syncMemBatch { // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. type Sync struct { - database DatabaseReader // Persistent database to check for existing entries + database ethdb.Reader // Persistent database to check for existing entries membatch *syncMemBatch // Memory buffer to avoid frequent database writes requests map[common.Hash]*request // Pending requests pertaining to a key hash queue *prque.Prque // Priority queue with the pending requests } // NewSync creates a new trie data download scheduler. -func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) *Sync { +func NewSync(root common.Hash, database ethdb.Reader, callback LeafCallback) *Sync { ts := &Sync{ database: database, membatch: newSyncMemBatch(), @@ -213,7 +213,7 @@ func (s *Sync) Process(results []SyncResult) (bool, int, error) { // Commit flushes the data stored in the internal membatch out to persistent // storage, returning the number of items written and any occurred error. -func (s *Sync) Commit(dbw ethdb.Putter) (int, error) { +func (s *Sync) Commit(dbw ethdb.Writer) (int, error) { // Dump the membatch into a database dbw for i, key := range s.membatch.order { if err := dbw.Put(key[:], s.membatch.batch[key]); err != nil { diff --git a/trie/sync_test.go b/trie/sync_test.go index ff15baa52..d80070f3e 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -21,13 +21,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) // makeTestTrie create a sample test trie to test node-wise reconstruction. func makeTestTrie() (*Database, *Trie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(ethdb.NewMemDatabase()) + triedb := NewDatabase(memorydb.New()) trie, _ := New(common.Hash{}, triedb) // Fill it with some arbitrary data @@ -88,13 +88,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error { // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { - dbA := NewDatabase(ethdb.NewMemDatabase()) - dbB := NewDatabase(ethdb.NewMemDatabase()) + dbA := NewDatabase(memorydb.New()) + dbB := NewDatabase(memorydb.New()) emptyA, _ := New(common.Hash{}, dbA) emptyB, _ := New(emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { - if req := NewSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { + if req := NewSync(trie.Hash(), memorydb.New(), nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -110,7 +110,7 @@ func testIterativeSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -143,7 +143,7 @@ func TestIterativeDelayedSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -181,7 +181,7 @@ func testIterativeRandomSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -222,7 +222,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -269,7 +269,7 @@ func TestDuplicateAvoidanceSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) @@ -309,7 +309,7 @@ func TestIncompleteSync(t *testing.T) { srcDb, srcTrie, _ := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) diff --git a/trie/trie_test.go b/trie/trie_test.go index 4d84aa96c..cf133706f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -33,6 +33,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,7 +45,7 @@ func init() { // Used for testing func newEmpty() *Trie { - trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) + trie, _ := New(common.Hash{}, NewDatabase(memorydb.New())) return trie } @@ -67,7 +69,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase())) + trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -80,7 +82,7 @@ func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) } func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) } func testMissingNode(t *testing.T, memonly bool) { - diskdb := ethdb.NewMemDatabase() + diskdb := memorydb.New() triedb := NewDatabase(diskdb) trie, _ := New(common.Hash{}, triedb) @@ -317,13 +319,13 @@ func TestLargeValue(t *testing.T) { } type countingDB struct { - ethdb.Database + ethdb.KeyValueStore gets map[string]int } func (db *countingDB) Get(key []byte) ([]byte, error) { db.gets[string(key)]++ - return db.Database.Get(key) + return db.KeyValueStore.Get(key) } // TestCacheUnload checks that decoded nodes are unloaded after a @@ -342,7 +344,7 @@ func TestCacheUnload(t *testing.T) { // Commit the trie repeatedly and access key1. // The branch containing it is loaded from DB exactly two times: // in the 0th and 6th iteration. - diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} + diskdb := &countingDB{KeyValueStore: trie.db.diskdb, gets: make(map[string]int)} triedb := NewDatabase(diskdb) trie, _ = New(root, triedb) trie.SetCacheLimit(5) @@ -412,7 +414,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { } func runRandTest(rt randTest) bool { - triedb := NewDatabase(ethdb.NewMemDatabase()) + triedb := NewDatabase(memorydb.New()) tr, _ := New(common.Hash{}, triedb) values := make(map[string]string) // tracks content of the trie @@ -540,7 +542,7 @@ func benchGet(b *testing.B, commit bool) { b.StopTimer() if commit { - ldb := trie.db.diskdb.(*ethdb.LDBDatabase) + ldb := trie.db.diskdb.(*leveldb.LevelDBDatabase) ldb.Close() os.RemoveAll(ldb.Path()) } @@ -596,7 +598,7 @@ func tempDB() (string, *Database) { if err != nil { panic(fmt.Sprintf("can't create temporary directory: %v", err)) } - diskdb, err := ethdb.NewLDBDatabase(dir, 256, 0) + diskdb, err := leveldb.New(dir, 256, 0, "") if err != nil { panic(fmt.Sprintf("can't create temporary database: %v", err)) } |