From 006c21efc7af8bdf04d003ef256d8e2eb30006bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 8 Mar 2019 15:56:20 +0200 Subject: cmd, core, eth, les, node: chain freezer on top of db rework --- ethdb/database.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index bab99aed1..764e304e3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -67,10 +67,23 @@ type KeyValueStore interface { io.Closer } +// Ancienter wraps the Ancient method for a backing immutable chain data store. +type Ancienter interface { + // Ancient retrieves an ancient binary blob from the append-only immutable files. + Ancient(kind string, number uint64) ([]byte, error) +} + +// AncientReader contains the methods required to access both key-value as well as +// immutable ancient data. +type AncientReader interface { + Reader + Ancienter +} + // Database contains all the methods required by the high level database to not // only access the key-value data store but also the chain freezer. type Database interface { - Reader + AncientReader Writer Batcher Iteratee -- cgit v1.2.3 From b6cac42e9ffc0b19a1e70416db85593f1cb0d30c Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 14 Mar 2019 14:59:47 +0800 Subject: core/rawdb: add file lock for freezer --- ethdb/database.go | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 764e304e3..01483f3d4 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -80,6 +80,13 @@ type AncientReader interface { Ancienter } +// AncientStore contains all the methods required to allow handling different +// ancient data stores backing immutable chain data store. +type AncientStore interface { + Ancienter + io.Closer +} + // Database contains all the methods required by the high level database to not // only access the key-value data store but also the chain freezer. type Database interface { -- cgit v1.2.3 From 80469bea0cc6dbfae749d944094a7c2357dc050d Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 25 Apr 2019 22:59:48 +0800 Subject: all: integrate the freezer with fast sync * all: freezer style syncing core, eth, les, light: clean up freezer relative APIs core, eth, les, trie, ethdb, light: clean a bit core, eth, les, light: add unit tests core, light: rewrite setHead function core, eth: fix downloader unit tests core: add receipt chain insertion test core: use constant instead of hardcoding table name core: fix rollback core: fix setHead core/rawdb: remove canonical block first and then iterate side chain core/rawdb, ethdb: add hasAncient interface eth/downloader: calculate ancient limit via cht first core, eth, ethdb: lots of fixes * eth/downloader: print ancient disable log only for fast sync --- ethdb/batch.go | 4 ++-- ethdb/database.go | 56 ++++++++++++++++++++++++++++++++++------------ ethdb/leveldb/leveldb.go | 4 ++-- ethdb/memorydb/memorydb.go | 2 +- 4 files changed, 47 insertions(+), 19 deletions(-) (limited to 'ethdb') diff --git a/ethdb/batch.go b/ethdb/batch.go index a9c406354..e261415bf 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -23,7 +23,7 @@ const IdealBatchSize = 100 * 1024 // Batch is a write-only database that commits changes to its host database // when Write is called. A batch cannot be used concurrently. type Batch interface { - Writer + KeyValueWriter // ValueSize retrieves the amount of data queued up for writing. ValueSize() int @@ -35,7 +35,7 @@ type Batch interface { Reset() // Replay replays the batch contents. - Replay(w Writer) error + Replay(w KeyValueWriter) error } // Batcher wraps the NewBatch method of a backing data store. diff --git a/ethdb/database.go b/ethdb/database.go index 01483f3d4..e3eff32db 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -19,8 +19,8 @@ package ethdb import "io" -// Reader wraps the Has and Get method of a backing data store. -type Reader interface { +// KeyValueReader wraps the Has and Get method of a backing data store. +type KeyValueReader interface { // Has retrieves if a key is present in the key-value data store. Has(key []byte) (bool, error) @@ -28,8 +28,8 @@ type Reader interface { Get(key []byte) ([]byte, error) } -// Writer wraps the Put method of a backing data store. -type Writer interface { +// KeyValueWriter wraps the Put method of a backing data store. +type KeyValueWriter interface { // Put inserts the given value into the key-value data store. Put(key []byte, value []byte) error @@ -58,8 +58,8 @@ type Compacter interface { // KeyValueStore contains all the methods required to allow handling different // key-value data stores backing the high level database. type KeyValueStore interface { - Reader - Writer + KeyValueReader + KeyValueWriter Batcher Iteratee Stater @@ -67,30 +67,58 @@ type KeyValueStore interface { io.Closer } -// Ancienter wraps the Ancient method for a backing immutable chain data store. -type Ancienter interface { +// AncientReader contains the methods required to read from immutable ancient data. +type AncientReader interface { + // HasAncient returns an indicator whether the specified data exists in the + // ancient store. + HasAncient(kind string, number uint64) (bool, error) + // Ancient retrieves an ancient binary blob from the append-only immutable files. Ancient(kind string, number uint64) ([]byte, error) + + // Ancients returns the ancient store length + Ancients() (uint64, error) +} + +// AncientWriter contains the methods required to write to immutable ancient data. +type AncientWriter interface { + // AppendAncient injects all binary blobs belong to block at the end of the + // append-only immutable table files. + AppendAncient(number uint64, hash, header, body, receipt, td []byte) error + + // TruncateAncients discards all but the first n ancient data from the ancient store. + TruncateAncients(n uint64) error + + // Sync flushes all in-memory ancient store data to disk. + Sync() error } -// AncientReader contains the methods required to access both key-value as well as +// Reader contains the methods required to read data from both key-value as well as // immutable ancient data. -type AncientReader interface { - Reader - Ancienter +type Reader interface { + KeyValueReader + AncientReader +} + +// Writer contains the methods required to write data to both key-value as well as +// immutable ancient data. +type Writer interface { + KeyValueWriter + AncientWriter } // AncientStore contains all the methods required to allow handling different // ancient data stores backing immutable chain data store. type AncientStore interface { - Ancienter + AncientReader + AncientWriter io.Closer } // Database contains all the methods required by the high level database to not // only access the key-value data store but also the chain freezer. type Database interface { - AncientReader + Reader Writer Batcher Iteratee diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 8eabee50e..f74e94d92 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -425,13 +425,13 @@ func (b *batch) Reset() { } // Replay replays the batch contents. -func (b *batch) Replay(w ethdb.Writer) error { +func (b *batch) Replay(w ethdb.KeyValueWriter) error { return b.b.Replay(&replayer{writer: w}) } // replayer is a small wrapper to implement the correct replay methods. type replayer struct { - writer ethdb.Writer + writer ethdb.KeyValueWriter failure error } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index cb8b27f3b..caa9b02a1 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -270,7 +270,7 @@ func (b *batch) Reset() { } // Replay replays the batch contents. -func (b *batch) Replay(w ethdb.Writer) error { +func (b *batch) Replay(w ethdb.KeyValueWriter) error { for _, keyvalue := range b.writes { if keyvalue.delete { if err := w.Delete(keyvalue.key); err != nil { -- cgit v1.2.3 From 37d280da411eb649ce22ab69827ac5aacd46534b Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 14 May 2019 22:07:44 +0800 Subject: core, cmd, vendor: fixes and database inspection tool (#15) * core, eth: some fixes for freezer * vendor, core/rawdb, cmd/geth: add db inspector * core, cmd/utils: check ancient store path forceily * cmd/geth, common, core/rawdb: a few fixes * cmd/geth: support windows file rename and fix rename error * core: support ancient plugin * core, cmd: streaming file copy * cmd, consensus, core, tests: keep genesis in leveldb * core: write txlookup during ancient init * core: bump database version --- ethdb/database.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index e3eff32db..1ba169bcf 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -76,8 +76,11 @@ type AncientReader interface { // Ancient retrieves an ancient binary blob from the append-only immutable files. Ancient(kind string, number uint64) ([]byte, error) - // Ancients returns the ancient store length + // Ancients returns the ancient item numbers in the ancient store. Ancients() (uint64, error) + + // AncientSize returns the ancient size of the specified category. + AncientSize(kind string) (uint64, error) } // AncientWriter contains the methods required to write to immutable ancient data. -- cgit v1.2.3