diff options
author | Mission Liao <mission.liao@dexon.org> | 2018-07-31 18:46:24 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-07-31 18:46:24 +0800 |
commit | 3778e956013cad171cd5954686831e2598de3045 (patch) | |
tree | f1d38286fc55dbf20def9a49a67449961ce89b44 | |
parent | c9cf5953512e4503f4781d6a441404ff9dfe5660 (diff) | |
download | tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.tar tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.tar.gz tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.tar.bz2 tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.tar.lz tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.tar.xz tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.tar.zst tangerine-consensus-3778e956013cad171cd5954686831e2598de3045.zip |
blockdb: allow to dump blocks to json-encoded file
- Allow to dump blockdb to a json file
- Compared to leveldb, a json file is easier to trace.
- Add interfaces block database:
- Close would be required by database that needs cleanup.
- BlockIterator is required when we need to access 'all' blocks,
adding a new method 'GetAll' as the constructor for iterators.
- Remove GetByValidatorAndHeight from blockdb.Reader
- This function is not used anywhere, to make interface
minimum, remove it.
- Fix typo: backend -> backed
-rw-r--r-- | blockdb/interfaces.go | 23 | ||||
-rw-r--r-- | blockdb/level-db.go | 114 | ||||
-rw-r--r-- | blockdb/level-db_test.go | 15 | ||||
-rw-r--r-- | blockdb/memory.go | 151 | ||||
-rw-r--r-- | blockdb/memory_test.go | 115 | ||||
-rw-r--r-- | core/blocklattice_test.go | 4 | ||||
-rw-r--r-- | simulation/simulation.go | 6 | ||||
-rw-r--r-- | simulation/validator.go | 13 |
8 files changed, 294 insertions, 147 deletions
diff --git a/blockdb/interfaces.go b/blockdb/interfaces.go index c2cf8a6..fd176bc 100644 --- a/blockdb/interfaces.go +++ b/blockdb/interfaces.go @@ -19,6 +19,7 @@ package blockdb import ( "errors" + "fmt" "github.com/dexon-foundation/dexon-consensus-core/common" "github.com/dexon-foundation/dexon-consensus-core/core/types" @@ -29,21 +30,31 @@ var ( ErrBlockExists = errors.New("block exists") // ErrBlockDoesNotExist is the error when block does not eixst. ErrBlockDoesNotExist = errors.New("block does not exist") - // ErrValidatorDoesNotExist is the error when validator does not eixst. - ErrValidatorDoesNotExist = errors.New("validator does not exist") + // ErrIterationFinished is the error to check if the iteration is finished. + ErrIterationFinished = errors.New("iteration finished") + // ErrEmptyPath is the error when the required path is empty. + ErrEmptyPath = fmt.Errorf("empty path") + // ErrClosed is the error when using DB after it's closed. + ErrClosed = fmt.Errorf("db closed") + // ErrNotImplemented is the error that some interface is not implemented. + ErrNotImplemented = fmt.Errorf("not implemented") ) // BlockDatabase is the interface for a BlockDatabase. type BlockDatabase interface { Reader Writer + + // Close allows database implementation able to + // release resource when finishing. + Close() error } // Reader defines the interface for reading blocks into DB. type Reader interface { Has(hash common.Hash) bool Get(hash common.Hash) (types.Block, error) - GetByValidatorAndHeight(vID types.ValidatorID, height uint64) (types.Block, error) + GetAll() (BlockIterator, error) } // Writer defines the interface for writing blocks into DB. @@ -51,3 +62,9 @@ type Writer interface { Update(block types.Block) error Put(block types.Block) error } + +// BlockIterator defines an iterator on blocks hold +// in a DB. +type BlockIterator interface { + Next() (types.Block, error) +} diff --git a/blockdb/level-db.go b/blockdb/level-db.go index c6d0a7b..dcceb2e 100644 --- a/blockdb/level-db.go +++ b/blockdb/level-db.go @@ -19,7 +19,6 @@ package blockdb import ( "encoding/json" - "sync" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" @@ -28,36 +27,30 @@ import ( "github.com/dexon-foundation/dexon-consensus-core/core/types" ) -// LevelDBBackendBlockDB is a leveldb backend BlockDB implementation. -type LevelDBBackendBlockDB struct { - db *leveldb.DB - index map[types.ValidatorID]map[uint64]common.Hash - indexLock sync.RWMutex +// LevelDBBackedBlockDB is a leveldb backed BlockDB implementation. +type LevelDBBackedBlockDB struct { + db *leveldb.DB } -// NewLevelDBBackendBlockDB initialize a leveldb-backed block database. -func NewLevelDBBackendBlockDB( - path string) (lvl *LevelDBBackendBlockDB, err error) { +// NewLevelDBBackedBlockDB initialize a leveldb-backed block database. +func NewLevelDBBackedBlockDB( + path string) (lvl *LevelDBBackedBlockDB, err error) { db, err := leveldb.OpenFile(path, nil) if err != nil { return } - lvl = &LevelDBBackendBlockDB{db: db} - err = lvl.syncIndex() - if err != nil { - return - } + lvl = &LevelDBBackedBlockDB{db: db} return } -// Close would release allocated resource. -func (lvl *LevelDBBackendBlockDB) Close() error { +// Close implement Closer interface, which would release allocated resource. +func (lvl *LevelDBBackedBlockDB) Close() error { return lvl.db.Close() } // Has implements the Reader.Has method. -func (lvl *LevelDBBackendBlockDB) Has(hash common.Hash) bool { +func (lvl *LevelDBBackedBlockDB) Has(hash common.Hash) bool { exists, err := lvl.db.Has([]byte(hash[:]), nil) if err != nil { // TODO(missionliao): Modify the interface to return error. @@ -67,7 +60,7 @@ func (lvl *LevelDBBackendBlockDB) Has(hash common.Hash) bool { } // Get implements the Reader.Get method. -func (lvl *LevelDBBackendBlockDB) Get( +func (lvl *LevelDBBackedBlockDB) Get( hash common.Hash) (block types.Block, err error) { queried, err := lvl.db.Get([]byte(hash[:]), nil) @@ -84,44 +77,8 @@ func (lvl *LevelDBBackendBlockDB) Get( return } -// GetByValidatorAndHeight implements -// the Reader.GetByValidatorAndHeight method. -func (lvl *LevelDBBackendBlockDB) GetByValidatorAndHeight( - vID types.ValidatorID, height uint64) (block types.Block, err error) { - - lvl.indexLock.RLock() - defer lvl.indexLock.RUnlock() - - // Get block's hash from in-memory index. - vMap, exists := lvl.index[vID] - if !exists { - err = ErrBlockDoesNotExist - return - } - hash, exists := vMap[height] - if !exists { - err = ErrBlockDoesNotExist - return - } - - // Get block from hash. - queried, err := lvl.db.Get([]byte(hash[:]), nil) - if err != nil { - if err == leveldb.ErrNotFound { - err = ErrBlockDoesNotExist - } - return - } - - err = json.Unmarshal(queried, &block) - if err != nil { - return - } - return -} - // Update implements the Writer.Update method. -func (lvl *LevelDBBackendBlockDB) Update(block types.Block) (err error) { +func (lvl *LevelDBBackedBlockDB) Update(block types.Block) (err error) { // NOTE: we didn't handle changes of block hash (and it // should not happen). marshaled, err := json.Marshal(&block) @@ -146,7 +103,7 @@ func (lvl *LevelDBBackendBlockDB) Update(block types.Block) (err error) { } // Put implements the Writer.Put method. -func (lvl *LevelDBBackendBlockDB) Put(block types.Block) (err error) { +func (lvl *LevelDBBackedBlockDB) Put(block types.Block) (err error) { marshaled, err := json.Marshal(&block) if err != nil { return @@ -167,47 +124,12 @@ func (lvl *LevelDBBackendBlockDB) Put(block types.Block) (err error) { if err != nil { return } - - // Build in-memory index. - lvl.addIndex(&block) return } -func (lvl *LevelDBBackendBlockDB) syncIndex() (err error) { - // Reset index. - lvl.index = make(map[types.ValidatorID]map[uint64]common.Hash) - - // Construct index from DB. - iter := lvl.db.NewIterator(nil, nil) - defer func() { - iter.Release() - if err == nil { - // Only return iterator's error when no error - // is presented so far. - err = iter.Error() - } - }() - - // Build index from blocks in DB, it may take time. - var block types.Block - for iter.Next() { - err = json.Unmarshal(iter.Value(), &block) - if err != nil { - return - } - lvl.addIndex(&block) - } - return -} - -func (lvl *LevelDBBackendBlockDB) addIndex(block *types.Block) { - lvl.indexLock.Lock() - defer lvl.indexLock.Unlock() - - heightMap, exists := lvl.index[block.ProposerID] - if !exists { - heightMap = make(map[uint64]common.Hash) - lvl.index[block.ProposerID] = heightMap - } - heightMap[block.Height] = block.Hash +// GetAll implements Reader.GetAll method, which allows callers +// to retrieve all blocks in DB. +func (lvl *LevelDBBackedBlockDB) GetAll() (BlockIterator, error) { + // TODO (mission): Implement this part via goleveldb's iterator. + return nil, ErrNotImplemented } diff --git a/blockdb/level-db_test.go b/blockdb/level-db_test.go index ef698f6..863adba 100644 --- a/blockdb/level-db_test.go +++ b/blockdb/level-db_test.go @@ -36,7 +36,7 @@ type LevelDBTestSuite struct { func (s *LevelDBTestSuite) TestBasicUsage() { dbName := fmt.Sprintf("test-db-%v.db", time.Now().UTC()) - db, err := NewLevelDBBackendBlockDB(dbName) + db, err := NewLevelDBBackedBlockDB(dbName) s.Require().Nil(err) defer func(dbName string) { err = db.Close() @@ -81,7 +81,7 @@ func (s *LevelDBTestSuite) TestBasicUsage() { s.Nil(err) // Try to get it back via ValidatorID and height. - queried, err = db.GetByValidatorAndHeight(block1.ProposerID, block1.Height) + queried, err = db.Get(block1.Hash) s.Nil(err) s.Equal(now, queried.Timestamps[queried.ProposerID]) @@ -89,7 +89,7 @@ func (s *LevelDBTestSuite) TestBasicUsage() { func (s *LevelDBTestSuite) TestSyncIndex() { dbName := fmt.Sprintf("test-db-%v-si.db", time.Now().UTC()) - db, err := NewLevelDBBackendBlockDB(dbName) + db, err := NewLevelDBBackedBlockDB(dbName) s.Require().Nil(err) defer func(dbName string) { err = db.Close() @@ -116,7 +116,7 @@ func (s *LevelDBTestSuite) TestSyncIndex() { s.Nil(err) // Load back blocks(syncIndex is called). - db, err = NewLevelDBBackendBlockDB(dbName) + db, err = NewLevelDBBackedBlockDB(dbName) s.Require().Nil(err) // Verify result. @@ -126,13 +126,6 @@ func (s *LevelDBTestSuite) TestSyncIndex() { s.Equal(block.ProposerID, queried.ProposerID) s.Equal(block.Height, queried.Height) } - - // Verify result using GetByValidatorAndHeight(). - for _, block := range blocks { - queried, err := db.GetByValidatorAndHeight(block.ProposerID, block.Height) - s.Nil(err) - s.Equal(block.Hash, queried.Hash) - } } func TestLevelDB(t *testing.T) { diff --git a/blockdb/memory.go b/blockdb/memory.go index 5f95d1a..eeda477 100644 --- a/blockdb/memory.go +++ b/blockdb/memory.go @@ -18,63 +18,162 @@ package blockdb import ( + "encoding/json" + "io/ioutil" + "os" + "sync" + "github.com/dexon-foundation/dexon-consensus-core/common" "github.com/dexon-foundation/dexon-consensus-core/core/types" ) -// MemBackendBlockDB is a memory backend BlockDB implementation. -type MemBackendBlockDB struct { - blocksByHash map[common.Hash]*types.Block - blocksByValidator map[types.ValidatorID]map[uint64]*types.Block +type seqIterator struct { + idx int + db *MemBackedBlockDB +} + +func (seq *seqIterator) Next() (types.Block, error) { + curIdx := seq.idx + seq.idx++ + return seq.db.getByIndex(curIdx) +} + +// MemBackedBlockDB is a memory backed BlockDB implementation. +type MemBackedBlockDB struct { + blocksMutex sync.RWMutex + blockHashSequence common.Hashes + blocksByHash map[common.Hash]*types.Block + persistantFilePath string } // NewMemBackedBlockDB initialize a memory-backed block database. -func NewMemBackedBlockDB() *MemBackendBlockDB { - return &MemBackendBlockDB{ +func NewMemBackedBlockDB(persistantFilePath ...string) (db *MemBackedBlockDB, err error) { + db = &MemBackedBlockDB{ + blockHashSequence: common.Hashes{}, blocksByHash: make(map[common.Hash]*types.Block), - blocksByValidator: make(map[types.ValidatorID]map[uint64]*types.Block), } + if len(persistantFilePath) == 0 || len(persistantFilePath[0]) == 0 { + return + } + db.persistantFilePath = persistantFilePath[0] + buf, err := ioutil.ReadFile(db.persistantFilePath) + if err != nil { + if !os.IsNotExist(err) { + // Something unexpected happened. + return + } + // It's expected behavior that file doesn't exists, we should not + // report error on it. + err = nil + return + } + + // Init this instance by file content, it's a temporary way + // to export those private field for JSON encoding. + toLoad := struct { + Sequence common.Hashes + ByHash map[common.Hash]*types.Block + }{} + err = json.Unmarshal(buf, &toLoad) + if err != nil { + return + } + db.blockHashSequence = toLoad.Sequence + db.blocksByHash = toLoad.ByHash + return } // Has returns wheter or not the DB has a block identified with the hash. -func (m *MemBackendBlockDB) Has(hash common.Hash) bool { +func (m *MemBackedBlockDB) Has(hash common.Hash) bool { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + _, ok := m.blocksByHash[hash] return ok } // Get returns a block given a hash. -func (m *MemBackendBlockDB) Get(hash common.Hash) (types.Block, error) { - b, ok := m.blocksByHash[hash] - if !ok { - return types.Block{}, ErrBlockDoesNotExist - } - return *b, nil +func (m *MemBackedBlockDB) Get(hash common.Hash) (types.Block, error) { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + return m.internalGet(hash) } -// GetByValidatorAndHeight returns a block given validator ID and hash. -func (m *MemBackendBlockDB) GetByValidatorAndHeight( - vID types.ValidatorID, height uint64) (types.Block, error) { - validatorBlocks, ok := m.blocksByValidator[vID] +func (m *MemBackedBlockDB) internalGet(hash common.Hash) (types.Block, error) { + b, ok := m.blocksByHash[hash] if !ok { - return types.Block{}, ErrValidatorDoesNotExist - } - block, ok2 := validatorBlocks[height] - if !ok2 { return types.Block{}, ErrBlockDoesNotExist } - return *block, nil + return *b, nil } // Put inserts a new block into the database. -func (m *MemBackendBlockDB) Put(block types.Block) error { +func (m *MemBackedBlockDB) Put(block types.Block) error { if m.Has(block.Hash) { return ErrBlockExists } - return m.Update(block) + + m.blocksMutex.Lock() + defer m.blocksMutex.Unlock() + + m.blockHashSequence = append(m.blockHashSequence, block.Hash) + m.blocksByHash[block.Hash] = &block + return nil } // Update updates a block in the database. -func (m *MemBackendBlockDB) Update(block types.Block) error { +func (m *MemBackedBlockDB) Update(block types.Block) error { + m.blocksMutex.Lock() + defer m.blocksMutex.Unlock() + m.blocksByHash[block.Hash] = &block return nil } + +// Close implement Closer interface, which would release allocated resource. +func (m *MemBackedBlockDB) Close() (err error) { + // Save internal state to a pretty-print json file. It's a temporary way + // to dump private file via JSON encoding. + if len(m.persistantFilePath) == 0 { + return + } + + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + toDump := struct { + Sequence common.Hashes + ByHash map[common.Hash]*types.Block + }{ + Sequence: m.blockHashSequence, + ByHash: m.blocksByHash, + } + + // Dump to JSON with 2-space indent. + buf, err := json.Marshal(&toDump) + if err != nil { + return + } + + err = ioutil.WriteFile(m.persistantFilePath, buf, 0644) + return +} + +func (m *MemBackedBlockDB) getByIndex(idx int) (types.Block, error) { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + if idx >= len(m.blockHashSequence) { + return types.Block{}, ErrIterationFinished + } + + hash := m.blockHashSequence[idx] + return m.internalGet(hash) +} + +// GetAll implement Reader.GetAll method, which allows caller +// to retrieve all blocks in DB. +func (m *MemBackedBlockDB) GetAll() (BlockIterator, error) { + return &seqIterator{db: m}, nil +} diff --git a/blockdb/memory_test.go b/blockdb/memory_test.go new file mode 100644 index 0000000..7d78f81 --- /dev/null +++ b/blockdb/memory_test.go @@ -0,0 +1,115 @@ +package blockdb + +import ( + "os" + "testing" + + "github.com/dexon-foundation/dexon-consensus-core/common" + "github.com/dexon-foundation/dexon-consensus-core/core/types" + "github.com/stretchr/testify/suite" +) + +type MemBackedBlockDBTestSuite struct { + suite.Suite + + v0 types.ValidatorID + b00, b01, b02 *types.Block +} + +func (s *MemBackedBlockDBTestSuite) SetupSuite() { + s.v0 = types.ValidatorID{Hash: common.NewRandomHash()} + + genesisHash := common.NewRandomHash() + s.b00 = &types.Block{ + ProposerID: s.v0, + ParentHash: genesisHash, + Hash: genesisHash, + Height: 0, + Acks: make(map[common.Hash]struct{}), + } + s.b01 = &types.Block{ + ProposerID: s.v0, + ParentHash: s.b00.Hash, + Hash: common.NewRandomHash(), + Height: 1, + Acks: map[common.Hash]struct{}{ + s.b00.Hash: struct{}{}, + }, + } + s.b02 = &types.Block{ + ProposerID: s.v0, + ParentHash: s.b01.Hash, + Hash: common.NewRandomHash(), + Height: 2, + Acks: map[common.Hash]struct{}{ + s.b01.Hash: struct{}{}, + }, + } +} + +func (s *MemBackedBlockDBTestSuite) TestSaveAndLoad() { + // Make sure we are able to save/load from file. + dbPath := "test-save-and-load.db" + + // Make sure the file pointed by 'dbPath' doesn't exist. + _, err := os.Stat(dbPath) + s.Require().NotNil(err) + + db, err := NewMemBackedBlockDB(dbPath) + s.Require().Nil(err) + s.Require().NotNil(db) + defer func() { + if db != nil { + s.Nil(os.Remove(dbPath)) + db = nil + } + }() + + s.Nil(db.Put(*s.b00)) + s.Nil(db.Put(*s.b01)) + s.Nil(db.Put(*s.b02)) + s.Nil(db.Close()) + + // Load the json file back to check if all inserted blocks + // exists. + db, err = NewMemBackedBlockDB(dbPath) + s.Require().Nil(err) + s.Require().NotNil(db) + s.True(db.Has(s.b00.Hash)) + s.True(db.Has(s.b01.Hash)) + s.True(db.Has(s.b02.Hash)) + s.Nil(db.Close()) +} + +func (s *MemBackedBlockDBTestSuite) TestIteration() { + // Make sure the file pointed by 'dbPath' doesn't exist. + db, err := NewMemBackedBlockDB() + s.Require().Nil(err) + s.Require().NotNil(db) + + // Setup database. + s.Nil(db.Put(*s.b00)) + s.Nil(db.Put(*s.b01)) + s.Nil(db.Put(*s.b02)) + + // Check if we can iterate all 3 blocks. + iter, err := db.GetAll() + s.Require().Nil(err) + touched := map[common.Hash]struct{}{} + for { + b, err := iter.Next() + if err == ErrIterationFinished { + break + } + s.Require().Nil(err) + touched[b.Hash] = struct{}{} + } + s.Len(touched, 3) + s.Contains(touched, s.b00.Hash) + s.Contains(touched, s.b01.Hash) + s.Contains(touched, s.b02.Hash) +} + +func TestMemBackedBlockDB(t *testing.T) { + suite.Run(t, new(MemBackedBlockDBTestSuite)) +} diff --git a/core/blocklattice_test.go b/core/blocklattice_test.go index 3dd5a3b..0f78dc5 100644 --- a/core/blocklattice_test.go +++ b/core/blocklattice_test.go @@ -68,7 +68,9 @@ func (s *BlockLatticeTest) SetupTest() { s.app = &TestApp{} - lattice = NewBlockLattice(blockdb.NewMemBackedBlockDB(), s.app) + db, err := blockdb.NewMemBackedBlockDB() + s.Require().Nil(err) + lattice = NewBlockLattice(db, s.app) for i := 0; i < 4; i++ { validators = append(validators, diff --git a/simulation/simulation.go b/simulation/simulation.go index 2ea768e..8542051 100644 --- a/simulation/simulation.go +++ b/simulation/simulation.go @@ -51,14 +51,14 @@ func Run(configPath string) { for i := 0; i < cfg.Validator.Num; i++ { id := types.ValidatorID{Hash: common.NewRandomHash()} - vs = append(vs, NewValidator(id, cfg.Validator, network, nil)) + vs = append(vs, NewValidator(id, cfg.Validator, network)) } } else if networkType == config.NetworkTypeTCPLocal { for i := 0; i < cfg.Validator.Num; i++ { id := types.ValidatorID{Hash: common.NewRandomHash()} network := NewTCPNetwork(true, cfg.Networking.PeerServer) go network.Start() - vs = append(vs, NewValidator(id, cfg.Validator, network, nil)) + vs = append(vs, NewValidator(id, cfg.Validator, network)) } } @@ -70,7 +70,7 @@ func Run(configPath string) { id := types.ValidatorID{Hash: common.NewRandomHash()} network := NewTCPNetwork(false, cfg.Networking.PeerServer) go network.Start() - v := NewValidator(id, cfg.Validator, network, nil) + v := NewValidator(id, cfg.Validator, network) go v.Run() vs = append(vs, v) } diff --git a/simulation/validator.go b/simulation/validator.go index 27e02fb..fb8afed 100644 --- a/simulation/validator.go +++ b/simulation/validator.go @@ -20,8 +20,6 @@ package simulation import ( "time" - "github.com/syndtr/goleveldb/leveldb" - "github.com/dexon-foundation/dexon-consensus-core/blockdb" "github.com/dexon-foundation/dexon-consensus-core/common" "github.com/dexon-foundation/dexon-consensus-core/core" @@ -35,7 +33,6 @@ type Validator struct { app *SimApp config config.Validator - db *leveldb.DB msgChannel chan interface{} isFinished chan struct{} @@ -51,16 +48,18 @@ type Validator struct { func NewValidator( id types.ValidatorID, config config.Validator, - network Network, - db *leveldb.DB) *Validator { + network Network) *Validator { app := NewSimApp(id, network) - lattice := core.NewBlockLattice(blockdb.NewMemBackedBlockDB(), app) + db, err := blockdb.NewMemBackedBlockDB() + if err != nil { + panic(err) + } + lattice := core.NewBlockLattice(db, app) return &Validator{ ID: id, config: config, network: network, app: app, - db: db, lattice: lattice, isFinished: make(chan struct{}), } |