diff options
Diffstat (limited to 'core/db')
| -rw-r--r-- | core/db/interfaces.go | 70 | ||||
| -rw-r--r-- | core/db/level-db.go | 127 | ||||
| -rw-r--r-- | core/db/level-db_test.go | 132 | ||||
| -rw-r--r-- | core/db/memory.go | 185 | ||||
| -rw-r--r-- | core/db/memory_test.go | 134 |
5 files changed, 648 insertions, 0 deletions
diff --git a/core/db/interfaces.go b/core/db/interfaces.go new file mode 100644 index 0000000..5e13dc6 --- /dev/null +++ b/core/db/interfaces.go @@ -0,0 +1,70 @@ +// Copyright 2018 The dexon-consensus Authors +// This file is part of the dexon-consensus library. +// +// The dexon-consensus library is free software: you can redistribute it +// and/or modify it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, either version 3 of the License, +// or (at your option) any later version. +// +// The dexon-consensus library is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the dexon-consensus library. If not, see +// <http://www.gnu.org/licenses/>. + +package db + +import ( + "errors" + "fmt" + + "github.com/dexon-foundation/dexon-consensus/common" + "github.com/dexon-foundation/dexon-consensus/core/types" +) + +var ( + // ErrBlockExists is the error when block eixsts. + ErrBlockExists = errors.New("block exists") + // ErrBlockDoesNotExist is the error when block does not eixst. + ErrBlockDoesNotExist = errors.New("block does not exist") + // ErrIterationFinished is the error to check if the iteration is finished. + ErrIterationFinished = errors.New("iteration finished") + // ErrEmptyPath is the error when the required path is empty. + ErrEmptyPath = fmt.Errorf("empty path") + // ErrClosed is the error when using DB after it's closed. + ErrClosed = fmt.Errorf("db closed") + // ErrNotImplemented is the error that some interface is not implemented. + ErrNotImplemented = fmt.Errorf("not implemented") +) + +// Database is the interface for a Database. +type Database interface { + Reader + Writer + + // Close allows database implementation able to + // release resource when finishing. + Close() error +} + +// Reader defines the interface for reading blocks into DB. +type Reader interface { + HasBlock(hash common.Hash) bool + GetBlock(hash common.Hash) (types.Block, error) + GetAllBlocks() (BlockIterator, error) +} + +// Writer defines the interface for writing blocks into DB. +type Writer interface { + UpdateBlock(block types.Block) error + PutBlock(block types.Block) error +} + +// BlockIterator defines an iterator on blocks hold +// in a DB. +type BlockIterator interface { + NextBlock() (types.Block, error) +} diff --git a/core/db/level-db.go b/core/db/level-db.go new file mode 100644 index 0000000..6983d3a --- /dev/null +++ b/core/db/level-db.go @@ -0,0 +1,127 @@ +// Copyright 2018 The dexon-consensus Authors +// This file is part of the dexon-consensus library. +// +// The dexon-consensus library is free software: you can redistribute it +// and/or modify it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, either version 3 of the License, +// or (at your option) any later version. +// +// The dexon-consensus library is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the dexon-consensus library. If not, see +// <http://www.gnu.org/licenses/>. + +package db + +import ( + "encoding/json" + + "github.com/syndtr/goleveldb/leveldb" + + "github.com/dexon-foundation/dexon-consensus/common" + "github.com/dexon-foundation/dexon-consensus/core/types" +) + +// LevelDBBackedDB is a leveldb backed DB implementation. +type LevelDBBackedDB struct { + db *leveldb.DB +} + +// NewLevelDBBackedDB initialize a leveldb-backed database. +func NewLevelDBBackedDB( + path string) (lvl *LevelDBBackedDB, err error) { + + dbInst, err := leveldb.OpenFile(path, nil) + if err != nil { + return + } + lvl = &LevelDBBackedDB{db: dbInst} + return +} + +// Close implement Closer interface, which would release allocated resource. +func (lvl *LevelDBBackedDB) Close() error { + return lvl.db.Close() +} + +// HasBlock implements the Reader.Has method. +func (lvl *LevelDBBackedDB) HasBlock(hash common.Hash) bool { + exists, err := lvl.db.Has([]byte(hash[:]), nil) + if err != nil { + // TODO(missionliao): Modify the interface to return error. + panic(err) + } + return exists +} + +// GetBlock implements the Reader.GetBlock method. +func (lvl *LevelDBBackedDB) GetBlock( + hash common.Hash) (block types.Block, err error) { + + queried, err := lvl.db.Get([]byte(hash[:]), nil) + if err != nil { + if err == leveldb.ErrNotFound { + err = ErrBlockDoesNotExist + } + return + } + err = json.Unmarshal(queried, &block) + if err != nil { + return + } + return +} + +// UpdateBlock implements the Writer.UpdateBlock method. +func (lvl *LevelDBBackedDB) UpdateBlock(block types.Block) (err error) { + // NOTE: we didn't handle changes of block hash (and it + // should not happen). + marshaled, err := json.Marshal(&block) + if err != nil { + return + } + + if !lvl.HasBlock(block.Hash) { + err = ErrBlockDoesNotExist + return + } + err = lvl.db.Put( + []byte(block.Hash[:]), + marshaled, + nil) + if err != nil { + return + } + return +} + +// PutBlock implements the Writer.PutBlock method. +func (lvl *LevelDBBackedDB) PutBlock(block types.Block) (err error) { + marshaled, err := json.Marshal(&block) + if err != nil { + return + } + if lvl.HasBlock(block.Hash) { + err = ErrBlockExists + return + } + err = lvl.db.Put( + []byte(block.Hash[:]), + marshaled, + nil) + if err != nil { + return + } + return +} + +// GetAllBlocks implements Reader.GetAllBlocks method, which allows callers +// to retrieve all blocks in DB. +func (lvl *LevelDBBackedDB) GetAllBlocks() (BlockIterator, error) { + // TODO (mission): Implement this part via goleveldb's iterator. + return nil, ErrNotImplemented +} diff --git a/core/db/level-db_test.go b/core/db/level-db_test.go new file mode 100644 index 0000000..1335d5d --- /dev/null +++ b/core/db/level-db_test.go @@ -0,0 +1,132 @@ +// Copyright 2018 The dexon-consensus Authors +// This file is part of the dexon-consensus library. +// +// The dexon-consensus library is free software: you can redistribute it +// and/or modify it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, either version 3 of the License, +// or (at your option) any later version. +// +// The dexon-consensus library is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the dexon-consensus library. If not, see +// <http://www.gnu.org/licenses/>. + +package db + +import ( + "fmt" + "testing" + "time" + + "os" + + "github.com/stretchr/testify/suite" + + "github.com/dexon-foundation/dexon-consensus/common" + "github.com/dexon-foundation/dexon-consensus/core/types" +) + +type LevelDBTestSuite struct { + suite.Suite +} + +func (s *LevelDBTestSuite) TestBasicUsage() { + dbName := fmt.Sprintf("test-db-%v.db", time.Now().UTC()) + dbInst, err := NewLevelDBBackedDB(dbName) + s.Require().Nil(err) + defer func(dbName string) { + err = dbInst.Close() + s.Nil(err) + err = os.RemoveAll(dbName) + s.Nil(err) + }(dbName) + + // Queried something from an empty database. + hash1 := common.NewRandomHash() + _, err = dbInst.GetBlock(hash1) + s.Equal(ErrBlockDoesNotExist, err) + + // Update on an empty database should not success. + node1 := types.NodeID{Hash: common.NewRandomHash()} + block1 := types.Block{ + ProposerID: node1, + Hash: hash1, + Position: types.Position{ + Height: 1, + }, + } + err = dbInst.UpdateBlock(block1) + s.Equal(ErrBlockDoesNotExist, err) + + // Put to create a new record should just work fine. + err = dbInst.PutBlock(block1) + s.Nil(err) + + // Get it back should work fine. + queried, err := dbInst.GetBlock(block1.Hash) + s.Nil(err) + s.Equal(queried.ProposerID, block1.ProposerID) + + // Test Update. + now := time.Now().UTC() + queried.Timestamp = now + + err = dbInst.UpdateBlock(queried) + s.Nil(err) + + // Try to get it back via NodeID and height. + queried, err = dbInst.GetBlock(block1.Hash) + + s.Nil(err) + s.Equal(now, queried.Timestamp) +} + +func (s *LevelDBTestSuite) TestSyncIndex() { + dbName := fmt.Sprintf("test-db-%v-si.db", time.Now().UTC()) + dbInst, err := NewLevelDBBackedDB(dbName) + s.Require().Nil(err) + defer func(dbName string) { + err = dbInst.Close() + s.Nil(err) + err = os.RemoveAll(dbName) + s.Nil(err) + }(dbName) + + // Create some blocks. + blocks := [10]types.Block{} + for i := range blocks { + block := types.Block{ + ProposerID: types.NodeID{Hash: common.NewRandomHash()}, + Hash: common.NewRandomHash(), + Position: types.Position{ + Height: uint64(i), + }, + } + dbInst.PutBlock(block) + blocks[i] = block + } + + // Save blocks to db. + err = dbInst.Close() + s.Nil(err) + + // Load back blocks(syncIndex is called). + dbInst, err = NewLevelDBBackedDB(dbName) + s.Require().Nil(err) + + // Verify result. + for _, block := range blocks { + queried, err := dbInst.GetBlock(block.Hash) + s.Nil(err) + s.Equal(block.ProposerID, queried.ProposerID) + s.Equal(block.Position.Height, queried.Position.Height) + } +} + +func TestLevelDB(t *testing.T) { + suite.Run(t, new(LevelDBTestSuite)) +} diff --git a/core/db/memory.go b/core/db/memory.go new file mode 100644 index 0000000..4246e4f --- /dev/null +++ b/core/db/memory.go @@ -0,0 +1,185 @@ +// Copyright 2018 The dexon-consensus Authors +// This file is part of the dexon-consensus library. +// +// The dexon-consensus library is free software: you can redistribute it +// and/or modify it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, either version 3 of the License, +// or (at your option) any later version. +// +// The dexon-consensus library is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the dexon-consensus library. If not, see +// <http://www.gnu.org/licenses/>. + +package db + +import ( + "encoding/json" + "io/ioutil" + "os" + "sync" + + "github.com/dexon-foundation/dexon-consensus/common" + "github.com/dexon-foundation/dexon-consensus/core/types" +) + +type blockSeqIterator struct { + idx int + db *MemBackedDB +} + +// NextBlock implemenets BlockIterator.NextBlock method. +func (seq *blockSeqIterator) NextBlock() (types.Block, error) { + curIdx := seq.idx + seq.idx++ + return seq.db.getBlockByIndex(curIdx) +} + +// MemBackedDB is a memory backed DB implementation. +type MemBackedDB struct { + blocksMutex sync.RWMutex + blockHashSequence common.Hashes + blocksByHash map[common.Hash]*types.Block + persistantFilePath string +} + +// NewMemBackedDB initialize a memory-backed database. +func NewMemBackedDB(persistantFilePath ...string) ( + dbInst *MemBackedDB, err error) { + dbInst = &MemBackedDB{ + blockHashSequence: common.Hashes{}, + blocksByHash: make(map[common.Hash]*types.Block), + } + if len(persistantFilePath) == 0 || len(persistantFilePath[0]) == 0 { + return + } + dbInst.persistantFilePath = persistantFilePath[0] + buf, err := ioutil.ReadFile(dbInst.persistantFilePath) + if err != nil { + if !os.IsNotExist(err) { + // Something unexpected happened. + return + } + // It's expected behavior that file doesn't exists, we should not + // report error on it. + err = nil + return + } + + // Init this instance by file content, it's a temporary way + // to export those private field for JSON encoding. + toLoad := struct { + Sequence common.Hashes + ByHash map[common.Hash]*types.Block + }{} + err = json.Unmarshal(buf, &toLoad) + if err != nil { + return + } + dbInst.blockHashSequence = toLoad.Sequence + dbInst.blocksByHash = toLoad.ByHash + return +} + +// HasBlock returns wheter or not the DB has a block identified with the hash. +func (m *MemBackedDB) HasBlock(hash common.Hash) bool { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + _, ok := m.blocksByHash[hash] + return ok +} + +// GetBlock returns a block given a hash. +func (m *MemBackedDB) GetBlock(hash common.Hash) (types.Block, error) { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + return m.internalGetBlock(hash) +} + +func (m *MemBackedDB) internalGetBlock(hash common.Hash) (types.Block, error) { + b, ok := m.blocksByHash[hash] + if !ok { + return types.Block{}, ErrBlockDoesNotExist + } + return *b, nil +} + +// PutBlock inserts a new block into the database. +func (m *MemBackedDB) PutBlock(block types.Block) error { + if m.HasBlock(block.Hash) { + return ErrBlockExists + } + + m.blocksMutex.Lock() + defer m.blocksMutex.Unlock() + + m.blockHashSequence = append(m.blockHashSequence, block.Hash) + m.blocksByHash[block.Hash] = &block + return nil +} + +// UpdateBlock updates a block in the database. +func (m *MemBackedDB) UpdateBlock(block types.Block) error { + if !m.HasBlock(block.Hash) { + return ErrBlockDoesNotExist + } + + m.blocksMutex.Lock() + defer m.blocksMutex.Unlock() + + m.blocksByHash[block.Hash] = &block + return nil +} + +// Close implement Closer interface, which would release allocated resource. +func (m *MemBackedDB) Close() (err error) { + // Save internal state to a pretty-print json file. It's a temporary way + // to dump private file via JSON encoding. + if len(m.persistantFilePath) == 0 { + return + } + + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + toDump := struct { + Sequence common.Hashes + ByHash map[common.Hash]*types.Block + }{ + Sequence: m.blockHashSequence, + ByHash: m.blocksByHash, + } + + // Dump to JSON with 2-space indent. + buf, err := json.Marshal(&toDump) + if err != nil { + return + } + + err = ioutil.WriteFile(m.persistantFilePath, buf, 0644) + return +} + +func (m *MemBackedDB) getBlockByIndex(idx int) (types.Block, error) { + m.blocksMutex.RLock() + defer m.blocksMutex.RUnlock() + + if idx >= len(m.blockHashSequence) { + return types.Block{}, ErrIterationFinished + } + + hash := m.blockHashSequence[idx] + return m.internalGetBlock(hash) +} + +// GetAllBlocks implement Reader.GetAllBlocks method, which allows caller +// to retrieve all blocks in DB. +func (m *MemBackedDB) GetAllBlocks() (BlockIterator, error) { + return &blockSeqIterator{db: m}, nil +} diff --git a/core/db/memory_test.go b/core/db/memory_test.go new file mode 100644 index 0000000..8fee582 --- /dev/null +++ b/core/db/memory_test.go @@ -0,0 +1,134 @@ +// Copyright 2018 The dexon-consensus Authors +// This file is part of the dexon-consensus library. +// +// The dexon-consensus library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, either version 3 of the License, +// or (at your option) any later version. +// +// The dexon-consensus library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the dexon-consensus library. If not, see +// <http://www.gnu.org/licenses/>. + +package db + +import ( + "os" + "testing" + + "github.com/dexon-foundation/dexon-consensus/common" + "github.com/dexon-foundation/dexon-consensus/core/types" + "github.com/stretchr/testify/suite" +) + +type MemBackedDBTestSuite struct { + suite.Suite + + v0 types.NodeID + b00, b01, b02 *types.Block +} + +func (s *MemBackedDBTestSuite) SetupSuite() { + s.v0 = types.NodeID{Hash: common.NewRandomHash()} + + genesisHash := common.NewRandomHash() + s.b00 = &types.Block{ + ProposerID: s.v0, + ParentHash: genesisHash, + Hash: genesisHash, + Position: types.Position{ + Height: 0, + }, + Acks: common.NewSortedHashes(common.Hashes{}), + } + s.b01 = &types.Block{ + ProposerID: s.v0, + ParentHash: s.b00.Hash, + Hash: common.NewRandomHash(), + Position: types.Position{ + Height: 1, + }, + Acks: common.NewSortedHashes(common.Hashes{s.b00.Hash}), + } + s.b02 = &types.Block{ + ProposerID: s.v0, + ParentHash: s.b01.Hash, + Hash: common.NewRandomHash(), + Position: types.Position{ + Height: 2, + }, + Acks: common.NewSortedHashes(common.Hashes{s.b01.Hash}), + } +} + +func (s *MemBackedDBTestSuite) TestSaveAndLoad() { + // Make sure we are able to save/load from file. + dbPath := "test-save-and-load.db" + + // Make sure the file pointed by 'dbPath' doesn't exist. + _, err := os.Stat(dbPath) + s.Require().NotNil(err) + + dbInst, err := NewMemBackedDB(dbPath) + s.Require().Nil(err) + s.Require().NotNil(dbInst) + defer func() { + if dbInst != nil { + s.Nil(os.Remove(dbPath)) + dbInst = nil + } + }() + + s.Nil(dbInst.PutBlock(*s.b00)) + s.Nil(dbInst.PutBlock(*s.b01)) + s.Nil(dbInst.PutBlock(*s.b02)) + s.Nil(dbInst.Close()) + + // Load the json file back to check if all inserted blocks + // exists. + dbInst, err = NewMemBackedDB(dbPath) + s.Require().Nil(err) + s.Require().NotNil(dbInst) + s.True(dbInst.HasBlock(s.b00.Hash)) + s.True(dbInst.HasBlock(s.b01.Hash)) + s.True(dbInst.HasBlock(s.b02.Hash)) + s.Nil(dbInst.Close()) +} + +func (s *MemBackedDBTestSuite) TestIteration() { + // Make sure the file pointed by 'dbPath' doesn't exist. + dbInst, err := NewMemBackedDB() + s.Require().Nil(err) + s.Require().NotNil(dbInst) + + // Setup database. + s.Nil(dbInst.PutBlock(*s.b00)) + s.Nil(dbInst.PutBlock(*s.b01)) + s.Nil(dbInst.PutBlock(*s.b02)) + + // Check if we can iterate all 3 blocks. + iter, err := dbInst.GetAllBlocks() + s.Require().Nil(err) + touched := common.Hashes{} + for { + b, err := iter.NextBlock() + if err == ErrIterationFinished { + break + } + s.Require().Nil(err) + touched = append(touched, b.Hash) + } + s.Len(touched, 3) + s.Contains(touched, s.b00.Hash) + s.Contains(touched, s.b01.Hash) + s.Contains(touched, s.b02.Hash) +} + +func TestMemBackedDB(t *testing.T) { + suite.Run(t, new(MemBackedDBTestSuite)) +} |