core: total-ordering: change early flag to mode (#227)

* core: total-ordering: change early flag to mode

10 files changed, 64 insertions, 49 deletions

diff --git a/core/consensus_test.go b/core/consensus_test.go
index 94c5018..ed1658f 100644
--- a/core/consensus_test.go
+++ b/core/consensus_test.go
@@ -361,22 +361,22 @@ func (s *ConsensusTestSuite) TestSimpleDeliverBlock() {
 		sort.Sort(delivered0)
 		req.Len(app.TotalOrdered, 4)
 		req.Equal(app.TotalOrdered[0].BlockHashes, delivered0)
-		req.False(app.TotalOrdered[0].Early)
+		req.Equal(app.TotalOrdered[0].Mode, TotalOrderingModeNormal)
 		// b11 is the sencond set delivered by total ordering.
 		delivered1 := common.Hashes{b11.Hash}
 		sort.Sort(delivered1)
 		req.Equal(app.TotalOrdered[1].BlockHashes, delivered1)
-		req.False(app.TotalOrdered[1].Early)
+		req.Equal(app.TotalOrdered[1].Mode, TotalOrderingModeNormal)
 		// b01, b21, b31 are the third set delivered by total ordering.
 		delivered2 := common.Hashes{b01.Hash, b21.Hash, b31.Hash}
 		sort.Sort(delivered2)
 		req.Equal(app.TotalOrdered[2].BlockHashes, delivered2)
-		req.False(app.TotalOrdered[2].Early)
+		req.Equal(app.TotalOrdered[2].Mode, TotalOrderingModeNormal)
 		// b02, b12, b22, b32 are the fourth set delivered by total ordering.
 		delivered3 := common.Hashes{b02.Hash, b12.Hash, b22.Hash, b32.Hash}
 		sort.Sort(delivered3)
 		req.Equal(app.TotalOrdered[3].BlockHashes, delivered3)
-		req.False(app.TotalOrdered[3].Early)
+		req.Equal(app.TotalOrdered[3].Mode, TotalOrderingModeNormal)
 		// Check generated timestamps.
 		req.Contains(app.Delivered, b00.Hash)
 		req.Contains(app.Delivered, b10.Hash)
diff --git a/core/interfaces.go b/core/interfaces.go
index 4a20354..97ab0a3 100644
--- a/core/interfaces.go
+++ b/core/interfaces.go
@@ -52,7 +52,7 @@ type Debug interface {
 
 	// TotalOrderingDelivered is called when the total ordering algorithm deliver
 	// a set of block.
-	TotalOrderingDelivered(common.Hashes, bool)
+	TotalOrderingDelivered(common.Hashes, uint32)
 }
 
 // Network describs the network interface that interacts with DEXON consensus
diff --git a/core/lattice.go b/core/lattice.go
index 82ebdc6..e6f9848 100644
--- a/core/lattice.go
+++ b/core/lattice.go
@@ -136,10 +136,10 @@ func (s *Lattice) ProcessBlock(
 	input *types.Block) (verified, delivered []*types.Block, err error) {
 
 	var (
-		tip, b         *types.Block
-		toDelivered    []*types.Block
-		inLattice      []*types.Block
-		earlyDelivered bool
+		tip, b        *types.Block
+		toDelivered   []*types.Block
+		inLattice     []*types.Block
+		deliveredMode uint32
 	)
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -172,7 +172,7 @@ func (s *Lattice) ProcessBlock(
 	}
 	// Perform total ordering for each block added to lattice.
 	for _, b = range inLattice {
-		toDelivered, earlyDelivered, err = s.toModule.processBlock(b)
+		toDelivered, deliveredMode, err = s.toModule.processBlock(b)
 		if err != nil {
 			// All errors from total ordering is serious, should panic.
 			panic(err)
@@ -185,7 +185,7 @@ func (s *Lattice) ProcessBlock(
 			hashes[idx] = toDelivered[idx].Hash
 		}
 		if s.debug != nil {
-			s.debug.TotalOrderingDelivered(hashes, earlyDelivered)
+			s.debug.TotalOrderingDelivered(hashes, deliveredMode)
 		}
 		// Perform timestamp generation.
 		if err = s.ctModule.processBlocks(toDelivered); err != nil {
diff --git a/core/nonblocking.go b/core/nonblocking.go
index e95a035..675675b 100644
--- a/core/nonblocking.go
+++ b/core/nonblocking.go
@@ -35,7 +35,7 @@ type stronglyAckedEvent struct {
 
 type totalOrderingDeliveredEvent struct {
 	blockHashes common.Hashes
-	early       bool
+	mode        uint32
 }
 
 type blockDeliveredEvent struct {
@@ -97,7 +97,7 @@ func (nb *nonBlocking) run() {
 		case blockConfirmedEvent:
 			nb.app.BlockConfirmed(*e.block)
 		case totalOrderingDeliveredEvent:
-			nb.debug.TotalOrderingDelivered(e.blockHashes, e.early)
+			nb.debug.TotalOrderingDelivered(e.blockHashes, e.mode)
 		case blockDeliveredEvent:
 			nb.app.BlockDelivered(e.blockHash, *e.result)
 		default:
@@ -148,9 +148,9 @@ func (nb *nonBlocking) StronglyAcked(blockHash common.Hash) {
 // TotalOrderingDelivered is called when the total ordering algorithm deliver
 // a set of block.
 func (nb *nonBlocking) TotalOrderingDelivered(
-	blockHashes common.Hashes, early bool) {
+	blockHashes common.Hashes, mode uint32) {
 	if nb.debug != nil {
-		nb.addEvent(totalOrderingDeliveredEvent{blockHashes, early})
+		nb.addEvent(totalOrderingDeliveredEvent{blockHashes, mode})
 	}
 }
 
diff --git a/core/nonblocking_test.go b/core/nonblocking_test.go
index cad7def..b1f1cb5 100644
--- a/core/nonblocking_test.go
+++ b/core/nonblocking_test.go
@@ -68,7 +68,7 @@ func (app *slowApp) StronglyAcked(blockHash common.Hash) {
 	app.stronglyAcked[blockHash] = struct{}{}
 }
 
-func (app *slowApp) TotalOrderingDelivered(blockHashes common.Hashes, early bool) {
+func (app *slowApp) TotalOrderingDelivered(blockHashes common.Hashes, mode uint32) {
 	time.Sleep(app.sleep)
 	for _, hash := range blockHashes {
 		app.totalOrderingDelivered[hash] = struct{}{}
@@ -140,7 +140,7 @@ func (s *NonBlockingTestSuite) TestNonBlocking() {
 		nbModule.StronglyAcked(hash)
 		nbModule.BlockDelivered(hash, types.FinalizationResult{})
 	}
-	nbModule.TotalOrderingDelivered(hashes, true)
+	nbModule.TotalOrderingDelivered(hashes, TotalOrderingModeEarly)
 
 	// nonBlocking should be non-blocking.
 	s.True(shouldFinish.After(time.Now().UTC()))
diff --git a/core/test/app.go b/core/test/app.go
index d9582c9..a48b3c8 100644
--- a/core/test/app.go
+++ b/core/test/app.go
@@ -62,7 +62,7 @@ type AppAckedRecord struct {
 // a total-ordering deliver notification.
 type AppTotalOrderRecord struct {
 	BlockHashes common.Hashes
-	Early       bool
+	Mode        uint32
 	When        time.Time
 }
 
@@ -126,13 +126,13 @@ func (app *App) StronglyAcked(blockHash common.Hash) {
 }
 
 // TotalOrderingDelivered implements Application interface.
-func (app *App) TotalOrderingDelivered(blockHashes common.Hashes, early bool) {
+func (app *App) TotalOrderingDelivered(blockHashes common.Hashes, mode uint32) {
 	app.totalOrderedLock.Lock()
 	defer app.totalOrderedLock.Unlock()
 
 	rec := &AppTotalOrderRecord{
 		BlockHashes: blockHashes,
-		Early:       early,
+		Mode:        mode,
 		When:        time.Now().UTC(),
 	}
 	app.TotalOrdered = append(app.TotalOrdered, rec)
diff --git a/core/test/app_test.go b/core/test/app_test.go
index a70fc82..8f2aae5 100644
--- a/core/test/app_test.go
+++ b/core/test/app_test.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/dexon-foundation/dexon-consensus-core/common"
+	"github.com/dexon-foundation/dexon-consensus-core/core"
 	"github.com/dexon-foundation/dexon-consensus-core/core/types"
 	"github.com/stretchr/testify/suite"
 )
@@ -38,7 +39,7 @@ func (s *AppTestSuite) SetupSuite() {
 			common.NewRandomHash(),
 			common.NewRandomHash(),
 		},
-		Early: false,
+		Mode: core.TotalOrderingModeNormal,
 	}
 	s.to2 = &AppTotalOrderRecord{
 		BlockHashes: common.Hashes{
@@ -46,13 +47,13 @@ func (s *AppTestSuite) SetupSuite() {
 			common.NewRandomHash(),
 			common.NewRandomHash(),
 		},
-		Early: false,
+		Mode: core.TotalOrderingModeNormal,
 	}
 	s.to3 = &AppTotalOrderRecord{
 		BlockHashes: common.Hashes{
 			common.NewRandomHash(),
 		},
-		Early: false,
+		Mode: core.TotalOrderingModeNormal,
 	}
 }
 
@@ -62,7 +63,7 @@ func (s *AppTestSuite) setupAppByTotalOrderDeliver(
 	for _, h := range to.BlockHashes {
 		app.StronglyAcked(h)
 	}
-	app.TotalOrderingDelivered(to.BlockHashes, to.Early)
+	app.TotalOrderingDelivered(to.BlockHashes, to.Mode)
 	for _, h := range to.BlockHashes {
 		// To make it simpler, use the index of hash sequence
 		// as the time.
@@ -98,7 +99,7 @@ func (s *AppTestSuite) TestCompare() {
 	s.setupAppByTotalOrderDeliver(app2, s.to2)
 	hash := common.NewRandomHash()
 	app2.StronglyAcked(hash)
-	app2.TotalOrderingDelivered(common.Hashes{hash}, false)
+	app2.TotalOrderingDelivered(common.Hashes{hash}, core.TotalOrderingModeNormal)
 	s.deliverBlockWithTimeFromSequenceLength(app2, hash)
 	req.Equal(ErrMismatchBlockHashSequence, app1.Compare(app2))
 	// An App with different consensus time for the same block.
@@ -108,7 +109,7 @@ func (s *AppTestSuite) TestCompare() {
 	for _, h := range s.to3.BlockHashes {
 		app3.StronglyAcked(h)
 	}
-	app3.TotalOrderingDelivered(s.to3.BlockHashes, s.to3.Early)
+	app3.TotalOrderingDelivered(s.to3.BlockHashes, s.to3.Mode)
 	wrongTime := time.Time{}.Add(
 		time.Duration(len(app3.DeliverSequence)) * time.Second)
 	wrongTime = wrongTime.Add(1 * time.Second)
@@ -139,7 +140,7 @@ func (s *AppTestSuite) TestVerify() {
 	for _, h := range s.to2.BlockHashes {
 		app2.StronglyAcked(h)
 	}
-	app2.TotalOrderingDelivered(s.to2.BlockHashes, s.to2.Early)
+	app2.TotalOrderingDelivered(s.to2.BlockHashes, s.to2.Mode)
 	s.deliverBlock(app2, s.to2.BlockHashes[0], time.Time{})
 	req.Equal(ErrConsensusTimestampOutOfOrder, app2.Verify())
 	// A delivered block is not found in total ordering delivers.
@@ -155,10 +156,10 @@ func (s *AppTestSuite) TestVerify() {
 	for _, h := range s.to2.BlockHashes {
 		app4.StronglyAcked(h)
 	}
-	app4.TotalOrderingDelivered(s.to2.BlockHashes, s.to2.Early)
+	app4.TotalOrderingDelivered(s.to2.BlockHashes, s.to2.Mode)
 	hash = common.NewRandomHash()
 	app4.StronglyAcked(hash)
-	app4.TotalOrderingDelivered(common.Hashes{hash}, false)
+	app4.TotalOrderingDelivered(common.Hashes{hash}, core.TotalOrderingModeNormal)
 	s.deliverBlockWithTimeFromSequenceLength(app4, hash)
 	// Witness ack on unknown block.
 	app5 := NewApp()
diff --git a/core/test/stopper_test.go b/core/test/stopper_test.go
index e9954b0..7678f99 100644
--- a/core/test/stopper_test.go
+++ b/core/test/stopper_test.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/dexon-foundation/dexon-consensus-core/common"
+	"github.com/dexon-foundation/dexon-consensus-core/core"
 	"github.com/dexon-foundation/dexon-consensus-core/core/blockdb"
 	"github.com/dexon-foundation/dexon-consensus-core/core/types"
 	"github.com/stretchr/testify/suite"
@@ -59,7 +60,7 @@ func (s *StopperTestSuite) TestStopByConfirmedBlocks() {
 			for _, h := range hashes {
 				app.StronglyAcked(h)
 			}
-			app.TotalOrderingDelivered(hashes, false)
+			app.TotalOrderingDelivered(hashes, core.TotalOrderingModeNormal)
 			for _, h := range hashes {
 				app.BlockDelivered(h, types.FinalizationResult{
 					Timestamp: time.Time{},
diff --git a/core/total-ordering.go b/core/total-ordering.go
index 182ec6c..a1c190f 100644
--- a/core/total-ordering.go
+++ b/core/total-ordering.go
@@ -32,6 +32,17 @@ const (
 	infinity uint64 = math.MaxUint64
 )
 
+const (
+	// TotalOrderingModeError returns mode error.
+	TotalOrderingModeError uint32 = iota
+	// TotalOrderingModeNormal returns mode normal.
+	TotalOrderingModeNormal
+	// TotalOrderingModeEarly returns mode early.
+	TotalOrderingModeEarly
+	// TotalOrderingModeFlush returns mode flush.
+	TotalOrderingModeFlush
+)
+
 var (
 	// ErrNotValidDAG would be reported when block subbmitted to totalOrdering
 	// didn't form a DAG.
@@ -1040,13 +1051,14 @@ func (to *totalOrdering) output(
 //  - generate preceding set
 //  - check if the preceding set deliverable by checking potential function
 func (to *totalOrdering) generateDeliverSet() (
-	delivered map[common.Hash]struct{}, early bool) {
+	delivered map[common.Hash]struct{}, mode uint32) {
 	var (
 		chainID, otherChainID uint32
 		info, otherInfo       *totalOrderingCandidateInfo
 		precedings            = make(map[uint32]struct{})
 		cfg                   = to.configs[to.curRound]
 	)
+	mode = TotalOrderingModeNormal
 	to.globalVector.updateCandidateInfo(to.dirtyChainIDs, to.objCache)
 	globalInfo := to.globalVector.cachedCandidateInfo
 	for _, chainID = range to.candidateChainIDs {
@@ -1173,7 +1185,7 @@ CheckNextCandidateLoop:
 		// The whole picture is not ready, we need to check if
 		// exteranl stability is met, and we can deliver earlier.
 		if checkAHV() && checkANS() {
-			early = true
+			mode = TotalOrderingModeEarly
 		} else {
 			return
 		}
@@ -1187,8 +1199,9 @@ CheckNextCandidateLoop:
 
 // flushBlocks flushes blocks.
 func (to *totalOrdering) flushBlocks(
-	b *types.Block) (flushed []*types.Block, early bool, err error) {
+	b *types.Block) (flushed []*types.Block, mode uint32, err error) {
 	cfg := to.configs[to.curRound]
+	mode = TotalOrderingModeFlush
 	if cfg.isValidLastBlock(b) {
 		to.flushReadyChains[b.Position.ChainID] = struct{}{}
 	}
@@ -1252,8 +1265,8 @@ func (to *totalOrdering) flushBlocks(
 
 // deliverBlocks delivers blocks by DEXON total ordering algorithm.
 func (to *totalOrdering) deliverBlocks() (
-	delivered []*types.Block, early bool, err error) {
-	hashes, early := to.generateDeliverSet()
+	delivered []*types.Block, mode uint32, err error) {
+	hashes, mode := to.generateDeliverSet()
 	cfg := to.configs[to.curRound]
 	// output precedings
 	delivered = to.output(hashes, cfg.numChains)
@@ -1293,7 +1306,7 @@ func (to *totalOrdering) deliverBlocks() (
 
 // processBlock is the entry point of totalOrdering.
 func (to *totalOrdering) processBlock(
-	b *types.Block) ([]*types.Block, bool, error) {
+	b *types.Block) ([]*types.Block, uint32, error) {
 	// NOTE: I assume the block 'b' is already safe for total ordering.
 	//       That means, all its acking blocks are during/after
 	//       total ordering stage.
@@ -1302,7 +1315,7 @@ func (to *totalOrdering) processBlock(
 	to.buildBlockRelation(b)
 	pos, err := to.updateVectors(b)
 	if err != nil {
-		return nil, false, err
+		return nil, uint32(0), err
 	}
 	// Mark the proposer of incoming block as dirty.
 	if b.Position.ChainID < cfg.numChains {
diff --git a/core/total-ordering_test.go b/core/total-ordering_test.go
index 83abd58..94fc619 100644
--- a/core/total-ordering_test.go
+++ b/core/total-ordering_test.go
@@ -100,9 +100,9 @@ func (s *TotalOrderingTestSuite) checkRandomResult(
 }
 
 func (s *TotalOrderingTestSuite) checkNotDeliver(to *totalOrdering, b *types.Block) {
-	blocks, eqrly, err := to.processBlock(b)
+	blocks, mode, err := to.processBlock(b)
 	s.Empty(blocks)
-	s.False(eqrly)
+	s.Equal(mode, TotalOrderingModeNormal)
 	s.Nil(err)
 }
 
@@ -443,9 +443,9 @@ func (s *TotalOrderingTestSuite) TestEarlyDeliver() {
 	s.Equal(candidate.ackedStatus[3].minHeight, b30.Position.Height)
 	s.Equal(candidate.ackedStatus[3].count, uint64(2))
 
-	blocks, early, err := to.processBlock(b32)
+	blocks, mode, err := to.processBlock(b32)
 	s.Require().Len(blocks, 1)
-	s.True(early)
+	s.Equal(mode, TotalOrderingModeEarly)
 	s.Nil(err)
 	s.checkHashSequence(blocks, common.Hashes{b00.Hash})
 
@@ -706,8 +706,8 @@ func (s *TotalOrderingTestSuite) TestBasicCaseForK2() {
 	s.Equal(candidate.ackedStatus[4].count, uint64(0))
 
 	// Check the first deliver.
-	blocks, early, err := to.processBlock(b02)
-	s.True(early)
+	blocks, mode, err := to.processBlock(b02)
+	s.Equal(mode, TotalOrderingModeEarly)
 	s.Nil(err)
 	s.checkHashSequence(blocks, common.Hashes{b00.Hash, b10.Hash})
 
@@ -747,8 +747,8 @@ func (s *TotalOrderingTestSuite) TestBasicCaseForK2() {
 	s.checkNotDeliver(to, b13)
 
 	// Check the second deliver.
-	blocks, early, err = to.processBlock(b03)
-	s.True(early)
+	blocks, mode, err = to.processBlock(b03)
+	s.Equal(mode, TotalOrderingModeEarly)
 	s.Nil(err)
 	s.checkHashSequence(blocks, common.Hashes{b11.Hash, b20.Hash})
 
@@ -799,8 +799,8 @@ func (s *TotalOrderingTestSuite) TestBasicCaseForK2() {
 
 	// Make 'Acking Node Set' contains blocks from all chains,
 	// this should trigger not-early deliver.
-	blocks, early, err = to.processBlock(b23)
-	s.False(early)
+	blocks, mode, err = to.processBlock(b23)
+	s.Equal(mode, TotalOrderingModeNormal)
 	s.Nil(err)
 	s.checkHashSequence(blocks, common.Hashes{b01.Hash, b30.Hash})
 
@@ -916,8 +916,8 @@ func (s *TotalOrderingTestSuite) TestBasicCaseForK0() {
 	req.Equal(candidate.ackedStatus[3].count, uint64(2))
 
 	// This new block should trigger non-early deliver.
-	blocks, early, err := to.processBlock(b40)
-	req.False(early)
+	blocks, mode, err := to.processBlock(b40)
+	req.Equal(mode, TotalOrderingModeNormal)
 	req.Nil(err)
 	s.checkHashSequence(blocks, common.Hashes{b20.Hash})