core: Fix BA3.0 implmenetation (#426)

* types: Add vote type

* core: remove stateFastRollback

* core: rename lockRound to lockIter

* core: Implement real ba3.0

* core: Add test for confirm

* Fix VoteFastCom in agreement result

* lockIter will always increase

7 files changed, 67 insertions, 58 deletions

diff --git a/core/agreement-state.go b/core/agreement-state.go
index 266e442..5b2ce52 100644
--- a/core/agreement-state.go
+++ b/core/agreement-state.go
@@ -37,7 +37,6 @@ type agreementStateType int
 const (
 	stateFast agreementStateType = iota
 	stateFastVote
-	stateFastRollback
 	stateInitial
 	statePreCommit
 	stateCommit
@@ -98,23 +97,8 @@ func newFastVoteState(a *agreementData) *fastVoteState {
 }
 
 func (s *fastVoteState) state() agreementStateType { return stateFastVote }
-func (s *fastVoteState) clocks() int               { return 2 }
+func (s *fastVoteState) clocks() int               { return 3 }
 func (s *fastVoteState) nextState() (agreementState, error) {
-	return newFastRollbackState(s.a), nil
-}
-
-//----- FastRollbackState -----
-type fastRollbackState struct {
-	a *agreementData
-}
-
-func newFastRollbackState(a *agreementData) *fastRollbackState {
-	return &fastRollbackState{a: a}
-}
-
-func (s *fastRollbackState) state() agreementStateType { return stateFastRollback }
-func (s *fastRollbackState) clocks() int               { return 1 }
-func (s *fastRollbackState) nextState() (agreementState, error) {
 	return newInitialState(s.a), nil
 }
 
@@ -182,8 +166,10 @@ func (s *commitState) nextState() (agreementState, error) {
 	defer s.a.lock.Unlock()
 	hash, ok := s.a.countVoteNoLock(s.a.period, types.VotePreCom)
 	if ok && hash != skipBlockHash {
-		s.a.lockValue = hash
-		s.a.lockRound = s.a.period
+		if s.a.period > s.a.lockIter {
+			s.a.lockValue = hash
+			s.a.lockIter = s.a.period
+		}
 	} else {
 		hash = skipBlockHash
 	}
diff --git a/core/agreement-state_test.go b/core/agreement-state_test.go
index 34088b7..557193d 100644
--- a/core/agreement-state_test.go
+++ b/core/agreement-state_test.go
@@ -165,25 +165,13 @@ func (s *AgreementStateTestSuite) TestFastVoteState() {
 	a := s.newAgreement(4)
 	state := newFastVoteState(a.data)
 	s.Equal(stateFastVote, state.state())
-	s.Equal(2, state.clocks())
+	s.Equal(3, state.clocks())
 
 	// The vote proposed is not implemented inside state.
 	a.data.period = 1
 	newState, err := state.nextState()
 	s.Require().NoError(err)
 	s.Require().Len(s.voteChan, 0)
-	s.Equal(stateFastRollback, newState.state())
-}
-
-func (s *AgreementStateTestSuite) TestFastRollbackState() {
-	a := s.newAgreement(4)
-	state := newFastRollbackState(a.data)
-	s.Equal(stateFastRollback, state.state())
-	s.Equal(1, state.clocks())
-
-	a.data.period = 1
-	newState, err := state.nextState()
-	s.Require().NoError(err)
 	s.Equal(stateInitial, newState.state())
 }
 
@@ -264,7 +252,7 @@ func (s *AgreementStateTestSuite) TestCommitState() {
 	s.Require().NoError(err)
 	s.Require().Len(s.voteChan, 1)
 	s.Equal(block.Hash, a.data.lockValue)
-	s.Equal(uint64(1), a.data.lockRound)
+	s.Equal(uint64(1), a.data.lockIter)
 	vote := <-s.voteChan
 	s.Equal(types.VoteCom, vote.Type)
 	s.Equal(block.Hash, vote.BlockHash)
diff --git a/core/agreement.go b/core/agreement.go
index c1f514d..97848c5 100644
--- a/core/agreement.go
+++ b/core/agreement.go
@@ -96,7 +96,7 @@ type agreementData struct {
 	isLeader     bool
 	leader       *leaderSelector
 	lockValue    common.Hash
-	lockRound    uint64
+	lockIter     uint64
 	period       uint64
 	requiredVote int
 	votes        map[uint64][]map[types.NodeID]*types.Vote
@@ -111,6 +111,7 @@ type agreement struct {
 	data           *agreementData
 	aID            *atomic.Value
 	notarySet      map[types.NodeID]struct{}
+	hasVoteFast    bool
 	hasOutput      bool
 	lock           sync.RWMutex
 	pendingBlock   []pendingBlock
@@ -168,9 +169,10 @@ func (a *agreement) restart(
 		a.data.requiredVote = len(notarySet)/3*2 + 1
 		a.data.leader.restart(crs)
 		a.data.lockValue = nullBlockHash
-		a.data.lockRound = 0
+		a.data.lockIter = 0
 		a.data.isLeader = a.data.ID == leader
 		a.fastForward = make(chan uint64, 1)
+		a.hasVoteFast = false
 		a.hasOutput = false
 		a.state = newFastState(a.data)
 		a.notarySet = notarySet
@@ -265,7 +267,7 @@ func (a *agreement) pullVotes() bool {
 	a.data.lock.RLock()
 	defer a.data.lock.RUnlock()
 	return a.state.state() == statePullVote ||
-		a.state.state() == stateFastRollback ||
+		a.state.state() == stateInitial ||
 		(a.state.state() == statePreCommit && (a.data.period%3) == 0)
 }
 
@@ -382,12 +384,24 @@ func (a *agreement) processVote(vote *types.Vote) error {
 	}
 	a.data.votes[vote.Period][vote.Type][vote.ProposerID] = vote
 	if !a.hasOutput &&
-		(vote.Type == types.VoteCom || vote.Type == types.VoteFast) {
+		(vote.Type == types.VoteCom ||
+			vote.Type == types.VoteFast ||
+			vote.Type == types.VoteFastCom) {
 		if hash, ok := a.data.countVoteNoLock(vote.Period, vote.Type); ok &&
 			hash != skipBlockHash {
-			a.hasOutput = true
-			a.data.recv.ConfirmBlock(hash,
-				a.data.votes[vote.Period][vote.Type])
+			if vote.Type == types.VoteFast {
+				if !a.hasVoteFast {
+					a.data.recv.ProposeVote(
+						types.NewVote(types.VoteFastCom, hash, vote.Period))
+					a.data.lockValue = hash
+					a.data.lockIter = math.MaxUint64
+					a.hasVoteFast = true
+				}
+			} else {
+				a.hasOutput = true
+				a.data.recv.ConfirmBlock(hash,
+					a.data.votes[vote.Period][vote.Type])
+			}
 			return nil
 		}
 	} else if a.hasOutput {
@@ -402,16 +416,18 @@ func (a *agreement) processVote(vote *types.Vote) error {
 		if hash, ok := a.data.countVoteNoLock(vote.Period, vote.Type); ok &&
 			hash != skipBlockHash {
 			// Condition 1.
-			if a.data.period >= vote.Period && vote.Period > a.data.lockRound &&
+			if a.data.period >= vote.Period && vote.Period > a.data.lockIter &&
 				vote.BlockHash != a.data.lockValue {
 				a.data.lockValue = hash
-				a.data.lockRound = vote.Period
+				a.data.lockIter = vote.Period
 				return nil
 			}
 			// Condition 2.
 			if vote.Period > a.data.period {
-				a.data.lockValue = hash
-				a.data.lockRound = vote.Period
+				if vote.Period > a.data.lockIter {
+					a.data.lockValue = hash
+					a.data.lockIter = vote.Period
+				}
 				a.fastForward <- vote.Period
 				return nil
 			}
@@ -505,7 +521,8 @@ func (a *agreement) processBlock(block *types.Block) error {
 	}
 	a.data.blocks[block.ProposerID] = block
 	a.addCandidateBlockNoLock(block)
-	if (a.state.state() == stateFast || a.state.state() == stateFastVote) &&
+	if block.ProposerID != a.data.ID &&
+		(a.state.state() == stateFast || a.state.state() == stateFastVote) &&
 		block.ProposerID == a.leader() {
 		go func() {
 			for func() bool {
diff --git a/core/agreement_test.go b/core/agreement_test.go
index 6e2a3f4..6b74255 100644
--- a/core/agreement_test.go
+++ b/core/agreement_test.go
@@ -179,8 +179,6 @@ func (s *AgreementTestSuite) TestSimpleConfirm() {
 	a.nextState()
 	// FastVoteState
 	a.nextState()
-	// FastRollbackState
-	a.nextState()
 	// InitialState
 	a.nextState()
 	// PreCommitState
@@ -210,7 +208,7 @@ func (s *AgreementTestSuite) TestSimpleConfirm() {
 	s.Equal(types.VoteCom, vote.Type)
 	s.Equal(blockHash, vote.BlockHash)
 	s.Equal(blockHash, a.data.lockValue)
-	s.Equal(uint64(1), a.data.lockRound)
+	s.Equal(uint64(1), a.data.lockIter)
 	for nID := range s.signers {
 		v := s.copyVote(vote, nID)
 		s.Require().NoError(a.processVote(v))
@@ -229,8 +227,6 @@ func (s *AgreementTestSuite) TestPartitionOnCommitVote() {
 	a.nextState()
 	// FastVoteState
 	a.nextState()
-	// FastRollbackState
-	a.nextState()
 	// InitialState
 	a.nextState()
 	// PreCommitState
@@ -260,7 +256,7 @@ func (s *AgreementTestSuite) TestPartitionOnCommitVote() {
 	s.Equal(types.VoteCom, vote.Type)
 	s.Equal(blockHash, vote.BlockHash)
 	s.Equal(blockHash, a.data.lockValue)
-	s.Equal(uint64(1), a.data.lockRound)
+	s.Equal(uint64(1), a.data.lockIter)
 	// RepeateVoteState
 	a.nextState()
 	s.True(a.pullVotes())
@@ -281,15 +277,27 @@ func (s *AgreementTestSuite) TestFastConfirmLeader() {
 	s.Require().True(exist)
 	s.Require().Equal(s.ID, block.ProposerID)
 	s.Require().NoError(a.processBlock(block))
+	// Wait some time for go routine in processBlock to finish.
+	time.Sleep(500 * time.Millisecond)
 	s.Require().Len(s.voteChan, 1)
 	vote := <-s.voteChan
 	s.Equal(types.VoteFast, vote.Type)
 	s.Equal(blockHash, vote.BlockHash)
+	s.Require().Len(s.voteChan, 0)
 	for nID := range s.signers {
 		v := s.copyVote(vote, nID)
 		s.Require().NoError(a.processVote(v))
 	}
 	// We have enough of Fast-Votes.
+	s.Require().Len(s.voteChan, 1)
+	vote = <-s.voteChan
+	s.Equal(types.VoteFastCom, vote.Type)
+	s.Equal(blockHash, vote.BlockHash)
+	for nID := range s.signers {
+		v := s.copyVote(vote, nID)
+		s.Require().NoError(a.processVote(v))
+	}
+	// We have enough of Fast-ConfirmVotes.
 	s.Require().Len(s.confirmChan, 1)
 	confirmBlock := <-s.confirmChan
 	s.Equal(blockHash, confirmBlock)
@@ -307,6 +315,8 @@ func (s *AgreementTestSuite) TestFastConfirmNonLeader() {
 	block := s.proposeBlock(leaderNode, a.data.leader.hashCRS)
 	s.Require().Equal(leaderNode, block.ProposerID)
 	s.Require().NoError(a.processBlock(block))
+	// Wait some time for go routine in processBlock to finish.
+	time.Sleep(500 * time.Millisecond)
 	var vote *types.Vote
 	select {
 	case vote = <-s.voteChan:
@@ -320,6 +330,13 @@ func (s *AgreementTestSuite) TestFastConfirmNonLeader() {
 		s.Require().NoError(a.processVote(v))
 	}
 	// We have enough of Fast-Votes.
+	s.Require().Len(s.voteChan, 1)
+	vote = <-s.voteChan
+	for nID := range s.signers {
+		v := s.copyVote(vote, nID)
+		s.Require().NoError(a.processVote(v))
+	}
+	// We have enough of Fast-ConfirmVotes.
 	s.Require().Len(s.confirmChan, 1)
 	confirmBlock := <-s.confirmChan
 	s.Equal(block.Hash, confirmBlock)
@@ -330,7 +347,7 @@ func (s *AgreementTestSuite) TestFastForwardCond1() {
 	a, _ := s.newAgreement(4, -1, func(*types.Block) (bool, error) {
 		return true, nil
 	})
-	a.data.lockRound = 1
+	a.data.lockIter = 1
 	a.data.period = 3
 	hash := common.NewRandomHash()
 	for nID := range a.notarySet {
@@ -347,11 +364,11 @@ func (s *AgreementTestSuite) TestFastForwardCond1() {
 	default:
 	}
 	s.Equal(hash, a.data.lockValue)
-	s.Equal(uint64(2), a.data.lockRound)
+	s.Equal(uint64(2), a.data.lockIter)
 	s.Equal(uint64(3), a.data.period)
 
 	// No fast forward if vote.BlockHash == SKIP
-	a.data.lockRound = 6
+	a.data.lockIter = 6
 	a.data.period = 8
 	a.data.lockValue = nullBlockHash
 	for nID := range a.notarySet {
@@ -366,7 +383,7 @@ func (s *AgreementTestSuite) TestFastForwardCond1() {
 	}
 
 	// No fast forward if lockValue == vote.BlockHash.
-	a.data.lockRound = 11
+	a.data.lockIter = 11
 	a.data.period = 13
 	a.data.lockValue = hash
 	for nID := range a.notarySet {
@@ -402,7 +419,7 @@ func (s *AgreementTestSuite) TestFastForwardCond2() {
 		s.FailNow("Expecting fast forward.")
 	}
 	s.Equal(hash, a.data.lockValue)
-	s.Equal(uint64(2), a.data.lockRound)
+	s.Equal(uint64(2), a.data.lockIter)
 	s.Equal(uint64(2), a.data.period)
 
 	// No fast forward if vote.BlockHash == SKIP
diff --git a/core/types/vote.go b/core/types/vote.go
index 97044f5..ae86e51 100644
--- a/core/types/vote.go
+++ b/core/types/vote.go
@@ -33,6 +33,7 @@ const (
 	VotePreCom
 	VoteCom
 	VoteFast
+	VoteFastCom
 	// Do not add any type below MaxVoteType.
 	MaxVoteType
 )
diff --git a/core/utils.go b/core/utils.go
index 8579e6e..14780e7 100644
--- a/core/utils.go
+++ b/core/utils.go
@@ -176,7 +176,7 @@ func VerifyAgreementResult(
 	voted := make(map[types.NodeID]struct{}, len(notarySet))
 	voteType := res.Votes[0].Type
 	votePeriod := res.Votes[0].Period
-	if voteType != types.VoteFast && voteType != types.VoteCom {
+	if voteType != types.VoteFastCom && voteType != types.VoteCom {
 		return ErrIncorrectVoteType
 	}
 	for _, vote := range res.Votes {
diff --git a/core/utils_test.go b/core/utils_test.go
index 5d61a48..fa5d260 100644
--- a/core/utils_test.go
+++ b/core/utils_test.go
@@ -96,7 +96,7 @@ func (s *UtilsTestSuite) TestVerifyAgreementResult() {
 	baResult.Votes[0].Type = types.VoteCom
 
 	// Vote type should be the same.
-	baResult.Votes[1].Type = types.VoteFast
+	baResult.Votes[1].Type = types.VoteFastCom
 	s.Equal(ErrIncorrectVoteType, VerifyAgreementResult(baResult, cache))
 	baResult.Votes[1].Type = types.VoteCom