core: fix false alarm (#564)

* ignore test simple

* core: update voteFilter to filter old er round

* circleci: save logs

* core: move check notarySet to agrmgr

* fixup

4 files changed, 53 insertions, 25 deletions

diff --git a/core/agreement-mgr.go b/core/agreement-mgr.go
index ac28955..4597fe9 100644
--- a/core/agreement-mgr.go
+++ b/core/agreement-mgr.go
@@ -24,6 +24,8 @@ import (
 	"sync"
 	"time"
 
+	lru "github.com/hashicorp/golang-lru"
+
 	"github.com/dexon-foundation/dexon-consensus/common"
 	"github.com/dexon-foundation/dexon-consensus/core/types"
 	typesDKG "github.com/dexon-foundation/dexon-consensus/core/types/dkg"
@@ -41,6 +43,7 @@ var (
 )
 
 const maxResultCache = 100
+const settingLimit = 3
 
 // genValidLeader generate a validLeader function for agreement modules.
 func genValidLeader(
@@ -130,12 +133,15 @@ type agreementMgr struct {
 	recv              *consensusBAReceiver
 	processedBAResult map[types.Position]struct{}
 	voteFilter        *utils.VoteFilter
+	settingCache      *lru.Cache
+	curRoundSetting   *baRoundSetting
 	waitGroup         sync.WaitGroup
 	isRunning         bool
 	lock              sync.RWMutex
 }
 
 func newAgreementMgr(con *Consensus) (mgr *agreementMgr, err error) {
+	settingCache, _ := lru.New(settingLimit)
 	mgr = &agreementMgr{
 		con:               con,
 		ID:                con.ID,
@@ -149,6 +155,7 @@ func newAgreementMgr(con *Consensus) (mgr *agreementMgr, err error) {
 		ctx:               con.ctx,
 		processedBAResult: make(map[types.Position]struct{}, maxResultCache),
 		voteFilter:        utils.NewVoteFilter(),
+		settingCache:      settingCache,
 	}
 	mgr.recv = &consensusBAReceiver{
 		consensus:     con,
@@ -165,21 +172,18 @@ func (mgr *agreementMgr) prepare() {
 		newLeaderSelector(genValidLeader(mgr), mgr.logger),
 		mgr.signer,
 		mgr.logger)
-	nodes, err := mgr.cache.GetNodeSet(round)
-	if err != nil {
+	setting := mgr.generateSetting(round)
+	if setting == nil {
+		mgr.logger.Warn("Unable to prepare init setting", "round", round)
 		return
 	}
-	agr.notarySet = nodes.GetSubSet(
-		int(mgr.config(round).notarySetSize),
-		types.NewNotarySetTarget(mgr.config(round).crs))
+	mgr.curRoundSetting = setting
+	agr.notarySet = mgr.curRoundSetting.dkgSet
 	// Hacky way to make agreement module self contained.
 	mgr.recv.agreementModule = agr
 	mgr.baModule = agr
 	if round >= DKGDelayRound {
-		setting := mgr.generateSetting(round)
-		if setting == nil {
-			mgr.logger.Warn("Unable to prepare init setting", "round", round)
-		} else if _, exist := setting.dkgSet[mgr.ID]; exist {
+		if _, exist := setting.dkgSet[mgr.ID]; exist {
 			mgr.logger.Debug("Preparing signer and npks.", "round", round)
 			npk, signer, err := mgr.con.cfgModule.getDKGInfo(round, false)
 			if err != nil {
@@ -268,9 +272,25 @@ func (mgr *agreementMgr) notifyRoundEvents(evts []utils.RoundEventParam) error {
 }
 
 func (mgr *agreementMgr) processVote(v *types.Vote) (err error) {
+	if !mgr.recv.isNotary {
+		return nil
+	}
 	if mgr.voteFilter.Filter(v) {
 		return nil
 	}
+	if v.Position.Round == mgr.curRoundSetting.round {
+		if _, exist := mgr.curRoundSetting.dkgSet[v.ProposerID]; !exist {
+			return ErrNotInNotarySet
+		}
+	} else if v.Position.Round == mgr.curRoundSetting.round+1 {
+		setting := mgr.generateSetting(v.Position.Round)
+		if setting == nil {
+			return ErrConfigurationNotReady
+		}
+		if _, exist := setting.dkgSet[v.ProposerID]; !exist {
+			return ErrNotInNotarySet
+		}
+	}
 	if err = mgr.baModule.processVote(v); err == nil {
 		mgr.baModule.updateFilter(mgr.voteFilter)
 		mgr.voteFilter.AddVote(v)
@@ -339,6 +359,7 @@ func (mgr *agreementMgr) processAgreementResult(
 				result.Position.Round)
 			return ErrConfigurationNotReady
 		}
+		mgr.curRoundSetting = setting
 		leader, err := mgr.calcLeader(setting.dkgSet, setting.crs, result.Position)
 		if err != nil {
 			return err
@@ -374,6 +395,9 @@ func (mgr *agreementMgr) stop() {
 }
 
 func (mgr *agreementMgr) generateSetting(round uint64) *baRoundSetting {
+	if setting, exist := mgr.settingCache.Get(round); exist {
+		return setting.(*baRoundSetting)
+	}
 	curConfig := mgr.config(round)
 	if curConfig == nil {
 		return nil
@@ -399,13 +423,15 @@ func (mgr *agreementMgr) generateSetting(round uint64) *baRoundSetting {
 			return nil
 		}
 	}
-	return &baRoundSetting{
+	setting := &baRoundSetting{
 		crs:    curConfig.crs,
 		dkgSet: dkgSet,
 		round:  round,
 		threshold: utils.GetBAThreshold(&types.Config{
 			NotarySetSize: curConfig.notarySetSize}),
 	}
+	mgr.settingCache.Add(round, setting)
+	return setting
 }
 
 func (mgr *agreementMgr) runBA(initRound uint64) {
@@ -465,6 +491,7 @@ Loop:
 		}
 		mgr.recv.isNotary = checkRound()
 		mgr.voteFilter = utils.NewVoteFilter()
+		mgr.voteFilter.Position.Round = currentRound
 		mgr.recv.emptyBlockHashMap = &sync.Map{}
 		if currentRound >= DKGDelayRound && mgr.recv.isNotary {
 			var err error
diff --git a/core/agreement.go b/core/agreement.go
index a6fb074..cb46771 100644
--- a/core/agreement.go
+++ b/core/agreement.go
@@ -361,9 +361,6 @@ func (a *agreement) sanityCheck(vote *types.Vote) error {
 	if vote.Type >= types.MaxVoteType {
 		return ErrInvalidVote
 	}
-	if _, exist := a.notarySet[vote.ProposerID]; !exist {
-		return ErrNotInNotarySet
-	}
 	ok, err := utils.VerifyVoteSignature(vote)
 	if err != nil {
 		return err
@@ -371,6 +368,10 @@ func (a *agreement) sanityCheck(vote *types.Vote) error {
 	if !ok {
 		return ErrIncorrectVoteSignature
 	}
+	if vote.Position.Round != a.agreementID().Round {
+		// TODO(jimmy): maybe we can verify partial signature at agreement-mgr.
+		return nil
+	}
 	if !a.data.recv.VerifyPartialSignature(vote) {
 		return ErrIncorrectVotePartialSignature
 	}
@@ -412,7 +413,7 @@ func (a *agreement) updateFilter(filter *utils.VoteFilter) {
 	filter.Confirm = a.hasOutput
 	filter.LockIter = a.data.lockIter
 	filter.Period = a.data.period
-	filter.Height = a.agreementID().Height
+	filter.Position.Height = a.agreementID().Height
 }
 
 // processVote is the entry point for processing Vote.
@@ -426,8 +427,8 @@ func (a *agreement) processVote(vote *types.Vote) error {
 
 	// Agreement module has stopped.
 	if isStop(aID) {
-		// Hacky way to not drop first votes for genesis height.
-		if vote.Position.Height == types.GenesisHeight {
+		// Hacky way to not drop first votes when round just begins.
+		if vote.Position.Round == aID.Round {
 			a.pendingVote = append(a.pendingVote, pendingVote{
 				vote:         vote,
 				receivedTime: time.Now().UTC(),
diff --git a/core/utils/vote-filter.go b/core/utils/vote-filter.go
index 2fc18bb..446d88a 100644
--- a/core/utils/vote-filter.go
+++ b/core/utils/vote-filter.go
@@ -25,7 +25,7 @@ import (
 // To maximize performance, this structure is not thread-safe and will never be.
 type VoteFilter struct {
 	Voted    map[types.VoteHeader]struct{}
-	Height   uint64
+	Position types.Position
 	LockIter uint64
 	Period   uint64
 	Confirm  bool
@@ -43,9 +43,9 @@ func (vf *VoteFilter) Filter(vote *types.Vote) bool {
 	if vote.Type == types.VoteInit {
 		return true
 	}
-	if vote.Position.Height < vf.Height {
+	if vote.Position.Older(vf.Position) {
 		return true
-	} else if vote.Position.Height > vf.Height {
+	} else if vote.Position.Newer(vf.Position) {
 		// It's impossible to check the vote of other height.
 		return false
 	}
diff --git a/core/utils/vote-filter_test.go b/core/utils/vote-filter_test.go
index 443efff..4e7a7b9 100644
--- a/core/utils/vote-filter_test.go
+++ b/core/utils/vote-filter_test.go
@@ -32,22 +32,22 @@ type VoteFilterTestSuite struct {
 
 func (s *VoteFilterTestSuite) TestFilterVotePass() {
 	filter := NewVoteFilter()
-	filter.Height = uint64(6)
+	filter.Position.Height = uint64(6)
 	filter.Period = uint64(3)
 	filter.LockIter = uint64(3)
 	// Pass with higher Height.
 	vote := types.NewVote(types.VotePreCom, common.NewRandomHash(), uint64(1))
-	vote.Position.Height = filter.Height + 1
+	vote.Position.Height = filter.Position.Height + 1
 	s.Require().False(filter.Filter(vote))
 	// Pass with VotePreCom.
 	vote = types.NewVote(types.VotePreCom, common.NewRandomHash(),
 		filter.LockIter)
-	vote.Position.Height = filter.Height
+	vote.Position.Height = filter.Position.Height
 	s.Require().False(filter.Filter(vote))
 	// Pass with VoteCom.
 	vote = types.NewVote(types.VoteCom, common.NewRandomHash(),
 		filter.Period)
-	vote.Position.Height = filter.Height
+	vote.Position.Height = filter.Position.Height
 	s.Require().False(filter.Filter(vote))
 	vote.Period--
 	s.Require().False(filter.Filter(vote))
@@ -81,9 +81,9 @@ func (s *VoteFilterTestSuite) TestFilterConfirm() {
 }
 func (s *VoteFilterTestSuite) TestFilterLowerHeight() {
 	filter := NewVoteFilter()
-	filter.Height = uint64(10)
+	filter.Position.Height = uint64(10)
 	vote := types.NewVote(types.VoteCom, common.NewRandomHash(), uint64(1))
-	vote.Position.Height = filter.Height - 1
+	vote.Position.Height = filter.Position.Height - 1
 	s.True(filter.Filter(vote))
 }