test: refine test utility (#61)

* Add functionality to test.App
* Add test utility to generate slices of types.ValidatorID

6 files changed, 307 insertions, 49 deletions

diff --git a/core/negative-ack_test.go b/core/negative-ack_test.go
index 87fcea0..990871e 100644
--- a/core/negative-ack_test.go
+++ b/core/negative-ack_test.go
@@ -23,7 +23,7 @@ import (
 
 	"github.com/stretchr/testify/suite"
 
-	"github.com/dexon-foundation/dexon-consensus-core/common"
+	"github.com/dexon-foundation/dexon-consensus-core/core/test"
 	"github.com/dexon-foundation/dexon-consensus-core/core/types"
 )
 
@@ -84,10 +84,7 @@ func genTimestamp(vids []types.ValidatorID, a []int) map[types.ValidatorID]time.
 }
 
 func genTestNegativeAck(num int) (*negativeAck, []types.ValidatorID) {
-	vids := []types.ValidatorID{}
-	for i := 0; i < num; i++ {
-		vids = append(vids, types.ValidatorID{Hash: common.NewRandomHash()})
-	}
+	vids := test.GenerateRandomValidatorIDs(num)
 	n := newNegativeAck(vids[0])
 	for i := 1; i < num; i++ {
 		n.addValidator(vids[i])
diff --git a/core/reliable-broadcast_test.go b/core/reliable-broadcast_test.go
index 81e640f..c1d0097 100644
--- a/core/reliable-broadcast_test.go
+++ b/core/reliable-broadcast_test.go
@@ -449,15 +449,13 @@ func (s *ReliableBroadcastTest) TestExtractBlocks() {
 
 func (s *ReliableBroadcastTest) TestRandomIntensiveAcking() {
 	rb := newReliableBroadcast()
-	vids := []types.ValidatorID{}
+	vids := test.GenerateRandomValidatorIDs(4)
 	heights := map[types.ValidatorID]uint64{}
 	extractedBlocks := []*types.Block{}
 
 	// Generate validators.
-	for i := 0; i < 4; i++ {
-		vid := types.ValidatorID{Hash: common.NewRandomHash()}
+	for _, vid := range vids {
 		rb.addValidator(vid)
-		vids = append(vids, vid)
 	}
 	// Generate genesis blocks.
 	for _, vid := range vids {
@@ -574,12 +572,10 @@ func (s *ReliableBroadcastTest) TestPrepareBlock() {
 		req         = s.Require()
 		rb          = newReliableBroadcast()
 		minInterval = 50 * time.Millisecond
-		validators  []types.ValidatorID
+		validators  = test.GenerateRandomValidatorIDs(4)
 	)
 	// Prepare validator IDs.
-	for i := 0; i < 4; i++ {
-		vID := types.ValidatorID{Hash: common.NewRandomHash()}
-		validators = append(validators, vID)
+	for _, vID := range validators {
 		rb.addValidator(vID)
 	}
 	// Setup genesis blocks.
diff --git a/core/test/app.go b/core/test/app.go
index 55ba7c5..aedcba9 100644
--- a/core/test/app.go
+++ b/core/test/app.go
@@ -18,44 +18,79 @@
 package test
 
 import (
+	"fmt"
+	"sync"
 	"time"
 
 	"github.com/dexon-foundation/dexon-consensus-core/common"
 )
 
+var (
+	// ErrEmptyDeliverSequence means there is no delivery event in this App
+	// instance.
+	ErrEmptyDeliverSequence = fmt.Errorf("emptry deliver sequence")
+	// ErrMismatchBlockHashSequence means the delivering sequence between two App
+	// instances are different.
+	ErrMismatchBlockHashSequence = fmt.Errorf("mismatch block hash sequence")
+	// ErrMismatchConsensusTime means the consensus timestamp between two blocks
+	// with the same hash from two App instances are different.
+	ErrMismatchConsensusTime = fmt.Errorf("mismatch consensus time")
+	// ErrApplicationIntegrityFailed means the internal datum in a App instance
+	// is not integrated.
+	ErrApplicationIntegrityFailed = fmt.Errorf("application integrity failed")
+	// ErrConsensusTimestampOutOfOrder means the later delivered block has
+	// consensus timestamp older than previous block.
+	ErrConsensusTimestampOutOfOrder = fmt.Errorf(
+		"consensus timestamp out of order")
+	// ErrDeliveredBlockNotAcked means some block delivered (confirmed) but
+	// not strongly acked.
+	ErrDeliveredBlockNotAcked = fmt.Errorf("delivered block not acked")
+	// ErrMismatchTotalOrderingAndDelivered mean the sequence of total ordering
+	// and delivered are different.
+	ErrMismatchTotalOrderingAndDelivered = fmt.Errorf(
+		"mismatch total ordering and delivered sequence")
+)
+
+type totalOrderDeliver struct {
+	BlockHashes common.Hashes
+	Early       bool
+}
+
 // App implements Application interface for testing purpose.
 type App struct {
-	Acked        map[common.Hash]struct{}
-	TotalOrdered []*struct {
-		BlockHashes common.Hashes
-		Early       bool
-	}
-	Delivered map[common.Hash]time.Time
+	Acked            map[common.Hash]struct{}
+	ackedLock        sync.RWMutex
+	TotalOrdered     []*totalOrderDeliver
+	totalOrderedLock sync.RWMutex
+	Delivered        map[common.Hash]time.Time
+	DeliverSequence  common.Hashes
+	deliveredLock    sync.RWMutex
 }
 
 // NewApp constructs a TestApp instance.
 func NewApp() *App {
 	return &App{
-		Acked: make(map[common.Hash]struct{}),
-		TotalOrdered: []*struct {
-			BlockHashes common.Hashes
-			Early       bool
-		}{},
-		Delivered: make(map[common.Hash]time.Time),
+		Acked:           make(map[common.Hash]struct{}),
+		TotalOrdered:    []*totalOrderDeliver{},
+		Delivered:       make(map[common.Hash]time.Time),
+		DeliverSequence: common.Hashes{},
 	}
 }
 
 // StronglyAcked implements Application interface.
 func (app *App) StronglyAcked(blockHash common.Hash) {
+	app.ackedLock.Lock()
+	defer app.ackedLock.Unlock()
+
 	app.Acked[blockHash] = struct{}{}
 }
 
 // TotalOrderingDeliver implements Application interface.
 func (app *App) TotalOrderingDeliver(blockHashes common.Hashes, early bool) {
-	app.TotalOrdered = append(app.TotalOrdered, &struct {
-		BlockHashes common.Hashes
-		Early       bool
-	}{
+	app.totalOrderedLock.Lock()
+	defer app.totalOrderedLock.Unlock()
+
+	app.TotalOrdered = append(app.TotalOrdered, &totalOrderDeliver{
 		BlockHashes: blockHashes,
 		Early:       early,
 	})
@@ -63,5 +98,96 @@ func (app *App) TotalOrderingDeliver(blockHashes common.Hashes, early bool) {
 
 // DeliverBlock implements Application interface.
 func (app *App) DeliverBlock(blockHash common.Hash, timestamp time.Time) {
+	app.deliveredLock.Lock()
+	defer app.deliveredLock.Unlock()
+
 	app.Delivered[blockHash] = timestamp
+	app.DeliverSequence = append(app.DeliverSequence, blockHash)
+}
+
+// Compare performs these checks against another App instance
+// and return erros if not passed:
+// - deliver sequence by comparing block hashes.
+// - consensus timestamp of each block are equal.
+func (app *App) Compare(other *App) error {
+	app.deliveredLock.RLock()
+	defer app.deliveredLock.RUnlock()
+	other.deliveredLock.RLock()
+	defer other.deliveredLock.RUnlock()
+
+	minLength := len(app.DeliverSequence)
+	if minLength > len(other.DeliverSequence) {
+		minLength = len(other.DeliverSequence)
+	}
+	if minLength == 0 {
+		return ErrEmptyDeliverSequence
+	}
+	for idx, h := range app.DeliverSequence[:minLength] {
+		hOther := other.DeliverSequence[idx]
+		if hOther != h {
+			return ErrMismatchBlockHashSequence
+		}
+		if app.Delivered[h] != other.Delivered[h] {
+			return ErrMismatchConsensusTime
+		}
+	}
+	return nil
+}
+
+// Verify checks the integrity of date received by this App instance.
+func (app *App) Verify() error {
+	app.deliveredLock.RLock()
+	defer app.deliveredLock.RUnlock()
+
+	if len(app.DeliverSequence) == 0 {
+		return ErrEmptyDeliverSequence
+	}
+	if len(app.DeliverSequence) != len(app.Delivered) {
+		return ErrApplicationIntegrityFailed
+	}
+
+	app.ackedLock.RLock()
+	defer app.ackedLock.RUnlock()
+
+	prevTime := time.Time{}
+	for _, h := range app.DeliverSequence {
+		// Make sure delivered block is strongly acked.
+		if _, acked := app.Acked[h]; !acked {
+			return ErrDeliveredBlockNotAcked
+		}
+		t, exists := app.Delivered[h]
+		if !exists {
+			return ErrApplicationIntegrityFailed
+		}
+		// Make sure the consensus time is incremental.
+		ok := prevTime.Before(t) || prevTime.Equal(t)
+		if !ok {
+			return ErrConsensusTimestampOutOfOrder
+		}
+		prevTime = t
+	}
+	// Make sure the order of delivered and total ordering are the same by
+	// comparing the concated string.
+	app.totalOrderedLock.RLock()
+	defer app.totalOrderedLock.RUnlock()
+
+	hashSequenceIdx := 0
+Loop:
+	for _, totalOrderDeliver := range app.TotalOrdered {
+		for _, h := range totalOrderDeliver.BlockHashes {
+			if hashSequenceIdx >= len(app.DeliverSequence) {
+				break Loop
+			}
+			if h != app.DeliverSequence[hashSequenceIdx] {
+				return ErrMismatchTotalOrderingAndDelivered
+			}
+			hashSequenceIdx++
+		}
+	}
+	if hashSequenceIdx != len(app.DeliverSequence) {
+		// The count of delivered blocks should be larger than those delivered
+		// by total ordering.
+		return ErrMismatchTotalOrderingAndDelivered
+	}
+	return nil
 }
diff --git a/core/test/app_test.go b/core/test/app_test.go
new file mode 100644
index 0000000..285773e
--- /dev/null
+++ b/core/test/app_test.go
@@ -0,0 +1,142 @@
+package test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/dexon-foundation/dexon-consensus-core/common"
+	"github.com/stretchr/testify/suite"
+)
+
+type AppTestSuite struct {
+	suite.Suite
+
+	to1, to2, to3 *totalOrderDeliver
+}
+
+func (s *AppTestSuite) SetupSuite() {
+	s.to1 = &totalOrderDeliver{
+		BlockHashes: common.Hashes{
+			common.NewRandomHash(),
+			common.NewRandomHash(),
+		},
+		Early: false,
+	}
+	s.to2 = &totalOrderDeliver{
+		BlockHashes: common.Hashes{
+			common.NewRandomHash(),
+			common.NewRandomHash(),
+			common.NewRandomHash(),
+		},
+		Early: false,
+	}
+	s.to3 = &totalOrderDeliver{
+		BlockHashes: common.Hashes{
+			common.NewRandomHash(),
+		},
+		Early: false,
+	}
+}
+
+func (s *AppTestSuite) setupAppByTotalOrderDeliver(
+	app *App, to *totalOrderDeliver) {
+
+	for _, h := range to.BlockHashes {
+		app.StronglyAcked(h)
+	}
+	app.TotalOrderingDeliver(to.BlockHashes, to.Early)
+	for _, h := range to.BlockHashes {
+		// To make it simpler, use the index of hash sequence
+		// as the time.
+		s.deliverBlockWithTimeFromSequenceLength(app, h)
+	}
+}
+
+func (s *AppTestSuite) deliverBlockWithTimeFromSequenceLength(
+	app *App, hash common.Hash) {
+
+	app.DeliverBlock(
+		hash,
+		time.Time{}.Add(time.Duration(len(app.DeliverSequence))*time.Second))
+}
+
+func (s *AppTestSuite) TestCompare() {
+	req := s.Require()
+
+	app1 := NewApp()
+	s.setupAppByTotalOrderDeliver(app1, s.to1)
+	s.setupAppByTotalOrderDeliver(app1, s.to2)
+	s.setupAppByTotalOrderDeliver(app1, s.to3)
+	// An App with different deliver sequence.
+	app2 := NewApp()
+	s.setupAppByTotalOrderDeliver(app2, s.to1)
+	s.setupAppByTotalOrderDeliver(app2, s.to2)
+	hash := common.NewRandomHash()
+	app2.StronglyAcked(hash)
+	app2.TotalOrderingDeliver(common.Hashes{hash}, false)
+	s.deliverBlockWithTimeFromSequenceLength(app2, hash)
+	req.Equal(ErrMismatchBlockHashSequence, app1.Compare(app2))
+	// An App with different consensus time for the same block.
+	app3 := NewApp()
+	s.setupAppByTotalOrderDeliver(app3, s.to1)
+	s.setupAppByTotalOrderDeliver(app3, s.to2)
+	for _, h := range s.to3.BlockHashes {
+		app3.StronglyAcked(h)
+	}
+	app3.TotalOrderingDeliver(s.to3.BlockHashes, s.to3.Early)
+	wrongTime := time.Time{}.Add(
+		time.Duration(len(app3.DeliverSequence)) * time.Second)
+	wrongTime = wrongTime.Add(1 * time.Second)
+	app3.DeliverBlock(s.to3.BlockHashes[0], wrongTime)
+	req.Equal(ErrMismatchConsensusTime, app1.Compare(app3))
+	req.Equal(ErrMismatchConsensusTime, app3.Compare(app1))
+	// An App without any delivered blocks.
+	app4 := NewApp()
+	req.Equal(ErrEmptyDeliverSequence, app4.Compare(app1))
+	req.Equal(ErrEmptyDeliverSequence, app1.Compare(app4))
+}
+
+func (s *AppTestSuite) TestVerify() {
+	req := s.Require()
+
+	// An OK App instance.
+	app1 := NewApp()
+	s.setupAppByTotalOrderDeliver(app1, s.to1)
+	s.setupAppByTotalOrderDeliver(app1, s.to2)
+	s.setupAppByTotalOrderDeliver(app1, s.to3)
+	req.Nil(app1.Verify())
+	// A delivered block without strongly ack
+	app1.DeliverBlock(common.NewRandomHash(), time.Time{})
+	req.Equal(ErrDeliveredBlockNotAcked, app1.Verify())
+	// The consensus time is out of order.
+	app2 := NewApp()
+	s.setupAppByTotalOrderDeliver(app2, s.to1)
+	for _, h := range s.to2.BlockHashes {
+		app2.StronglyAcked(h)
+	}
+	app2.TotalOrderingDeliver(s.to2.BlockHashes, s.to2.Early)
+	app2.DeliverBlock(s.to2.BlockHashes[0], time.Time{})
+	req.Equal(ErrConsensusTimestampOutOfOrder, app2.Verify())
+	// A delivered block is not found in total ordering delivers.
+	app3 := NewApp()
+	s.setupAppByTotalOrderDeliver(app3, s.to1)
+	hash := common.NewRandomHash()
+	app3.StronglyAcked(hash)
+	s.deliverBlockWithTimeFromSequenceLength(app3, hash)
+	req.Equal(ErrMismatchTotalOrderingAndDelivered, app3.Verify())
+	// A delivered block is not found in total ordering delivers.
+	app4 := NewApp()
+	s.setupAppByTotalOrderDeliver(app4, s.to1)
+	for _, h := range s.to2.BlockHashes {
+		app4.StronglyAcked(h)
+	}
+	app4.TotalOrderingDeliver(s.to2.BlockHashes, s.to2.Early)
+	hash = common.NewRandomHash()
+	app4.StronglyAcked(hash)
+	app4.TotalOrderingDeliver(common.Hashes{hash}, false)
+	s.deliverBlockWithTimeFromSequenceLength(app4, hash)
+}
+
+func TestApp(t *testing.T) {
+	suite.Run(t, new(AppTestSuite))
+}
diff --git a/core/test/utils.go b/core/test/utils.go
index eae12a0..35fbdd5 100644
--- a/core/test/utils.go
+++ b/core/test/utils.go
@@ -29,3 +29,12 @@ func stableRandomHash(blockConv types.BlockConverter) (common.Hash, error) {
 	}
 	return common.NewRandomHash(), nil
 }
+
+// GenerateRandomValidatorIDs generates randomly a slices of types.ValidatorID.
+func GenerateRandomValidatorIDs(validatorCount int) (vIDs types.ValidatorIDs) {
+	vIDs = types.ValidatorIDs{}
+	for i := 0; i < validatorCount; i++ {
+		vIDs = append(vIDs, types.ValidatorID{Hash: common.NewRandomHash()})
+	}
+	return
+}
diff --git a/core/total-ordering_test.go b/core/total-ordering_test.go
index 69297d3..dca92d7 100644
--- a/core/total-ordering_test.go
+++ b/core/total-ordering_test.go
@@ -33,18 +33,6 @@ type TotalOrderingTestSuite struct {
 	suite.Suite
 }
 
-func (s *TotalOrderingTestSuite) generateValidatorIDs(
-	count int) []types.ValidatorID {
-
-	validatorIDs := []types.ValidatorID{}
-	for i := 0; i < count; i++ {
-		validatorIDs = append(validatorIDs,
-			types.ValidatorID{Hash: common.NewRandomHash()})
-	}
-
-	return validatorIDs
-}
-
 func (s *TotalOrderingTestSuite) genGenesisBlock(
 	vID types.ValidatorID, acks map[common.Hash]struct{}) *types.Block {
 
@@ -128,7 +116,7 @@ func (s *TotalOrderingTestSuite) TestBlockRelation() {
 }
 
 func (s *TotalOrderingTestSuite) TestCreateAckingHeightVectorFromHeightVector() {
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 	global := ackingStatusVector{
 		validators[0]: &struct{ minHeight, count uint64 }{
 			minHeight: 0, count: 5},
@@ -172,7 +160,7 @@ func (s *TotalOrderingTestSuite) TestCreateAckingHeightVectorFromHeightVector()
 }
 
 func (s *TotalOrderingTestSuite) TestCreateAckingNodeSetFromHeightVector() {
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 	global := ackingStatusVector{
 		validators[0]: &struct{ minHeight, count uint64 }{
 			minHeight: 0, count: 5},
@@ -194,7 +182,7 @@ func (s *TotalOrderingTestSuite) TestCreateAckingNodeSetFromHeightVector() {
 }
 
 func (s *TotalOrderingTestSuite) TestGrade() {
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 	to := newTotalOrdering(1, 3, 5) // K doesn't matter when calculating preceding.
 
 	ans := map[types.ValidatorID]struct{}{
@@ -231,7 +219,7 @@ func (s *TotalOrderingTestSuite) TestGrade() {
 func (s *TotalOrderingTestSuite) TestCycleDetection() {
 	// Make sure we don't get hang by cycle from
 	// block's acks.
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 
 	// create blocks with cycles in acking relation.
 	cycledHash := common.NewRandomHash()
@@ -283,7 +271,7 @@ func (s *TotalOrderingTestSuite) TestCycleDetection() {
 }
 
 func (s *TotalOrderingTestSuite) TestNotValidDAGDetection() {
-	validators := s.generateValidatorIDs(4)
+	validators := test.GenerateRandomValidatorIDs(4)
 	to := newTotalOrdering(1, 3, 5)
 
 	b00 := s.genGenesisBlock(validators[0], map[common.Hash]struct{}{})
@@ -312,7 +300,7 @@ func (s *TotalOrderingTestSuite) TestEarlyDeliver() {
 	//  Even when B is not received, A should
 	//  be able to be delivered.
 	to := newTotalOrdering(2, 3, 5)
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 
 	genNextBlock := func(b *types.Block) *types.Block {
 		return &types.Block{
@@ -428,7 +416,7 @@ func (s *TotalOrderingTestSuite) TestEarlyDeliver() {
 func (s *TotalOrderingTestSuite) TestBasicCaseForK2() {
 	// It's a handcrafted test case.
 	to := newTotalOrdering(2, 3, 5)
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 
 	b00 := s.genGenesisBlock(validators[0], map[common.Hash]struct{}{})
 	b10 := s.genGenesisBlock(validators[1], map[common.Hash]struct{}{})
@@ -759,7 +747,7 @@ func (s *TotalOrderingTestSuite) TestBasicCaseForK0() {
 	//  v   v   v    v
 	//  o   o   o <- o        Height: 0
 	to := newTotalOrdering(0, 3, 5)
-	validators := s.generateValidatorIDs(5)
+	validators := test.GenerateRandomValidatorIDs(5)
 
 	b00 := s.genGenesisBlock(validators[0], map[common.Hash]struct{}{})
 	b10 := s.genGenesisBlock(validators[1], map[common.Hash]struct{}{})