build: add end to end integration test for recovery mechanism (#336)

2 files changed, 80 insertions, 0 deletions

diff --git a/build/recovery-test.sh b/build/recovery-test.sh
new file mode 100755
index 000000000..f305f743c
--- /dev/null
+++ b/build/recovery-test.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+tarAndUpload()
+{
+  name=travis-fail-$(date +%s).tar.gz
+  tar -zcvf $name test
+  echo "Verify fail and upload $name"
+  PKG_CONFIG_PATH=/usr/local/opt/openssl/lib/pkgconfig go run build/testtool/testtool.go upload $name dexon-builds
+}
+
+endpoint=http://127.0.0.1:8545
+
+timeout=700
+
+echo "Wait for recovery"
+cmd="PKG_CONFIG_PATH=/usr/local/opt/openssl/lib/pkgconfig go run build/testtool/testtool.go waitForRecovery $endpoint $timeout"
+eval $cmd
+code=$?
+if [ $code == 1 ]; then
+  tarAndUpload
+  exit 1
+fi
diff --git a/build/testtool/testtool.go b/build/testtool/testtool.go
index 3f880b3bd..a801cfbb3 100644
--- a/build/testtool/testtool.go
+++ b/build/testtool/testtool.go
@@ -16,6 +16,7 @@ import (
 	"github.com/dexon-foundation/dexon"
 	"github.com/dexon-foundation/dexon/accounts/abi"
 	"github.com/dexon-foundation/dexon/cmd/zoo/monkey"
+	"github.com/dexon-foundation/dexon/core/types"
 	"github.com/dexon-foundation/dexon/core/vm"
 	"github.com/dexon-foundation/dexon/crypto"
 	"github.com/dexon-foundation/dexon/ethclient"
@@ -34,6 +35,8 @@ func main() {
 		doVerifyGovMPK(os.Args[2:])
 	case "monkeyTest":
 		doMonkeyTest(os.Args[2:])
+	case "waitForRecovery":
+		doWaitForRecovery(os.Args[2:])
 	case "upload":
 		doUpload(os.Args[2:])
 	}
@@ -206,3 +209,58 @@ func doMonkeyTest(args []string) {
 		}
 	}
 }
+
+func doWaitForRecovery(args []string) {
+	if len(args) < 2 {
+		log.Fatal("arg length is not enough")
+	}
+
+	client, err := ethclient.Dial(args[0])
+	if err != nil {
+		log.Fatalf("new ethclient fail: %v", err)
+	}
+
+	lastBlock, err := client.BlockByNumber(context.Background(), nil)
+	if err != nil {
+		log.Fatalf("get last block fail %v", err)
+	}
+	t, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		log.Fatalf("timeout args invalid, %s", args[1])
+	}
+
+	sleep := time.Duration(t) * time.Second
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	go func() {
+		for ctx.Err() == nil {
+			select {
+			case <-time.After(1 * time.Minute):
+				log.Printf("tick")
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	log.Printf("Sleep %s", sleep)
+	time.Sleep(sleep)
+	cancel()
+
+	// Check block height is increasing 5 time for safe.
+	var block *types.Block
+	for i := 0; i < 5; i++ {
+		block, err = client.BlockByNumber(context.Background(), nil)
+		if err != nil {
+			log.Fatalf("get current block fail, err=%v, i=%d", err, i)
+		}
+		if block.NumberU64() <= lastBlock.NumberU64() {
+			log.Fatalf("recovery fail, last=%d, current=%d",
+				lastBlock.NumberU64(), block.NumberU64())
+		}
+		log.Printf("last=%d, current=%d, ok, sleep a while", lastBlock.NumberU64(), block.NumberU64())
+		lastBlock = block
+		time.Sleep(2 * time.Second)
+	}
+}