simulation: support config change (#381)

12 files changed, 365 insertions, 114 deletions

diff --git a/cmd/dexcon-simulation/main.go b/cmd/dexcon-simulation/main.go
index 3953777..a9a8b10 100644
--- a/cmd/dexcon-simulation/main.go
+++ b/cmd/dexcon-simulation/main.go
@@ -20,7 +20,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"math/rand"
 	"net/http"
@@ -38,7 +37,7 @@ var initialize = flag.Bool("init", false, "initialize config file")
 var configFile = flag.String("config", "", "path to simulation config file")
 var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
 var memprofile = flag.String("memprofile", "", "write memory profile to `file`")
-var logfile = flag.String("log", "", "write log to `file`.log")
+var logfile = flag.String("log", "", "write log to `file`-nodeID.log")
 
 func main() {
 	flag.Parse()
@@ -70,15 +69,6 @@ func main() {
 		defer pprof.StopCPUProfile()
 	}
 
-	if *logfile != "" {
-		f, err := os.Create(*logfile)
-		if err != nil {
-			log.Fatal("could not create log file: ", err)
-		}
-		mw := io.MultiWriter(os.Stdout, f)
-		log.SetOutput(mw)
-	}
-
 	cfg, err := config.Read(*configFile)
 	if err != nil {
 		panic(err)
diff --git a/core/test/state-change-request.go b/core/test/state-change-request.go
index 687eef7..21e623b 100644
--- a/core/test/state-change-request.go
+++ b/core/test/state-change-request.go
@@ -55,6 +55,44 @@ const (
 	StateAddNode
 )
 
+func (t StateChangeType) String() string {
+	switch t {
+	case StateChangeNothing:
+		return "Nothing"
+	case StateAddCRS:
+		return "AddCRS"
+	case StateAddDKGComplaint:
+		return "AddDKGComplaint"
+	case StateAddDKGMasterPublicKey:
+		return "AddDKGMasterPublicKey"
+	case StateAddDKGMPKReady:
+		return "AddDKGMPKReady"
+	case StateAddDKGFinal:
+		return "AddDKGFinal"
+	case StateChangeNumChains:
+		return "ChangeNumChains"
+	case StateChangeLambdaBA:
+		return "ChangeLambdaBA"
+	case StateChangeLambdaDKG:
+		return "ChangeLambdaDKG"
+	case StateChangeRoundInterval:
+		return "ChangeRoundInterval"
+	case StateChangeMinBlockInterval:
+		return "ChangeMinBlockInterval"
+	case StateChangeK:
+		return "ChangeK"
+	case StateChangePhiRatio:
+		return "ChangePhiRatio"
+	case StateChangeNotarySetSize:
+		return "ChangeNotarySetSize"
+	case StateChangeDKGSetSize:
+		return "ChangeDKGSetSize"
+	case StateAddNode:
+		return "AddNode"
+	}
+	panic(fmt.Errorf("attempting to dump unknown type of state change: %d", t))
+}
+
 // StateChangeRequest carries information of state change request.
 type StateChangeRequest struct {
 	Type    StateChangeType `json:"type"`
@@ -141,56 +179,42 @@ func (req *StateChangeRequest) Equal(other *StateChangeRequest) error {
 
 // String dump the state change request into string form.
 func (req *StateChangeRequest) String() (ret string) {
-	ret = "stateChangeRequest"
+	ret = fmt.Sprintf("stateChangeRequest{Type:%s", req.Type)
 	switch req.Type {
 	case StateChangeNothing:
-		ret += "{Type:Nothing}"
 	case StateAddCRS:
 		crsReq := req.Payload.(*crsAdditionRequest)
-		ret += fmt.Sprintf("{Type:AddCRS Round:%v CRS:%s",
-			crsReq.Round,
-			crsReq.CRS.String()[:6])
-	case StateAddDKGComplaint:
 		ret += fmt.Sprintf(
-			"{Type:AddDKGComplaint %s", req.Payload.(*typesDKG.Complaint))
+			"Round:%v CRS:%s", crsReq.Round, crsReq.CRS.String()[:6])
+	case StateAddDKGComplaint:
+		ret += fmt.Sprintf("%s", req.Payload.(*typesDKG.Complaint))
 	case StateAddDKGMasterPublicKey:
-		ret += fmt.Sprintf(
-			"{Type:AddDKGMasterPublicKey %s",
-			req.Payload.(*typesDKG.MasterPublicKey))
+		ret += fmt.Sprintf("%s", req.Payload.(*typesDKG.MasterPublicKey))
 	case StateAddDKGMPKReady:
-		ret += fmt.Sprintf(
-			"{Type:AddDKGMPKReady %s", req.Payload.(*typesDKG.MPKReady))
+		ret += fmt.Sprintf("%s", req.Payload.(*typesDKG.MPKReady))
 	case StateAddDKGFinal:
-		ret += fmt.Sprintf(
-			"{Type:AddDKGFinal %s", req.Payload.(*typesDKG.Finalize))
+		ret += fmt.Sprintf("%s", req.Payload.(*typesDKG.Finalize))
 	case StateChangeNumChains:
-		ret += fmt.Sprintf("{Type:ChangeNumChains %v", req.Payload.(uint32))
+		ret += fmt.Sprintf("%v", req.Payload.(uint32))
 	case StateChangeLambdaBA:
-		ret += fmt.Sprintf(
-			"{Type:ChangeLambdaBA %v", time.Duration(req.Payload.(uint64)))
+		ret += fmt.Sprintf("%v", time.Duration(req.Payload.(uint64)))
 	case StateChangeLambdaDKG:
-		ret += fmt.Sprintf(
-			"{Type:ChangeLambdaDKG %v", time.Duration(req.Payload.(uint64)))
+		ret += fmt.Sprintf("%v", time.Duration(req.Payload.(uint64)))
 	case StateChangeRoundInterval:
-		ret += fmt.Sprintf(
-			"{Type:ChangeRoundInterval %v", time.Duration(req.Payload.(uint64)))
+		ret += fmt.Sprintf("%v", time.Duration(req.Payload.(uint64)))
 	case StateChangeMinBlockInterval:
-		ret += fmt.Sprintf(
-			"{Type:ChangeMinBlockInterval %v",
-			time.Duration(req.Payload.(uint64)))
+		ret += fmt.Sprintf("%v", time.Duration(req.Payload.(uint64)))
 	case StateChangeK:
-		ret += fmt.Sprintf("Type:ChangeK %v", req.Payload.(uint64))
+		ret += fmt.Sprintf("%v", req.Payload.(uint64))
 	case StateChangePhiRatio:
-		ret += fmt.Sprintf(
-			"{Type:ChangePhiRatio %v", math.Float32frombits(req.Payload.(uint32)))
+		ret += fmt.Sprintf("%v", math.Float32frombits(req.Payload.(uint32)))
 	case StateChangeNotarySetSize:
-		ret += fmt.Sprintf("{Type:ChangeNotarySetSize %v", req.Payload.(uint32))
+		ret += fmt.Sprintf("%v", req.Payload.(uint32))
 	case StateChangeDKGSetSize:
-		ret += fmt.Sprintf("{Type:ChangeDKGSetSize %v", req.Payload.(uint32))
+		ret += fmt.Sprintf("%v", req.Payload.(uint32))
 	case StateAddNode:
 		ret += fmt.Sprintf(
-			"{Type:AddNode %s",
-			types.NewNodeID(req.Payload.(crypto.PublicKey)).String()[:6])
+			"%s", types.NewNodeID(req.Payload.(crypto.PublicKey)).String()[:6])
 	default:
 		panic(fmt.Errorf(
 			"attempting to dump unknown type of state change request: %v",
diff --git a/core/test/state.go b/core/test/state.go
index b834b11..a1af47c 100644
--- a/core/test/state.go
+++ b/core/test/state.go
@@ -172,7 +172,7 @@ func (s *State) Snapshot() (*types.Config, []crypto.PublicKey) {
 	for _, key := range s.nodes {
 		nodes = append(nodes, key)
 	}
-	return &types.Config{
+	cfg := &types.Config{
 		NumChains:        s.numChains,
 		LambdaBA:         s.lambdaBA,
 		LambdaDKG:        s.lambdaDKG,
@@ -182,7 +182,9 @@ func (s *State) Snapshot() (*types.Config, []crypto.PublicKey) {
 		DKGSetSize:       s.dkgSetSize,
 		RoundInterval:    s.roundInterval,
 		MinBlockInterval: s.minBlockInterval,
-	}, nodes
+	}
+	s.logger.Info("Snapshot config", "config", cfg)
+	return cfg, nodes
 }
 
 // AttachLogger allows to attach custom logger.
@@ -761,6 +763,7 @@ func (s *State) ProposeCRS(round uint64, crs common.Hash) (err error) {
 // RequestChange submits a state change request.
 func (s *State) RequestChange(
 	t StateChangeType, payload interface{}) (err error) {
+	s.logger.Info("Request Change to State", "type", t, "value", payload)
 	// Patch input parameter's type.
 	switch t {
 	case StateAddNode:
diff --git a/simulation/config/config.go b/simulation/config/config.go
index 023c4df..797145c 100644
--- a/simulation/config/config.go
+++ b/simulation/config/config.go
@@ -18,24 +18,27 @@
 package config
 
 import (
+	"fmt"
 	"math"
 	"os"
 
+	"github.com/dexon-foundation/dexon-consensus/core"
 	"github.com/dexon-foundation/dexon-consensus/core/test"
 	"github.com/naoina/toml"
 )
 
 // Consensus settings.
 type Consensus struct {
-	PhiRatio      float32
-	K             int
-	ChainNum      uint32
-	GenesisCRS    string `toml:"genesis_crs"`
-	LambdaBA      int    `toml:"lambda_ba"`
-	LambdaDKG     int    `toml:"lambda_dkg"`
-	RoundInterval int
-	NotarySetSize uint32
-	DKGSetSize    uint32 `toml:"dkg_set_size"`
+	PhiRatio         float32
+	K                int
+	NumChains        uint32
+	GenesisCRS       string `toml:"genesis_crs"`
+	LambdaBA         int    `toml:"lambda_ba"`
+	LambdaDKG        int    `toml:"lambda_dkg"`
+	RoundInterval    int
+	NotarySetSize    uint32
+	DKGSetSize       uint32 `toml:"dkg_set_size"`
+	MinBlockInterval int
 }
 
 // Legacy config.
@@ -50,6 +53,7 @@ type Node struct {
 	Legacy    Legacy
 	Num       uint32
 	MaxBlock  uint64
+	Changes   []Change
 }
 
 // Networking config.
@@ -67,6 +71,25 @@ type Scheduler struct {
 	WorkerNum int
 }
 
+// Change represent future configuration changes.
+type Change struct {
+	Round uint64
+	Type  string
+	Value string
+}
+
+// RegisterChange reigster this change to a test.Governance instance.
+func (c Change) RegisterChange(gov *test.Governance) error {
+	if c.Round < core.ConfigRoundShift {
+		panic(fmt.Errorf(
+			"attempt to register config change that never be executed: %v",
+			c.Round))
+	}
+	t := StateChangeTypeFromString(c.Type)
+	return gov.RegisterConfigChange(
+		c.Round, t, StateChangeValueFromString(t, c.Value))
+}
+
 // Config represents the configuration for simulation.
 type Config struct {
 	Title      string
@@ -87,15 +110,16 @@ func GenerateDefault(path string) error {
 		Title: "DEXON Consensus Simulation Config",
 		Node: Node{
 			Consensus: Consensus{
-				PhiRatio:      float32(2) / 3,
-				K:             1,
-				ChainNum:      7,
-				GenesisCRS:    "In DEXON we trust.",
-				LambdaBA:      250,
-				LambdaDKG:     1000,
-				RoundInterval: 30 * 1000,
-				NotarySetSize: 7,
-				DKGSetSize:    7,
+				PhiRatio:         float32(2) / 3,
+				K:                1,
+				NumChains:        7,
+				GenesisCRS:       "In DEXON we trust.",
+				LambdaBA:         250,
+				LambdaDKG:        1000,
+				RoundInterval:    30 * 1000,
+				NotarySetSize:    7,
+				DKGSetSize:       7,
+				MinBlockInterval: 750,
 			},
 			Legacy: Legacy{
 				ProposeIntervalMean:  500,
diff --git a/simulation/config/utils.go b/simulation/config/utils.go
new file mode 100644
index 0000000..9d97fbd
--- /dev/null
+++ b/simulation/config/utils.go
@@ -0,0 +1,80 @@
+// Copyright 2018 The dexon-consensus Authors
+// This file is part of the dexon-consensus library.
+//
+// The dexon-consensus library is free software: you can redistribute it
+// and/or modify it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation, either version 3 of the License,
+// or (at your option) any later version.
+//
+// The dexon-consensus library is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the dexon-consensus library. If not, see
+// <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/dexon-foundation/dexon-consensus/core/test"
+)
+
+// StateChangeTypeFromString convert a string to test.StateChangeType.
+func StateChangeTypeFromString(s string) test.StateChangeType {
+	switch s {
+	case "num_chains":
+		return test.StateChangeNumChains
+	case "lambda_ba":
+		return test.StateChangeLambdaBA
+	case "lambda_dkg":
+		return test.StateChangeLambdaDKG
+	case "round_interval":
+		return test.StateChangeRoundInterval
+	case "min_block_interval":
+		return test.StateChangeMinBlockInterval
+	case "k":
+		return test.StateChangeK
+	case "phi_ratio":
+		return test.StateChangePhiRatio
+	case "notary_set_size":
+		return test.StateChangeNotarySetSize
+	case "dkg_set_size":
+		return test.StateChangeDKGSetSize
+	}
+	panic(fmt.Errorf("unsupported state change type %s", s))
+}
+
+// StateChangeValueFromString converts a string to a value for state change
+// request.
+func StateChangeValueFromString(
+	t test.StateChangeType, v string) interface{} {
+	switch t {
+	case test.StateChangeNumChains, test.StateChangeNotarySetSize,
+		test.StateChangeDKGSetSize:
+		ret, err := strconv.ParseUint(v, 10, 32)
+		if err != nil {
+			panic(err)
+		}
+		return uint32(ret)
+	case test.StateChangeLambdaBA, test.StateChangeLambdaDKG,
+		test.StateChangeRoundInterval, test.StateChangeMinBlockInterval,
+		test.StateChangeK:
+		ret, err := strconv.ParseInt(v, 10, 32)
+		if err != nil {
+			panic(err)
+		}
+		return int(ret)
+	case test.StateChangePhiRatio:
+		ret, err := strconv.ParseFloat(v, 32)
+		if err != nil {
+			panic(err)
+		}
+		return float32(ret)
+	}
+	panic(fmt.Errorf("unsupported state change type %s", t))
+}
diff --git a/simulation/marshaller.go b/simulation/marshaller.go
index 86eab3e..6f8aee4 100644
--- a/simulation/marshaller.go
+++ b/simulation/marshaller.go
@@ -30,12 +30,12 @@ type jsonMarshaller struct{}
 func (m *jsonMarshaller) Unmarshal(
 	msgType string, payload []byte) (msg interface{}, err error) {
 	switch msgType {
-	case "info-status":
-		var status infoStatus
-		if err = json.Unmarshal(payload, &status); err != nil {
+	case "server-notif":
+		var notif serverNotification
+		if err = json.Unmarshal(payload, &notif); err != nil {
 			break
 		}
-		msg = status
+		msg = notif
 	case "blocklist":
 		var blocks BlockList
 		if err = json.Unmarshal(payload, &blocks); err != nil {
@@ -61,8 +61,8 @@ func (m *jsonMarshaller) Unmarshal(
 func (m *jsonMarshaller) Marshal(msg interface{}) (
 	msgType string, payload []byte, err error) {
 	switch msg.(type) {
-	case infoStatus:
-		msgType = "info-status"
+	case serverNotification:
+		msgType = "server-notif"
 	case *BlockList:
 		msgType = "blocklist"
 	case *message:
diff --git a/simulation/node.go b/simulation/node.go
index e766da8..dc4a725 100644
--- a/simulation/node.go
+++ b/simulation/node.go
@@ -31,17 +31,18 @@ import (
 	"github.com/dexon-foundation/dexon-consensus/simulation/config"
 )
 
-type infoStatus string
+type serverNotification string
 
 const (
-	statusInit     infoStatus = "init"
-	statusNormal   infoStatus = "normal"
-	statusShutdown infoStatus = "shutdown"
+	ntfShutdown         serverNotification = "shutdown"
+	ntfSelectedAsMaster serverNotification = "as_master"
+	ntfReady            serverNotification = "ready"
 )
 
 type messageType string
 
 const (
+	setupOK        messageType = "setupOK"
 	shutdownAck    messageType = "shutdownAck"
 	blockTimestamp messageType = "blockTimestamps"
 )
@@ -60,24 +61,25 @@ type node struct {
 	netModule *test.Network
 	ID        types.NodeID
 	prvKey    crypto.PrivateKey
+	logger    common.Logger
 	consensus *core.Consensus
+	cfg       *config.Config
 }
 
 // newNode returns a new empty node.
-func newNode(
-	prvKey crypto.PrivateKey,
-	config config.Config) *node {
+func newNode(prvKey crypto.PrivateKey, logger common.Logger,
+	cfg config.Config) *node {
 	pubKey := prvKey.PublicKey()
 	netModule := test.NewNetwork(
 		pubKey,
 		&test.NormalLatencyModel{
-			Mean:  config.Networking.Mean,
-			Sigma: config.Networking.Sigma,
+			Mean:  cfg.Networking.Mean,
+			Sigma: cfg.Networking.Sigma,
 		},
 		test.NewDefaultMarshaller(&jsonMarshaller{}),
 		test.NetworkConfig{
-			Type:       config.Networking.Type,
-			PeerServer: config.Networking.PeerServer,
+			Type:       cfg.Networking.Type,
+			PeerServer: cfg.Networking.PeerServer,
 			PeerPort:   peerPort,
 		})
 	id := types.NewNodeID(pubKey)
@@ -86,40 +88,22 @@ func newNode(
 		panic(err)
 	}
 	// Sync config to state in governance.
-	cConfig := config.Node.Consensus
 	gov, err := test.NewGovernance(
 		test.NewState(
-			[]crypto.PublicKey{pubKey},
-			time.Millisecond,
-			&common.NullLogger{},
-			true),
+			[]crypto.PublicKey{pubKey}, time.Millisecond, logger, true),
 		core.ConfigRoundShift)
 	if err != nil {
 		panic(err)
 	}
-	gov.State().RequestChange(test.StateChangeK, cConfig.K)
-	gov.State().RequestChange(test.StateChangePhiRatio, cConfig.PhiRatio)
-	gov.State().RequestChange(test.StateChangeNumChains, cConfig.ChainNum)
-	gov.State().RequestChange(
-		test.StateChangeNotarySetSize, cConfig.NotarySetSize)
-	gov.State().RequestChange(test.StateChangeDKGSetSize, cConfig.DKGSetSize)
-	gov.State().RequestChange(test.StateChangeLambdaBA, time.Duration(
-		cConfig.LambdaBA)*time.Millisecond)
-	gov.State().RequestChange(test.StateChangeLambdaDKG, time.Duration(
-		cConfig.LambdaDKG)*time.Millisecond)
-	gov.State().RequestChange(test.StateChangeRoundInterval, time.Duration(
-		cConfig.RoundInterval)*time.Millisecond)
-	gov.State().RequestChange(
-		test.StateChangeMinBlockInterval,
-		3*time.Duration(cConfig.LambdaBA)*time.Millisecond)
-	gov.State().ProposeCRS(0, crypto.Keccak256Hash([]byte(cConfig.GenesisCRS)))
 	return &node{
 		ID:        id,
 		prvKey:    prvKey,
+		logger:    logger,
 		app:       newSimApp(id, netModule, gov.State()),
 		gov:       gov,
 		db:        dbInst,
 		netModule: netModule,
+		cfg:       &cfg,
 	}
 }
 
@@ -130,7 +114,7 @@ func (n *node) GetID() types.NodeID {
 
 // run starts the node.
 func (n *node) run(
-	serverEndpoint interface{}, dMoment time.Time, logger common.Logger) {
+	serverEndpoint interface{}, dMoment time.Time) {
 	// Run network.
 	if err := n.netModule.Setup(serverEndpoint); err != nil {
 		panic(err)
@@ -145,10 +129,32 @@ func (n *node) run(
 		n.gov.State().RequestChange(test.StateAddNode, pubKey)
 		hashes = append(hashes, nID.Hash)
 	}
-	// This notification is implictly called in full node.
-	n.gov.NotifyRoundHeight(0, 0)
-	// Setup of governance is ready, can be switched to remote mode.
-	n.gov.SwitchToRemoteMode(n.netModule)
+	n.prepareConfigs()
+	if err := n.netModule.Report(&message{Type: setupOK}); err != nil {
+		panic(err)
+	}
+	// Wait for a "ready" server notification.
+readyLoop:
+	for {
+		msg := <-msgChannel
+		ntf := msg.(serverNotification)
+		switch ntf {
+		case ntfReady:
+			break readyLoop
+		case ntfSelectedAsMaster:
+			n.logger.Info(
+				"receive 'selected-as-master' notification from server")
+			for _, c := range n.cfg.Node.Changes {
+				if c.Round <= core.ConfigRoundShift+1 {
+					continue
+				}
+				n.logger.Info("register config change", "change", c)
+				c.RegisterChange(n.gov)
+			}
+		default:
+			panic(fmt.Errorf("receive unexpected server notification: %v", ntf))
+		}
+	}
 	// Setup Consensus.
 	n.consensus = core.NewConsensusForSimulation(
 		dMoment,
@@ -157,7 +163,7 @@ func (n *node) run(
 		n.db,
 		n.netModule,
 		n.prvKey,
-		logger)
+		n.logger)
 	go n.consensus.Run()
 
 	// Blocks forever.
@@ -165,8 +171,9 @@ MainLoop:
 	for {
 		msg := <-msgChannel
 		switch val := msg.(type) {
-		case infoStatus:
-			if val == statusShutdown {
+		case serverNotification:
+			if val == ntfShutdown {
+				n.logger.Info("receive shutdown notification from server")
 				break MainLoop
 			}
 		default:
@@ -178,10 +185,40 @@ MainLoop:
 	if err := n.db.Close(); err != nil {
 		fmt.Println(err)
 	}
-	n.netModule.Report(&message{
-		Type: shutdownAck,
-	})
+	if err := n.netModule.Report(&message{Type: shutdownAck}); err != nil {
+		panic(err)
+	}
 	// TODO(mission): once we have a way to know if consensus is stopped, stop
 	//                the network module.
 	return
 }
+
+func (n *node) prepareConfigs() {
+	// Prepare configurations.
+	cConfig := n.cfg.Node.Consensus
+	n.gov.State().RequestChange(test.StateChangeK, cConfig.K)
+	n.gov.State().RequestChange(test.StateChangePhiRatio, cConfig.PhiRatio)
+	n.gov.State().RequestChange(test.StateChangeNumChains, cConfig.NumChains)
+	n.gov.State().RequestChange(
+		test.StateChangeNotarySetSize, cConfig.NotarySetSize)
+	n.gov.State().RequestChange(test.StateChangeDKGSetSize, cConfig.DKGSetSize)
+	n.gov.State().RequestChange(test.StateChangeLambdaBA, time.Duration(
+		cConfig.LambdaBA)*time.Millisecond)
+	n.gov.State().RequestChange(test.StateChangeLambdaDKG, time.Duration(
+		cConfig.LambdaDKG)*time.Millisecond)
+	n.gov.State().RequestChange(test.StateChangeRoundInterval, time.Duration(
+		cConfig.RoundInterval)*time.Millisecond)
+	n.gov.State().RequestChange(test.StateChangeMinBlockInterval, time.Duration(
+		cConfig.MinBlockInterval)*time.Millisecond)
+	n.gov.State().ProposeCRS(0, crypto.Keccak256Hash([]byte(cConfig.GenesisCRS)))
+	// These rounds are not safe to be registered as pending state change
+	// requests.
+	for i := uint64(0); i <= core.ConfigRoundShift+1; i++ {
+		n.logger.Info("prepare config", "round", i)
+		prepareConfigs(i, n.cfg.Node.Changes, n.gov)
+	}
+	// This notification is implictly called in full node.
+	n.gov.NotifyRoundHeight(0, 0)
+	// Setup of configuration is ready, can be switched to remote mode.
+	n.gov.SwitchToRemoteMode(n.netModule)
+}
diff --git a/simulation/peer-server.go b/simulation/peer-server.go
index 6d1121f..14a825a 100644
--- a/simulation/peer-server.go
+++ b/simulation/peer-server.go
@@ -90,7 +90,7 @@ func (p *PeerServer) handleBlockList(id types.NodeID, blocks *BlockList) {
 		}
 		p.verifiedLen += uint64(length)
 		if p.verifiedLen >= p.cfg.Node.MaxBlock {
-			if err := p.trans.Broadcast(statusShutdown); err != nil {
+			if err := p.trans.Broadcast(ntfShutdown); err != nil {
 				panic(err)
 			}
 		}
@@ -201,6 +201,34 @@ func (p *PeerServer) Run() {
 	for _, pubKey := range p.trans.Peers() {
 		p.peers[types.NewNodeID(pubKey)] = struct{}{}
 	}
+	// Pick a mater node to execute pending config changes.
+	for nID := range p.peers {
+		if err := p.trans.Send(nID, ntfSelectedAsMaster); err != nil {
+			panic(err)
+		}
+		break
+	}
+	// Wait for peers to report 'setupOK' message.
+	readyPeers := make(map[types.NodeID]struct{})
+	for {
+		e := <-p.msgChannel
+		if !p.isNode(e.From) {
+			break
+		}
+		msg := e.Msg.(*message)
+		if msg.Type != setupOK {
+			panic(fmt.Errorf("receive an unexpected peer report: %v", msg))
+		}
+		log.Println("receive setupOK message from", e.From)
+		readyPeers[e.From] = struct{}{}
+		if len(readyPeers) == len(p.peers) {
+			break
+		}
+	}
+	if err := p.trans.Broadcast(ntfReady); err != nil {
+		panic(err)
+	}
+	log.Println("Simulation is ready to go with", len(p.peers), "nodes")
 	// Initialize total order result cache.
 	for id := range p.peers {
 		p.peerTotalOrder[id] = NewTotalOrderResult(id)
diff --git a/simulation/simulation.go b/simulation/simulation.go
index 3a0e9bb..10ccb4c 100644
--- a/simulation/simulation.go
+++ b/simulation/simulation.go
@@ -71,11 +71,11 @@ func Run(cfg *config.Config, logPrefix string) {
 		if err != nil {
 			panic(err)
 		}
-		v := newNode(prv, *cfg)
+		v := newNode(prv, logger, *cfg)
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			v.run(serverEndpoint, dMoment, logger)
+			v.run(serverEndpoint, dMoment)
 		}()
 	}
 
diff --git a/simulation/utils.go b/simulation/utils.go
index a18e8a4..a305d2b 100644
--- a/simulation/utils.go
+++ b/simulation/utils.go
@@ -20,6 +20,9 @@ package simulation
 import (
 	"math"
 	"sort"
+
+	"github.com/dexon-foundation/dexon-consensus/core/test"
+	"github.com/dexon-foundation/dexon-consensus/simulation/config"
 )
 
 func calculateMeanStdDeviationFloat64s(a []float64) (float64, float64) {
@@ -57,3 +60,18 @@ func getMinMedianMaxFloat64s(a []float64) (float64, float64, float64) {
 	sort.Float64s(aCopied)
 	return aCopied[0], aCopied[len(aCopied)/2], aCopied[len(aCopied)-1]
 }
+
+func prepareConfigs(
+	round uint64, cfgs []config.Change, gov *test.Governance) {
+	for _, c := range cfgs {
+		if c.Round != round {
+			continue
+		}
+		t := config.StateChangeTypeFromString(c.Type)
+		if err := gov.State().RequestChange(
+			t, config.StateChangeValueFromString(t, c.Value)); err != nil {
+			panic(err)
+		}
+	}
+	gov.CatchUpWithRound(round)
+}
diff --git a/test_config/test-config-change.toml b/test_config/test-config-change.toml
new file mode 100644
index 0000000..5950b30
--- /dev/null
+++ b/test_config/test-config-change.toml
@@ -0,0 +1,46 @@
+title = "DEXON Consensus Simulation Config"
+
+[node]
+num = 7
+max_block = 18446744073709551615
+
+# node.consensus is the genesis configuration.
+[node.consensus]
+phi_ratio = 6.666666865348816e-01
+k = 1
+num_chains = 4
+genesis_crs = "In DEXON we trust."
+lambda_ba = 250
+lambda_dkg = 4000
+round_interval = 100000
+notary_set_size = 7
+dkg_set_size = 7
+min_block_interval = 750
+
+# node.config_changes describe the changes of configuration for each round.
+[[node.changes]]
+round = 1
+type = "num_chains"
+value = "5"
+[[node.changes]]
+round = 2
+type = "num_chains"
+value = "6"
+[[node.changes]]
+round = 3
+type = "num_chains"
+value = "4"
+
+[node.legacy]
+propose_interval_mean = 5e+02
+propose_interval_sigma = 5e+01
+
+[networking]
+type = "fake"
+peer_server = "127.0.0.1"
+mean = 1e+01
+sigma = 1e+01
+loss_rate_value = 0e+00
+
+[scheduler]
+worker_num = 2
diff --git a/test_config/test.toml b/test_config/test.toml
index 4c4d979..0261d8d 100644
--- a/test_config/test.toml
+++ b/test_config/test.toml
@@ -14,6 +14,7 @@ lambda_dkg = 4000
 round_interval = 200000
 notary_set_size = 7
 dkg_set_size = 7
+min_block_interval = 750
 
 [node.legacy]
 propose_interval_mean = 5e+02