From cd094147171175d49c82dd9783ce880f6307e175 Mon Sep 17 00:00:00 2001 From: Sonic Date: Wed, 17 Oct 2018 14:10:52 +0800 Subject: dex: let notary nodes connect to some dkg nodes BuildConnection builds notary and dkg connections together --- dex/handler.go | 33 ++++---- dex/helper_test.go | 9 ++- dex/peer.go | 216 +++++++++++++++++++++++++++++++++++++++++++---------- dex/peer_test.go | 131 +------------------------------- 4 files changed, 204 insertions(+), 185 deletions(-) diff --git a/dex/handler.go b/dex/handler.go index b605c907a..cdadd2874 100644 --- a/dex/handler.go +++ b/dex/handler.go @@ -824,8 +824,16 @@ func (pm *ProtocolManager) BroadcastMetas(metas []*NodeMeta) { } func (pm *ProtocolManager) BroadcastVote(vote *coreTypes.Vote) { - for _, peer := range pm.peers.allPeers() { - peer.AsyncSendVote(vote) + label := peerLabel{ + set: notaryset, + chainID: vote.Position.ChainID, + round: vote.Position.Round, + } + h := rlpHash(vote) + for _, peer := range pm.peers.PeersWithLabel(label) { + if !peer.knownVotes.Contains(h) { + peer.AsyncSendVote(vote) + } } } @@ -923,11 +931,9 @@ func (pm *ProtocolManager) peerSetLoop() { round := pm.gov.LenCRS() - 1 log.Trace("first len crs", "len", round+1, "round", round) if round >= 1 { - pm.peers.BuildNotaryConn(round - 1) - pm.peers.BuildDKGConn(round - 1) + pm.peers.BuildConnection(round - 1) } - pm.peers.BuildNotaryConn(round) - pm.peers.BuildDKGConn(round) + pm.peers.BuildConnection(round) for { select { @@ -938,21 +944,16 @@ func (pm *ProtocolManager) peerSetLoop() { break } if newRound == round+1 { - pm.peers.BuildNotaryConn(newRound) - pm.peers.BuildDKGConn(newRound) - pm.peers.ForgetNotaryConn(round - 1) - pm.peers.ForgetDKGConn(round - 1) + pm.peers.BuildConnection(newRound) + pm.peers.ForgetConnection(round - 1) } else { // just forget all network connection and rebuild. - pm.peers.ForgetNotaryConn(round) - pm.peers.ForgetDKGConn(round) + pm.peers.ForgetConnection(round) if newRound >= 1 { - pm.peers.BuildNotaryConn(newRound - 1) - pm.peers.BuildDKGConn(newRound - 1) + pm.peers.BuildConnection(newRound - 1) } - pm.peers.BuildNotaryConn(newRound) - pm.peers.BuildDKGConn(newRound) + pm.peers.BuildConnection(newRound) } round = newRound case <-time.After(5 * time.Second): diff --git a/dex/helper_test.go b/dex/helper_test.go index 5b73c4afa..a077304b1 100644 --- a/dex/helper_test.go +++ b/dex/helper_test.go @@ -111,7 +111,14 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func panic(err) } - pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, &testGovernance{}) + tgov := &testGovernance{ + numChainsFunc: func(uint64) uint32 { return 3 }, + lenCRSFunc: func() uint64 { return 1 }, + dkgSetFunc: func(uint64) (map[string]struct{}, error) { return nil, nil }, + notarySetFunc: func(uint64, uint32) (map[string]struct{}, error) { return nil, nil }, + } + + pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, tgov) if err != nil { return nil, nil, err } diff --git a/dex/peer.go b/dex/peer.go index 1f2b9518d..1279a190b 100644 --- a/dex/peer.go +++ b/dex/peer.go @@ -25,11 +25,13 @@ import ( mapset "github.com/deckarep/golang-set" coreTypes "github.com/dexon-foundation/dexon-consensus-core/core/types" + "github.com/dexon-foundation/dexon/common" "github.com/dexon-foundation/dexon/core/types" "github.com/dexon-foundation/dexon/log" "github.com/dexon-foundation/dexon/p2p" - "github.com/dexon-foundation/dexon/p2p/discover" + "github.com/dexon-foundation/dexon/p2p/enode" + "github.com/dexon-foundation/dexon/p2p/enr" "github.com/dexon-foundation/dexon/rlp" ) @@ -100,7 +102,6 @@ type peer struct { rw p2p.MsgReadWriter version int // Protocol version negotiated - labels mapset.Set head common.Hash td *big.Int @@ -128,7 +129,6 @@ func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { Peer: p, rw: rw, version: version, - labels: mapset.NewSet(), id: p.ID().String(), knownTxs: mapset.NewSet(), knownMetas: mapset.NewSet(), @@ -218,14 +218,6 @@ func (p *peer) close() { close(p.term) } -func (p *peer) addLabel(label peerLabel) { - p.labels.Add(label) -} - -func (p *peer) removeLabel(label peerLabel) { - p.labels.Remove(label) -} - // Info gathers and returns a collection of metadata known about a peer. func (p *peer) Info() *PeerInfo { hash, td := p.Head() @@ -593,6 +585,7 @@ type peerSet struct { gov governance peer2Labels map[string]map[peerLabel]struct{} label2Peers map[peerLabel]map[string]struct{} + history map[uint64]struct{} notaryHistory map[uint64]struct{} dkgHistory map[uint64]struct{} } @@ -606,6 +599,7 @@ func newPeerSet(gov governance, srvr p2pServer, tab *nodeTable) *peerSet { tab: tab, peer2Labels: make(map[string]map[peerLabel]struct{}), label2Peers: make(map[peerLabel]map[string]struct{}), + history: make(map[uint64]struct{}), notaryHistory: make(map[uint64]struct{}), dkgHistory: make(map[uint64]struct{}), } @@ -692,6 +686,18 @@ func (ps *peerSet) PeersWithoutTx(hash common.Hash) []*peer { return list } +func (ps *peerSet) PeersWithLabel(label peerLabel) []*peer { + ps.lock.RLock() + defer ps.lock.RUnlock() + list := make([]*peer, 0, len(ps.label2Peers[label])) + for id := range ps.label2Peers[label] { + if p, ok := ps.peers[id]; ok { + list = append(list, p) + } + } + return list +} + func (ps *peerSet) PeersWithoutVote(hash common.Hash, label peerLabel) []*peer { ps.lock.RLock() defer ps.lock.RUnlock() @@ -775,6 +781,140 @@ func (ps *peerSet) Close() { ps.closed = true } +func (ps *peerSet) BuildConnection(round uint64) { + ps.lock.Lock() + defer ps.lock.Unlock() + defer ps.dumpPeerLabel(fmt.Sprintf("BuildConnection: %d", round)) + + ps.history[round] = struct{}{} + + selfID := ps.srvr.Self().ID().String() + + dkgIDs, err := ps.gov.DKGSet(round) + if err != nil { + log.Error("get dkg set fail", "round", round, "err", err) + } + + // build dkg connection + _, inDKGSet := dkgIDs[selfID] + if inDKGSet { + delete(dkgIDs, selfID) + dkgLabel := peerLabel{set: dkgset, round: round} + for id := range dkgIDs { + ps.addDirectPeer(id, dkgLabel) + } + } + + var inOneNotarySet bool + for cid := uint32(0); cid < ps.gov.GetNumChains(round); cid++ { + notaryIDs, err := ps.gov.NotarySet(round, cid) + if err != nil { + log.Error("get notary set fail", + "round", round, "chain id", cid, "err", err) + continue + } + + label := peerLabel{set: notaryset, chainID: cid, round: round} + // not in notary set, add group + if _, ok := notaryIDs[selfID]; !ok { + var nodes []*enode.Node + for id := range notaryIDs { + nodes = append(nodes, ps.newNode(id)) + ps.addLabel(id, label) + } + ps.srvr.AddGroup(notarySetName(cid, round), nodes, groupNodeNum) + continue + } + + delete(notaryIDs, selfID) + for id := range notaryIDs { + ps.addDirectPeer(id, label) + } + inOneNotarySet = true + } + + // build some connections to DKG nodes + if !inDKGSet && inOneNotarySet { + var nodes []*enode.Node + label := peerLabel{set: dkgset, round: round} + for id := range dkgIDs { + nodes = append(nodes, ps.newNode(id)) + ps.addLabel(id, label) + } + ps.srvr.AddGroup(dkgSetName(round), nodes, groupNodeNum) + } +} + +func (ps *peerSet) ForgetConnection(round uint64) { + ps.lock.Lock() + defer ps.lock.Unlock() + defer ps.dumpPeerLabel(fmt.Sprintf("ForgetConnection: %d", round)) + + for r := range ps.history { + if r <= round { + ps.forgetConnection(round) + delete(ps.history, r) + } + } +} + +func (ps *peerSet) forgetConnection(round uint64) { + selfID := ps.srvr.Self().ID().String() + dkgIDs, err := ps.gov.DKGSet(round) + if err != nil { + log.Error("get dkg set fail", "round", round, "err", err) + } + + _, inDKGSet := dkgIDs[selfID] + if inDKGSet { + delete(dkgIDs, selfID) + label := peerLabel{set: dkgset, round: round} + for id := range dkgIDs { + ps.removeDirectPeer(id, label) + } + } + + var inOneNotarySet bool + for cid := uint32(0); cid < ps.gov.GetNumChains(round); cid++ { + notaryIDs, err := ps.gov.NotarySet(round, cid) + if err != nil { + log.Error("get notary set fail", + "round", round, "chain id", cid, "err", err) + continue + } + + label := peerLabel{set: notaryset, chainID: cid, round: round} + + // not in notary set, add group + if _, ok := notaryIDs[selfID]; !ok { + var nodes []*enode.Node + for id := range notaryIDs { + nodes = append(nodes, ps.newNode(id)) + ps.removeLabel(id, label) + } + ps.srvr.RemoveGroup(notarySetName(cid, round)) + continue + } + + delete(notaryIDs, selfID) + for id := range notaryIDs { + ps.removeDirectPeer(id, label) + } + inOneNotarySet = true + } + + // build some connections to DKG nodes + if !inDKGSet && inOneNotarySet { + var nodes []*enode.Node + label := peerLabel{set: dkgset, round: round} + for id := range dkgIDs { + nodes = append(nodes, ps.newNode(id)) + ps.removeLabel(id, label) + } + ps.srvr.RemoveGroup(dkgSetName(round)) + } +} + func (ps *peerSet) BuildNotaryConn(round uint64) { ps.lock.Lock() defer ps.lock.Unlock() @@ -786,7 +926,7 @@ func (ps *peerSet) BuildNotaryConn(round uint64) { ps.notaryHistory[round] = struct{}{} - selfID := ps.srvr.Self().ID.String() + selfID := ps.srvr.Self().ID().String() for chainID := uint32(0); chainID < ps.gov.GetNumChains(round); chainID++ { s, err := ps.gov.NotarySet(round, chainID) if err != nil { @@ -797,7 +937,7 @@ func (ps *peerSet) BuildNotaryConn(round uint64) { // not in notary set, add group if _, ok := s[selfID]; !ok { - var nodes []*discover.Node + var nodes []*enode.Node for id := range s { nodes = append(nodes, ps.newNode(id)) } @@ -843,7 +983,7 @@ func (ps *peerSet) ForgetNotaryConn(round uint64) { } func (ps *peerSet) forgetNotaryConn(round uint64) { - selfID := ps.srvr.Self().ID.String() + selfID := ps.srvr.Self().ID().String() for chainID := uint32(0); chainID < ps.gov.GetNumChains(round); chainID++ { s, err := ps.gov.NotarySet(round, chainID) if err != nil { @@ -872,11 +1012,15 @@ func notarySetName(chainID uint32, round uint64) string { return fmt.Sprintf("%d-%d-notaryset", chainID, round) } +func dkgSetName(round uint64) string { + return fmt.Sprintf("%d-dkgset", round) +} + func (ps *peerSet) BuildDKGConn(round uint64) { ps.lock.Lock() defer ps.lock.Unlock() defer ps.dumpPeerLabel(fmt.Sprintf("BuildDKGConn: %d", round)) - selfID := ps.srvr.Self().ID.String() + selfID := ps.srvr.Self().ID().String() s, err := ps.gov.DKGSet(round) if err != nil { log.Error("get dkg set fail", "round", round) @@ -912,7 +1056,7 @@ func (ps *peerSet) ForgetDKGConn(round uint64) { } func (ps *peerSet) forgetDKGConn(round uint64) { - selfID := ps.srvr.Self().ID.String() + selfID := ps.srvr.Self().ID().String() s, err := ps.gov.DKGSet(round) if err != nil { log.Error("get dkg set fail", "round", round) @@ -932,45 +1076,41 @@ func (ps *peerSet) forgetDKGConn(round uint64) { } } -// make sure the ps.lock is hold +// make sure the ps.lock is held func (ps *peerSet) addDirectPeer(id string, label peerLabel) { - log.Trace("peerSet addDirectPeer", "id", id[:8], "round", label.round, "cid", label.chainID) - // if the peer exists add the label - if p, ok := ps.peers[id]; ok { - p.addLabel(label) + ps.addLabel(id, label) + ps.srvr.AddDirectPeer(ps.newNode(id)) +} + +// make sure the ps.lock is held +func (ps *peerSet) removeDirectPeer(id string, label peerLabel) { + ps.removeLabel(id, label) + if len(ps.peer2Labels[id]) == 0 { + ps.srvr.RemoveDirectPeer(ps.newNode(id)) } +} +// make sure the ps.lock is held +func (ps *peerSet) addLabel(id string, label peerLabel) { if _, ok := ps.peer2Labels[id]; !ok { ps.peer2Labels[id] = make(map[peerLabel]struct{}) } - if _, ok := ps.label2Peers[label]; !ok { ps.label2Peers[label] = make(map[string]struct{}) } - ps.peer2Labels[id][label] = struct{}{} ps.label2Peers[label][id] = struct{}{} - ps.srvr.AddDirectPeer(ps.newNode(id)) } -// make sure the ps.lock is hold -func (ps *peerSet) removeDirectPeer(id string, label peerLabel) { - if p, ok := ps.peers[id]; ok { - p.removeLabel(label) - } - +// make sure the ps.lock is held +func (ps *peerSet) removeLabel(id string, label peerLabel) { delete(ps.peer2Labels[id], label) - + delete(ps.label2Peers[label], id) if len(ps.peer2Labels[id]) == 0 { - ps.srvr.RemoveDirectPeer(ps.newNode(id)) delete(ps.peer2Labels, id) } - - if _, ok := ps.label2Peers[label]; ok { - delete(ps.label2Peers[label], id) - if len(ps.label2Peers[label]) == 0 { - delete(ps.label2Peers, label) - } + if len(ps.label2Peers[label]) == 0 { + delete(ps.label2Peers, label) } } diff --git a/dex/peer_test.go b/dex/peer_test.go index c38848b66..a4dc9269f 100644 --- a/dex/peer_test.go +++ b/dex/peer_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - mapset "github.com/deckarep/golang-set" "github.com/dexon-foundation/dexon/crypto" "github.com/dexon-foundation/dexon/p2p/discover" "github.com/dexon-foundation/dexon/p2p/enode" @@ -67,18 +66,6 @@ func TestPeerSetBuildAndForgetNotaryConn(t *testing.T) { // build round 10 ps.BuildNotaryConn(10) - err = checkLabels(peer1, []peerLabel{ - peerLabel{notaryset, 0, 10}, - }) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{notaryset, 0, 10}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(1).String(): []peerLabel{ peerLabel{notaryset, 0, 10}, @@ -111,20 +98,6 @@ func TestPeerSetBuildAndForgetNotaryConn(t *testing.T) { // build round 11 ps.BuildNotaryConn(11) - err = checkLabels(peer1, []peerLabel{ - peerLabel{notaryset, 0, 10}, - peerLabel{notaryset, 0, 11}, - }) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{notaryset, 0, 10}, - peerLabel{notaryset, 2, 11}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(1).String(): []peerLabel{ peerLabel{notaryset, 0, 10}, @@ -166,21 +139,6 @@ func TestPeerSetBuildAndForgetNotaryConn(t *testing.T) { // build round 12 ps.BuildNotaryConn(12) - err = checkLabels(peer1, []peerLabel{ - peerLabel{notaryset, 0, 10}, - peerLabel{notaryset, 0, 11}, - }) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{notaryset, 0, 10}, - peerLabel{notaryset, 2, 11}, - peerLabel{notaryset, 2, 12}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(1).String(): []peerLabel{ peerLabel{notaryset, 0, 10}, @@ -237,16 +195,6 @@ func TestPeerSetBuildAndForgetNotaryConn(t *testing.T) { // forget round 11 ps.ForgetNotaryConn(11) - err = checkLabels(peer1, []peerLabel{}) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{notaryset, 2, 12}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(2).String(): []peerLabel{ peerLabel{notaryset, 2, 12}, @@ -288,14 +236,6 @@ func TestPeerSetBuildAndForgetNotaryConn(t *testing.T) { // forget round 12 ps.ForgetNotaryConn(12) - err = checkLabels(peer1, []peerLabel{}) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{}) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{}) if err != nil { t.Error(err) @@ -350,18 +290,6 @@ func TestPeerSetBuildDKGConn(t *testing.T) { // build round 10 ps.BuildDKGConn(10) - err = checkLabels(peer1, []peerLabel{ - peerLabel{dkgset, 0, 10}, - }) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{dkgset, 0, 10}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(1).String(): []peerLabel{ peerLabel{dkgset, 0, 10}, @@ -387,18 +315,6 @@ func TestPeerSetBuildDKGConn(t *testing.T) { // build round 11 ps.BuildDKGConn(11) - err = checkLabels(peer1, []peerLabel{ - peerLabel{dkgset, 0, 10}, - }) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{dkgset, 0, 10}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(1).String(): []peerLabel{ peerLabel{dkgset, 0, 10}, @@ -424,18 +340,6 @@ func TestPeerSetBuildDKGConn(t *testing.T) { // build round 12 ps.BuildDKGConn(12) - err = checkLabels(peer1, []peerLabel{ - peerLabel{dkgset, 0, 10}, - }) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{ - peerLabel{dkgset, 0, 10}, - }) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(1).String(): []peerLabel{ peerLabel{dkgset, 0, 10}, @@ -467,14 +371,6 @@ func TestPeerSetBuildDKGConn(t *testing.T) { // forget round 11 ps.ForgetDKGConn(11) - err = checkLabels(peer1, []peerLabel{}) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{}) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{ nodeID(3).String(): []peerLabel{ peerLabel{dkgset, 0, 12}, @@ -499,14 +395,6 @@ func TestPeerSetBuildDKGConn(t *testing.T) { // forget round 12 ps.ForgetDKGConn(12) - err = checkLabels(peer1, []peerLabel{}) - if err != nil { - t.Error(err) - } - err = checkLabels(peer2, []peerLabel{}) - if err != nil { - t.Error(err) - } err = checkPeer2Labels(ps, map[string][]peerLabel{}) if err != nil { t.Error(err) @@ -521,20 +409,6 @@ func TestPeerSetBuildDKGConn(t *testing.T) { } } -func checkLabels(p *peer, want []peerLabel) error { - if p.labels.Cardinality() != len(want) { - return fmt.Errorf("num of labels mismatch: got %d, want %d", - p.labels.Cardinality(), len(want)) - } - - for _, label := range want { - if !p.labels.Contains(label) { - return fmt.Errorf("label %+v not exist", label) - } - } - return nil -} - func checkPeer2Labels(ps *peerSet, want map[string][]peerLabel) error { if len(ps.peer2Labels) != len(want) { return fmt.Errorf("peer num mismatch: got %d, want %d", @@ -629,8 +503,5 @@ func newTestNodeSet(nodes []enode.ID) map[string]struct{} { } func newDummyPeer(id enode.ID) *peer { - return &peer{ - labels: mapset.NewSet(), - id: id.String(), - } + return &peer{id: id.String()} } -- cgit v1.2.3