p2p, p2p/discover: add signed ENR generation (#17753)

This PR adds enode.LocalNode and integrates it into the p2p
subsystem. This new object is the keeper of the local node
record. For now, a new version of the record is produced every
time the client restarts. We'll make it smarter to avoid that in
the future.

There are a couple of other changes in this commit: discovery now
waits for all of its goroutines at shutdown and the p2p server
now closes the node database after discovery has shut down. This
fixes a leveldb crash in tests. p2p server startup is faster
because it doesn't need to wait for the external IP query
anymore.

2 files changed, 268 insertions, 0 deletions

diff --git a/p2p/netutil/iptrack.go b/p2p/netutil/iptrack.go
new file mode 100644
index 000000000..b9cbd5e1c
--- /dev/null
+++ b/p2p/netutil/iptrack.go
@@ -0,0 +1,130 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package netutil
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+)
+
+// IPTracker predicts the external endpoint, i.e. IP address and port, of the local host
+// based on statements made by other hosts.
+type IPTracker struct {
+	window          time.Duration
+	contactWindow   time.Duration
+	minStatements   int
+	clock           mclock.Clock
+	statements      map[string]ipStatement
+	contact         map[string]mclock.AbsTime
+	lastStatementGC mclock.AbsTime
+	lastContactGC   mclock.AbsTime
+}
+
+type ipStatement struct {
+	endpoint string
+	time     mclock.AbsTime
+}
+
+// NewIPTracker creates an IP tracker.
+//
+// The window parameters configure the amount of past network events which are kept. The
+// minStatements parameter enforces a minimum number of statements which must be recorded
+// before any prediction is made. Higher values for these parameters decrease 'flapping' of
+// predictions as network conditions change. Window duration values should typically be in
+// the range of minutes.
+func NewIPTracker(window, contactWindow time.Duration, minStatements int) *IPTracker {
+	return &IPTracker{
+		window:        window,
+		contactWindow: contactWindow,
+		statements:    make(map[string]ipStatement),
+		minStatements: minStatements,
+		contact:       make(map[string]mclock.AbsTime),
+		clock:         mclock.System{},
+	}
+}
+
+// PredictFullConeNAT checks whether the local host is behind full cone NAT. It predicts by
+// checking whether any statement has been received from a node we didn't contact before
+// the statement was made.
+func (it *IPTracker) PredictFullConeNAT() bool {
+	now := it.clock.Now()
+	it.gcContact(now)
+	it.gcStatements(now)
+	for host, st := range it.statements {
+		if c, ok := it.contact[host]; !ok || c > st.time {
+			return true
+		}
+	}
+	return false
+}
+
+// PredictEndpoint returns the current prediction of the external endpoint.
+func (it *IPTracker) PredictEndpoint() string {
+	it.gcStatements(it.clock.Now())
+
+	// The current strategy is simple: find the endpoint with most statements.
+	counts := make(map[string]int)
+	maxcount, max := 0, ""
+	for _, s := range it.statements {
+		c := counts[s.endpoint] + 1
+		counts[s.endpoint] = c
+		if c > maxcount && c >= it.minStatements {
+			maxcount, max = c, s.endpoint
+		}
+	}
+	return max
+}
+
+// AddStatement records that a certain host thinks our external endpoint is the one given.
+func (it *IPTracker) AddStatement(host, endpoint string) {
+	now := it.clock.Now()
+	it.statements[host] = ipStatement{endpoint, now}
+	if time.Duration(now-it.lastStatementGC) >= it.window {
+		it.gcStatements(now)
+	}
+}
+
+// AddContact records that a packet containing our endpoint information has been sent to a
+// certain host.
+func (it *IPTracker) AddContact(host string) {
+	now := it.clock.Now()
+	it.contact[host] = now
+	if time.Duration(now-it.lastContactGC) >= it.contactWindow {
+		it.gcContact(now)
+	}
+}
+
+func (it *IPTracker) gcStatements(now mclock.AbsTime) {
+	it.lastStatementGC = now
+	cutoff := now.Add(-it.window)
+	for host, s := range it.statements {
+		if s.time < cutoff {
+			delete(it.statements, host)
+		}
+	}
+}
+
+func (it *IPTracker) gcContact(now mclock.AbsTime) {
+	it.lastContactGC = now
+	cutoff := now.Add(-it.contactWindow)
+	for host, ct := range it.contact {
+		if ct < cutoff {
+			delete(it.contact, host)
+		}
+	}
+}
diff --git a/p2p/netutil/iptrack_test.go b/p2p/netutil/iptrack_test.go
new file mode 100644
index 000000000..a9a2998a6
--- /dev/null
+++ b/p2p/netutil/iptrack_test.go
@@ -0,0 +1,138 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package netutil
+
+import (
+	"fmt"
+	mrand "math/rand"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+)
+
+const (
+	opStatement = iota
+	opContact
+	opPredict
+	opCheckFullCone
+)
+
+type iptrackTestEvent struct {
+	op       int
+	time     int // absolute, in milliseconds
+	ip, from string
+}
+
+func TestIPTracker(t *testing.T) {
+	tests := map[string][]iptrackTestEvent{
+		"minStatements": {
+			{opPredict, 0, "", ""},
+			{opStatement, 0, "127.0.0.1", "127.0.0.2"},
+			{opPredict, 1000, "", ""},
+			{opStatement, 1000, "127.0.0.1", "127.0.0.3"},
+			{opPredict, 1000, "", ""},
+			{opStatement, 1000, "127.0.0.1", "127.0.0.4"},
+			{opPredict, 1000, "127.0.0.1", ""},
+		},
+		"window": {
+			{opStatement, 0, "127.0.0.1", "127.0.0.2"},
+			{opStatement, 2000, "127.0.0.1", "127.0.0.3"},
+			{opStatement, 3000, "127.0.0.1", "127.0.0.4"},
+			{opPredict, 10000, "127.0.0.1", ""},
+			{opPredict, 10001, "", ""}, // first statement expired
+			{opStatement, 10100, "127.0.0.1", "127.0.0.2"},
+			{opPredict, 10200, "127.0.0.1", ""},
+		},
+		"fullcone": {
+			{opContact, 0, "", "127.0.0.2"},
+			{opStatement, 10, "127.0.0.1", "127.0.0.2"},
+			{opContact, 2000, "", "127.0.0.3"},
+			{opStatement, 2010, "127.0.0.1", "127.0.0.3"},
+			{opContact, 3000, "", "127.0.0.4"},
+			{opStatement, 3010, "127.0.0.1", "127.0.0.4"},
+			{opCheckFullCone, 3500, "false", ""},
+		},
+		"fullcone_2": {
+			{opContact, 0, "", "127.0.0.2"},
+			{opStatement, 10, "127.0.0.1", "127.0.0.2"},
+			{opContact, 2000, "", "127.0.0.3"},
+			{opStatement, 2010, "127.0.0.1", "127.0.0.3"},
+			{opStatement, 3000, "127.0.0.1", "127.0.0.4"},
+			{opContact, 3010, "", "127.0.0.4"},
+			{opCheckFullCone, 3500, "true", ""},
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) { runIPTrackerTest(t, test) })
+	}
+}
+
+func runIPTrackerTest(t *testing.T, evs []iptrackTestEvent) {
+	var (
+		clock mclock.Simulated
+		it    = NewIPTracker(10*time.Second, 10*time.Second, 3)
+	)
+	it.clock = &clock
+	for i, ev := range evs {
+		evtime := time.Duration(ev.time) * time.Millisecond
+		clock.Run(evtime - time.Duration(clock.Now()))
+		switch ev.op {
+		case opStatement:
+			it.AddStatement(ev.from, ev.ip)
+		case opContact:
+			it.AddContact(ev.from)
+		case opPredict:
+			if pred := it.PredictEndpoint(); pred != ev.ip {
+				t.Errorf("op %d: wrong prediction %q, want %q", i, pred, ev.ip)
+			}
+		case opCheckFullCone:
+			pred := fmt.Sprintf("%t", it.PredictFullConeNAT())
+			if pred != ev.ip {
+				t.Errorf("op %d: wrong prediction %s, want %s", i, pred, ev.ip)
+			}
+		}
+	}
+}
+
+// This checks that old statements and contacts are GCed even if Predict* isn't called.
+func TestIPTrackerForceGC(t *testing.T) {
+	var (
+		clock  mclock.Simulated
+		window = 10 * time.Second
+		rate   = 50 * time.Millisecond
+		max    = int(window/rate) + 1
+		it     = NewIPTracker(window, window, 3)
+	)
+	it.clock = &clock
+
+	for i := 0; i < 5*max; i++ {
+		e1 := make([]byte, 4)
+		e2 := make([]byte, 4)
+		mrand.Read(e1)
+		mrand.Read(e2)
+		it.AddStatement(string(e1), string(e2))
+		it.AddContact(string(e1))
+		clock.Run(rate)
+	}
+	if len(it.contact) > 2*max {
+		t.Errorf("contacts not GCed, have %d", len(it.contact))
+	}
+	if len(it.statements) > 2*max {
+		t.Errorf("statements not GCed, have %d", len(it.statements))
+	}
+}