swarm: network simulation for swarm tests (#769)

* cmd/swarm: minor cli flag text adjustments

* cmd/swarm, swarm/storage, swarm: fix  mingw on windows test issues

* cmd/swarm: support for smoke tests on the production swarm cluster

* cmd/swarm/swarm-smoke: simplify cluster logic as per suggestion

* changed colour of landing page

* landing page reacts to enter keypress

* swarm/api/http: sticky footer for swarm landing page using flex

* swarm/api/http: sticky footer for error pages and fix for multiple choices

* swarm: propagate ctx to internal apis (#754)

* swarm/simnet: add basic node/service functions

* swarm/netsim: add buckets for global state and kademlia health check

* swarm/netsim: Use sync.Map as bucket and provide cleanup function for...

* swarm, swarm/netsim: adjust SwarmNetworkTest

* swarm/netsim: fix tests

* swarm: added visualization option to sim net redesign

* swarm/netsim: support multiple services per node

* swarm/netsim: remove redundant return statement

* swarm/netsim: add comments

* swarm: shutdown HTTP in Simulation.Close

* swarm: sim HTTP server timeout

* swarm/netsim: add more simulation methods and peer events examples

* swarm/netsim: add WaitKademlia example

* swarm/netsim: fix comments

* swarm/netsim: terminate peer events goroutines on simulation done

* swarm, swarm/netsim: naming updates

* swarm/netsim: return not healthy kademlias on WaitTillHealthy

* swarm: fix WaitTillHealthy call in testSwarmNetwork

* swarm/netsim: allow bucket to have any type for a key

* swarm: Added snapshots to new netsim

* swarm/netsim: add more tests for bucket

* swarm/netsim: move http related things into separate files

* swarm/netsim: add AddNodeWithService option

* swarm/netsim: add more tests and Start* methods

* swarm/netsim: add peer events and kademlia tests

* swarm/netsim: fix some tests flakiness

* swarm/netsim: improve random nodes selection, fix TestStartStop* tests

* swarm/netsim: remove time measurement from TestClose to avoid flakiness

* swarm/netsim: builder pattern for netsim HTTP server (#773)

* swarm/netsim: add connect related tests

* swarm/netsim: add comment for TestPeerEvents

* swarm: rename netsim package to network/simulation

1 files changed, 201 insertions, 0 deletions

diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go
new file mode 100644
index 000000000..2241dfca2
--- /dev/null
+++ b/swarm/network/simulation/simulation.go
@@ -0,0 +1,201 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package simulation
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+)
+
+// Common errors that are returned by functions in this package.
+var (
+	ErrNodeNotFound = errors.New("node not found")
+	ErrNoPivotNode  = errors.New("no pivot node set")
+)
+
+// Simulation provides methods on network, nodes and services
+// to manage them.
+type Simulation struct {
+	// Net is exposed as a way to access lower level functionalities
+	// of p2p/simulations.Network.
+	Net *simulations.Network
+
+	serviceNames []string
+	cleanupFuncs []func()
+	buckets      map[discover.NodeID]*sync.Map
+	pivotNodeID  *discover.NodeID
+	shutdownWG   sync.WaitGroup
+	done         chan struct{}
+	mu           sync.RWMutex
+
+	httpSrv *http.Server        //attach a HTTP server via SimulationOptions
+	handler *simulations.Server //HTTP handler for the server
+	runC    chan struct{}       //channel where frontend signals it is ready
+}
+
+// ServiceFunc is used in New to declare new service constructor.
+// The first argument provides ServiceContext from the adapters package
+// giving for example the access to NodeID. Second argument is the sync.Map
+// where all "global" state related to the service should be kept.
+// All cleanups needed for constructed service and any other constructed
+// objects should ne provided in a single returned cleanup function.
+type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error)
+
+// New creates a new Simulation instance with new
+// simulations.Network initialized with provided services.
+func New(services map[string]ServiceFunc) (s *Simulation) {
+	s = &Simulation{
+		buckets: make(map[discover.NodeID]*sync.Map),
+		done:    make(chan struct{}),
+	}
+
+	adapterServices := make(map[string]adapters.ServiceFunc, len(services))
+	for name, serviceFunc := range services {
+		s.serviceNames = append(s.serviceNames, name)
+		adapterServices[name] = func(ctx *adapters.ServiceContext) (node.Service, error) {
+			b := new(sync.Map)
+			service, cleanup, err := serviceFunc(ctx, b)
+			if err != nil {
+				return nil, err
+			}
+			s.mu.Lock()
+			defer s.mu.Unlock()
+			if cleanup != nil {
+				s.cleanupFuncs = append(s.cleanupFuncs, cleanup)
+			}
+			s.buckets[ctx.Config.ID] = b
+			return service, nil
+		}
+	}
+
+	s.Net = simulations.NewNetwork(
+		adapters.NewSimAdapter(adapterServices),
+		&simulations.NetworkConfig{ID: "0"},
+	)
+
+	return s
+}
+
+// RunFunc is the function that will be called
+// on Simulation.Run method call.
+type RunFunc func(context.Context, *Simulation) error
+
+// Result is the returned value of Simulation.Run method.
+type Result struct {
+	Duration time.Duration
+	Error    error
+}
+
+// Run calls the RunFunc function while taking care of
+// cancelation provided through the Context.
+func (s *Simulation) Run(ctx context.Context, f RunFunc) (r Result) {
+	//if the option is set to run a HTTP server with the simulation,
+	//init the server and start it
+	start := time.Now()
+	if s.httpSrv != nil {
+		log.Info("Waiting for frontend to be ready...(send POST /runsim to HTTP server)")
+		//wait for the frontend to connect
+		select {
+		case <-s.runC:
+		case <-ctx.Done():
+			return Result{
+				Duration: time.Since(start),
+				Error:    ctx.Err(),
+			}
+		}
+		log.Info("Received signal from frontend - starting simulation run.")
+	}
+	errc := make(chan error)
+	quit := make(chan struct{})
+	defer close(quit)
+	go func() {
+		select {
+		case errc <- f(ctx, s):
+		case <-quit:
+		}
+	}()
+	var err error
+	select {
+	case <-ctx.Done():
+		err = ctx.Err()
+	case err = <-errc:
+	}
+	return Result{
+		Duration: time.Since(start),
+		Error:    err,
+	}
+}
+
+// Maximal number of parallel calls to cleanup functions on
+// Simulation.Close.
+var maxParallelCleanups = 10
+
+// Close calls all cleanup functions that are returned by
+// ServiceFunc, waits for all of them to finish and other
+// functions that explicitly block shutdownWG
+// (like Simulation.PeerEvents) and shuts down the network
+// at the end. It is used to clean all resources from the
+// simulation.
+func (s *Simulation) Close() {
+	close(s.done)
+	sem := make(chan struct{}, maxParallelCleanups)
+	s.mu.RLock()
+	cleanupFuncs := make([]func(), len(s.cleanupFuncs))
+	for i, f := range s.cleanupFuncs {
+		if f != nil {
+			cleanupFuncs[i] = f
+		}
+	}
+	s.mu.RUnlock()
+	for _, cleanup := range cleanupFuncs {
+		s.shutdownWG.Add(1)
+		sem <- struct{}{}
+		go func(cleanup func()) {
+			defer s.shutdownWG.Done()
+			defer func() { <-sem }()
+
+			cleanup()
+		}(cleanup)
+	}
+	if s.httpSrv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		err := s.httpSrv.Shutdown(ctx)
+		if err != nil {
+			log.Error("Error shutting down HTTP server!", "err", err)
+		}
+		close(s.runC)
+	}
+	s.shutdownWG.Wait()
+	s.Net.Shutdown()
+}
+
+// Done returns a channel that is closed when the simulation
+// is closed by Close method. It is useful for signaling termination
+// of all possible goroutines that are created within the test.
+func (s *Simulation) Done() <-chan struct{} {
+	return s.done
+}