From 7b258c96816df56e642df7e314e8052213af70fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 19 Oct 2017 14:40:43 +0300 Subject: cmd/puppeth: concurrent server dials and health checks --- cmd/puppeth/wizard.go | 4 +- cmd/puppeth/wizard_intro.go | 24 +++-- cmd/puppeth/wizard_netstats.go | 207 ++++++++++++++++++++++++----------------- 3 files changed, 142 insertions(+), 93 deletions(-) (limited to 'cmd') diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index eb6d9e5aa..e554dbd13 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -28,6 +28,7 @@ import ( "sort" "strconv" "strings" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -75,7 +76,8 @@ type wizard struct { servers map[string]*sshClient // SSH connections to servers to administer services map[string][]string // Ethereum services known to be running on servers - in *bufio.Reader // Wrapper around stdin to allow reading user input + in *bufio.Reader // Wrapper around stdin to allow reading user input + lock sync.Mutex // Lock to protect configs during concurrent service discovery } // read reads a single line from stdin, trimming if from spaces. diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index a5fea6f85..005ee47a5 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/ethereum/go-ethereum/log" ) @@ -80,14 +81,25 @@ func (w *wizard) run() { } else if err := json.Unmarshal(blob, &w.conf); err != nil { log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err) } else { + // Dial all previously known servers concurrently + var pend sync.WaitGroup for server, pubkey := range w.conf.Servers { - log.Info("Dialing previously configured server", "server", server) - client, err := dial(server, pubkey) - if err != nil { - log.Error("Previous server unreachable", "server", server, "err", err) - } - w.servers[server] = client + pend.Add(1) + + go func(server string, pubkey []byte) { + defer pend.Done() + + log.Info("Dialing previously configured server", "server", server) + client, err := dial(server, pubkey) + if err != nil { + log.Error("Previous server unreachable", "server", server, "err", err) + } + w.lock.Lock() + w.servers[server] = client + w.lock.Unlock() + }(server, pubkey) } + pend.Wait() w.networkStats() } // Basics done, loop ad infinitum about what to do diff --git a/cmd/puppeth/wizard_netstats.go b/cmd/puppeth/wizard_netstats.go index 7d8e84242..906dfeda7 100644 --- a/cmd/puppeth/wizard_netstats.go +++ b/cmd/puppeth/wizard_netstats.go @@ -21,6 +21,7 @@ import ( "os" "sort" "strings" + "sync" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" @@ -34,112 +35,143 @@ func (w *wizard) networkStats() { log.Error("No remote machines to gather stats from") return } - protips := new(protips) + // Clear out some previous configs to refill from current scan + w.conf.ethstats = "" + w.conf.bootFull = w.conf.bootFull[:0] + w.conf.bootLight = w.conf.bootLight[:0] // Iterate over all the specified hosts and check their status - stats := make(serverStats) + var pend sync.WaitGroup + stats := make(serverStats) for server, pubkey := range w.conf.Servers { - client := w.servers[server] - logger := log.New("server", server) - logger.Info("Starting remote server health-check") - - stat := &serverStat{ - address: client.address, - services: make(map[string]map[string]string), - } - stats[client.server] = stat - - if client == nil { - conn, err := dial(server, pubkey) - if err != nil { - logger.Error("Failed to establish remote connection", "err", err) - stat.failure = err.Error() - continue + pend.Add(1) + + // Gather the service stats for each server concurrently + go func(server string, pubkey []byte) { + defer pend.Done() + + stat := w.gatherStats(server, pubkey, w.servers[server]) + + // All status checks complete, report and check next server + w.lock.Lock() + defer w.lock.Unlock() + + delete(w.services, server) + for service := range stat.services { + w.services[server] = append(w.services[server], service) } - client = conn + stats[server] = stat + }(server, pubkey) + } + pend.Wait() + + // Print any collected stats and return + stats.render() +} + +// gatherStats gathers service statistics for a particular remote server. +func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *serverStat { + // Gather some global stats to feed into the wizard + var ( + genesis string + ethstats string + bootFull []string + bootLight []string + ) + // Ensure a valid SSH connection to the remote server + logger := log.New("server", server) + logger.Info("Starting remote server health-check") + + stat := &serverStat{ + address: client.address, + services: make(map[string]map[string]string), + } + if client == nil { + conn, err := dial(server, pubkey) + if err != nil { + logger.Error("Failed to establish remote connection", "err", err) + stat.failure = err.Error() + return stat } - // Client connected one way or another, run health-checks - logger.Debug("Checking for nginx availability") - if infos, err := checkNginx(client, w.network); err != nil { - if err != ErrServiceUnknown { - stat.services["nginx"] = map[string]string{"offline": err.Error()} - } - } else { - stat.services["nginx"] = infos.Report() + client = conn + } + // Client connected one way or another, run health-checks + logger.Debug("Checking for nginx availability") + if infos, err := checkNginx(client, w.network); err != nil { + if err != ErrServiceUnknown { + stat.services["nginx"] = map[string]string{"offline": err.Error()} } - logger.Debug("Checking for ethstats availability") - if infos, err := checkEthstats(client, w.network); err != nil { - if err != ErrServiceUnknown { - stat.services["ethstats"] = map[string]string{"offline": err.Error()} - } - } else { - stat.services["ethstats"] = infos.Report() - protips.ethstats = infos.config + } else { + stat.services["nginx"] = infos.Report() + } + logger.Debug("Checking for ethstats availability") + if infos, err := checkEthstats(client, w.network); err != nil { + if err != ErrServiceUnknown { + stat.services["ethstats"] = map[string]string{"offline": err.Error()} } - logger.Debug("Checking for bootnode availability") - if infos, err := checkNode(client, w.network, true); err != nil { - if err != ErrServiceUnknown { - stat.services["bootnode"] = map[string]string{"offline": err.Error()} - } - } else { - stat.services["bootnode"] = infos.Report() - - protips.genesis = string(infos.genesis) - protips.bootFull = append(protips.bootFull, infos.enodeFull) - if infos.enodeLight != "" { - protips.bootLight = append(protips.bootLight, infos.enodeLight) - } + } else { + stat.services["ethstats"] = infos.Report() + ethstats = infos.config + } + logger.Debug("Checking for bootnode availability") + if infos, err := checkNode(client, w.network, true); err != nil { + if err != ErrServiceUnknown { + stat.services["bootnode"] = map[string]string{"offline": err.Error()} } - logger.Debug("Checking for sealnode availability") - if infos, err := checkNode(client, w.network, false); err != nil { - if err != ErrServiceUnknown { - stat.services["sealnode"] = map[string]string{"offline": err.Error()} - } - } else { - stat.services["sealnode"] = infos.Report() - protips.genesis = string(infos.genesis) + } else { + stat.services["bootnode"] = infos.Report() + + genesis = string(infos.genesis) + bootFull = append(bootFull, infos.enodeFull) + if infos.enodeLight != "" { + bootLight = append(bootLight, infos.enodeLight) } - logger.Debug("Checking for faucet availability") - if infos, err := checkFaucet(client, w.network); err != nil { - if err != ErrServiceUnknown { - stat.services["faucet"] = map[string]string{"offline": err.Error()} - } - } else { - stat.services["faucet"] = infos.Report() + } + logger.Debug("Checking for sealnode availability") + if infos, err := checkNode(client, w.network, false); err != nil { + if err != ErrServiceUnknown { + stat.services["sealnode"] = map[string]string{"offline": err.Error()} } - logger.Debug("Checking for dashboard availability") - if infos, err := checkDashboard(client, w.network); err != nil { - if err != ErrServiceUnknown { - stat.services["dashboard"] = map[string]string{"offline": err.Error()} - } - } else { - stat.services["dashboard"] = infos.Report() + } else { + stat.services["sealnode"] = infos.Report() + genesis = string(infos.genesis) + } + logger.Debug("Checking for faucet availability") + if infos, err := checkFaucet(client, w.network); err != nil { + if err != ErrServiceUnknown { + stat.services["faucet"] = map[string]string{"offline": err.Error()} } - // All status checks complete, report and check next server - delete(w.services, server) - for service := range stat.services { - w.services[server] = append(w.services[server], service) + } else { + stat.services["faucet"] = infos.Report() + } + logger.Debug("Checking for dashboard availability") + if infos, err := checkDashboard(client, w.network); err != nil { + if err != ErrServiceUnknown { + stat.services["dashboard"] = map[string]string{"offline": err.Error()} } + } else { + stat.services["dashboard"] = infos.Report() } - // If a genesis block was found, load it into our configs - if protips.genesis != "" && w.conf.genesis == nil { - genesis := new(core.Genesis) - if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil { + // Feed and newly discovered information into the wizard + w.lock.Lock() + defer w.lock.Unlock() + + if genesis != "" && w.conf.genesis == nil { + g := new(core.Genesis) + if err := json.Unmarshal([]byte(genesis), g); err != nil { log.Error("Failed to parse remote genesis", "err", err) } else { - w.conf.genesis = genesis - protips.network = genesis.Config.ChainId.Int64() + w.conf.genesis = g } } - if protips.ethstats != "" { - w.conf.ethstats = protips.ethstats + if ethstats != "" { + w.conf.ethstats = ethstats } - w.conf.bootFull = protips.bootFull - w.conf.bootLight = protips.bootLight + w.conf.bootFull = append(w.conf.bootFull, bootFull...) + w.conf.bootLight = append(w.conf.bootLight, bootLight...) - // Print any collected stats and return - stats.render() + return stat } // serverStat is a collection of service configuration parameters and health @@ -205,6 +237,9 @@ func (stats serverStats) render() { } sort.Strings(services) + if len(services) == 0 { + table.Append([]string{server, stats[server].address, "", "", ""}) + } for j, service := range services { // Add an empty line between all services if j > 0 { -- cgit v1.2.3