From 18f702faf7bcb9c7ca95a082b78aece2a4ed664f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 6 Feb 2019 14:34:08 +0200 Subject: cmd/puppeth: handle pre-set Petersburg number, save changed fork rules --- cmd/puppeth/wizard_genesis.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'cmd') diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 8abfe7c41..6aed09f14 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -222,14 +222,18 @@ func (w *wizard) manageGenesis() { fmt.Println() fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) - + if w.conf.Genesis.Config.PetersburgBlock == nil { + w.conf.Genesis.Config.PetersburgBlock = w.conf.Genesis.Config.ConstantinopleBlock + } fmt.Println() - fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) - w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.PetersburgBlock) + w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.PetersburgBlock) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) + w.conf.flush() + case "2": // Save whatever genesis configuration we currently have fmt.Println() -- cgit v1.2.3 From 2072c26a96badbe45d6df56a4cd68ffd1b6fb12e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 16 Nov 2018 17:58:24 +0200 Subject: cmd, core, params: add support for Goerli (cherry picked from commit b0ed083ead2d58cc25754eacdb48046eb2bc81cb) --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 23 +++++++++++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) (limited to 'cmd') diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ebaeba9f4..c62fd87f2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -121,6 +121,7 @@ var ( utils.DeveloperPeriodFlag, utils.TestnetFlag, utils.RinkebyFlag, + utils.GoerliFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.ConstantinopleOverrideFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 25a702dd7..ed7e98bba 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -74,6 +74,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.NetworkIdFlag, utils.TestnetFlag, utils.RinkebyFlag, + utils.GoerliFlag, utils.SyncModeFlag, utils.GCModeFlag, utils.EthStatsURLFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 60e45d095..78d2b44d6 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -140,6 +140,10 @@ var ( Name: "rinkeby", Usage: "Rinkeby network: pre-configured proof-of-authority test network", } + GoerliFlag = cli.BoolFlag{ + Name: "goerli", + Usage: "Görli network: pre-configured proof-of-authority test network", + } ConstantinopleOverrideFlag = cli.Uint64Flag{ Name: "override.constantinople", Usage: "Manually specify constantinople fork-block, overriding the bundled setting", @@ -647,6 +651,9 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(RinkebyFlag.Name) { return filepath.Join(path, "rinkeby") } + if ctx.GlobalBool(GoerliFlag.Name) { + return filepath.Join(path, "goerli") + } return path } Fatalf("Cannot determine default data directory, please set manually (--datadir)") @@ -701,6 +708,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.TestnetBootnodes case ctx.GlobalBool(RinkebyFlag.Name): urls = params.RinkebyBootnodes + case ctx.GlobalBool(GoerliFlag.Name): + urls = params.GoerliBootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -728,6 +737,8 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { } case ctx.GlobalBool(RinkebyFlag.Name): urls = params.RinkebyBootnodes + case ctx.GlobalBool(GoerliFlag.Name): + urls = params.GoerliBootnodes case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } @@ -980,7 +991,6 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setHTTP(ctx, cfg) setWS(ctx, cfg) setNodeUserIdent(ctx, cfg) - setDataDir(ctx, cfg) if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { @@ -1004,6 +1014,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet") case ctx.GlobalBool(RinkebyFlag.Name): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") + case ctx.GlobalBool(GoerliFlag.Name): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") } } @@ -1160,7 +1172,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags - checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag) + checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag) checkExclusive(ctx, LightServFlag, SyncModeFlag, "light") ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -1256,6 +1268,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.NetworkId = 4 } cfg.Genesis = core.DefaultRinkebyGenesisBlock() + case ctx.GlobalBool(GoerliFlag.Name): + if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + cfg.NetworkId = 5 + } + cfg.Genesis = core.DefaultGoerliGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1396,6 +1413,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultTestnetGenesisBlock() case ctx.GlobalBool(RinkebyFlag.Name): genesis = core.DefaultRinkebyGenesisBlock() + case ctx.GlobalBool(GoerliFlag.Name): + genesis = core.DefaultGoerliGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } -- cgit v1.2.3 From 048b463b301b1ac61be187d7195665d4ad31f51f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 12 Feb 2019 11:29:05 +0100 Subject: common/fdlimit: cap on MacOS file limits, fixes #18994 (#19035) * common/fdlimit: cap on MacOS file limits, fixes #18994 * common/fdlimit: fix Maximum-check to respect OPEN_MAX * common/fdlimit: return error if OPEN_MAX is exceeded in Raise() * common/fdlimit: goimports * common/fdlimit: check value after setting fdlimit * common/fdlimit: make comment a bit more descriptive * cmd/utils: make fdlimit happy path a bit cleaner (cherry picked from commit f48da43bae183a04a23d298cb1790d2f8d2cec51) --- cmd/utils/flags.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cmd') diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 78d2b44d6..a65a36204 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -847,10 +847,11 @@ func makeDatabaseHandles() int { if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) } - if err := fdlimit.Raise(uint64(limit)); err != nil { + raised, err := fdlimit.Raise(uint64(limit)) + if err != nil { Fatalf("Failed to raise file descriptor allowance: %v", err) } - return limit / 2 // Leave half for networking and other stuff + return int(raised / 2) // Leave half for networking and other stuff } // MakeAddress converts an account specified directly as a hex encoded string or -- cgit v1.2.3 From a3f31f51f38040e0aef76f4da95ae462fb649f81 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 16 Jan 2019 19:03:02 +0530 Subject: cmd/swarm/swarm-snapshot: swarm snapshot generator (#18453) * cmd/swarm/swarm-snapshot: add binary to create network snapshots * cmd/swarm/swarm-snapshot: refactor and extend tests * p2p/simulations: remove unused triggerChecks func and fix linter * internal/cmdtest: raise the timeout for killing TestCmd * cmd/swarm/swarm-snapshot: add more comments and other minor adjustments * cmd/swarm/swarm-snapshot: remove redundant check in createSnapshot * cmd/swarm/swarm-snapshot: change comment wording * p2p/simulations: revert Simulation.Run from master https://github.com/ethersphere/go-ethereum/pull/1077/files#r247078904 * cmd/swarm/swarm-snapshot: address pr comments * swarm/network/simulations/discovery: removed snapshot write to file * cmd/swarm/swarm-snapshot, swarm/network/simulations: removed redundant connection event check, fixed lint error (cherry picked from commit 34f11e752f61b81c13cdde0649a3c7b14f801c69) --- cmd/swarm/swarm-snapshot/create.go | 157 ++++++++++++++++++++++++++++++++ cmd/swarm/swarm-snapshot/create_test.go | 138 ++++++++++++++++++++++++++++ cmd/swarm/swarm-snapshot/main.go | 82 +++++++++++++++++ cmd/swarm/swarm-snapshot/run_test.go | 49 ++++++++++ 4 files changed, 426 insertions(+) create mode 100644 cmd/swarm/swarm-snapshot/create.go create mode 100644 cmd/swarm/swarm-snapshot/create_test.go create mode 100644 cmd/swarm/swarm-snapshot/main.go create mode 100644 cmd/swarm/swarm-snapshot/run_test.go (limited to 'cmd') diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go new file mode 100644 index 000000000..127fde8ae --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create.go @@ -0,0 +1,157 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/simulation" + cli "gopkg.in/urfave/cli.v1" +) + +// create is used as the entry function for "create" app command. +func create(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + if len(ctx.Args()) < 1 { + return errors.New("argument should be the filename to verify or write-to") + } + filename, err := touchPath(ctx.Args()[0]) + if err != nil { + return err + } + return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ",")) +} + +// createSnapshot creates a new snapshot on filesystem with provided filename, +// number of nodes and service names. +func createSnapshot(filename string, nodes int, services []string) (err error) { + log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services) + + sim := simulation.New(map[string]simulation.ServiceFunc{ + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + addr := network.NewAddr(ctx.Config.Node()) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + hp := network.NewHiveParams() + hp.KeepAliveInterval = time.Duration(200) * time.Millisecond + hp.Discovery = true // discovery must be enabled when creating a snapshot + + config := &network.BzzConfig{ + OverlayAddr: addr.Over(), + UnderlayAddr: addr.Under(), + HiveParams: hp, + } + return network.NewBzz(config, kad, nil, nil, nil), nil, nil + }, + }) + defer sim.Close() + + _, err = sim.AddNodes(nodes) + if err != nil { + return fmt.Errorf("add nodes: %v", err) + } + + err = sim.Net.ConnectNodesRing(nil) + if err != nil { + return fmt.Errorf("connect nodes: %v", err) + } + + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelSimRun() + if _, err := sim.WaitTillHealthy(ctx); err != nil { + return fmt.Errorf("wait for healthy kademlia: %v", err) + } + + var snap *simulations.Snapshot + if len(services) > 0 { + // If service names are provided, include them in the snapshot. + // But, check if "bzz" service is not among them to remove it + // form the snapshot as it exists on snapshot creation. + var removeServices []string + var wantBzz bool + for _, s := range services { + if s == "bzz" { + wantBzz = true + break + } + } + if !wantBzz { + removeServices = []string{"bzz"} + } + snap, err = sim.Net.SnapshotWithServices(services, removeServices) + } else { + snap, err = sim.Net.Snapshot() + } + if err != nil { + return fmt.Errorf("create snapshot: %v", err) + } + jsonsnapshot, err := json.Marshal(snap) + if err != nil { + return fmt.Errorf("json encode snapshot: %v", err) + } + return ioutil.WriteFile(filename, jsonsnapshot, 0666) +} + +// touchPath creates an empty file and all subdirectories +// that are missing. +func touchPath(filename string) (string, error) { + if path.IsAbs(filename) { + if _, err := os.Stat(filename); err == nil { + // path exists, overwrite + return filename, nil + } + } + + d, f := path.Split(filename) + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return "", err + } + + _, err = os.Stat(path.Join(dir, filename)) + if err == nil { + // path exists, overwrite + return filename, nil + } + + dirPath := path.Join(dir, d) + filePath := path.Join(dirPath, f) + if d != "" { + err = os.MkdirAll(dirPath, os.ModeDir) + if err != nil { + return "", err + } + } + + return filePath, nil +} diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go new file mode 100644 index 000000000..dbd5b12cd --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -0,0 +1,138 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "sort" + "strconv" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/p2p/simulations" +) + +// TestSnapshotCreate is a high level e2e test that tests for snapshot generation. +// It runs a few "create" commands with different flag values and loads generated +// snapshot files to validate their content. +func TestSnapshotCreate(t *testing.T) { + for _, v := range []struct { + name string + nodes int + services string + }{ + { + name: "defaults", + }, + { + name: "more nodes", + nodes: defaultNodes + 5, + }, + { + name: "services", + services: "stream,pss,zorglub", + }, + { + name: "services with bzz", + services: "bzz,pss", + }, + } { + t.Run(v.name, func(t *testing.T) { + t.Parallel() + + file, err := ioutil.TempFile("", "swarm-snapshot") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + if err = file.Close(); err != nil { + t.Error(err) + } + + args := []string{"create"} + if v.nodes > 0 { + args = append(args, "--nodes", strconv.Itoa(v.nodes)) + } + if v.services != "" { + args = append(args, "--services", v.services) + } + testCmd := runSnapshot(t, append(args, file.Name())...) + + testCmd.ExpectExit() + if code := testCmd.ExitStatus(); code != 0 { + t.Fatalf("command exit code %v, expected 0", code) + } + + f, err := os.Open(file.Name()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := f.Close() + if err != nil { + t.Error("closing snapshot file", "err", err) + } + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(b, &snap) + if err != nil { + t.Fatal(err) + } + + wantNodes := v.nodes + if wantNodes == 0 { + wantNodes = defaultNodes + } + gotNodes := len(snap.Nodes) + if gotNodes != wantNodes { + t.Errorf("got %v nodes, want %v", gotNodes, wantNodes) + } + + if len(snap.Conns) == 0 { + t.Error("no connections in a snapshot") + } + + var wantServices []string + if v.services != "" { + wantServices = strings.Split(v.services, ",") + } else { + wantServices = []string{"bzz"} + } + // sort service names so they can be comparable + // as strings to every node sorted services + sort.Strings(wantServices) + + for i, n := range snap.Nodes { + gotServices := n.Node.Config.Services + sort.Strings(gotServices) + if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) { + t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices) + } + } + + }) + } +} diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go new file mode 100644 index 000000000..184727e4d --- /dev/null +++ b/cmd/swarm/swarm-snapshot/main.go @@ -0,0 +1,82 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +// default value for "create" command --nodes flag +const defaultNodes = 10 + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Snapshot Utility. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Snapshot Utility") + + app.Name = "swarm-snapshot" + app.Usage = "" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "create", + Aliases: []string{"c"}, + Usage: "create a swarm snapshot", + Action: create, + // Flags only for "create" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.IntFlag{ + Name: "nodes", + Value: defaultNodes, + Usage: "number of nodes", + }, + cli.StringFlag{ + Name: "services", + Value: "bzz", + Usage: "comma separated list of services to boot the nodes with", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/swarm-snapshot/run_test.go b/cmd/swarm/swarm-snapshot/run_test.go new file mode 100644 index 000000000..d9a041597 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-snapshot", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-snapshot", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} -- cgit v1.2.3 From 7383db4dace3679226215f1eb2b36c78ffb2b264 Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 17 Jan 2019 11:25:27 -0500 Subject: Upload speed (#18442) (cherry picked from commit 257bfff316e4efb8952fbeb67c91f86af579cb0a) --- cmd/swarm/swarm-smoke/main.go | 6 +++ cmd/swarm/swarm-smoke/upload_speed.go | 96 +++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 cmd/swarm/swarm-smoke/upload_speed.go (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 66cecdc5c..f7f358ef1 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -148,6 +148,12 @@ func main() { Usage: "feed update generate, upload and sync", Action: cliFeedUploadAndSync, }, + { + Name: "upload_speed", + Aliases: []string{"u"}, + Usage: "measure upload speed", + Action: cliUploadSpeed, + }, } sort.Sort(cli.FlagsByName(app.Flags)) diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go new file mode 100644 index 000000000..d55b5fe8e --- /dev/null +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -0,0 +1,96 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "fmt" + "os" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" + + cli "gopkg.in/urfave/cli.v1" +) + +var endpoint string + +//just use the first endpoint +func generateEndpoint(scheme string, cluster string, app string, from int) { + if cluster == "prod" { + endpoint = fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from) + } else { + endpoint = fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster) + } +} + +func cliUploadSpeed(c *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + metrics.GetOrRegisterCounter("upload-speed", nil).Inc(1) + + errc := make(chan error) + go func() { + errc <- uploadSpeed(c) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter("upload-speed.fail", nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter("upload-speed.timeout", nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uploadSpeed(c *cli.Context) error { + defer func(now time.Time) { + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime, "kb", filesize) + metrics.GetOrRegisterCounter("upload-speed.total-time", nil).Inc(int64(totalTime)) + }(time.Now()) + + generateEndpoint(scheme, cluster, appName, from) + seed := int(time.Now().UnixNano() / 1e6) + log.Info("uploading to "+endpoint, "seed", seed) + + randomBytes := testutil.RandomBytes(seed, filesize*1000) + + t1 := time.Now() + hash, err := upload(&randomBytes, endpoint) + if err != nil { + log.Error(err.Error()) + return err + } + metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1))) + + fhash, err := digest(bytes.NewReader(randomBytes)) + if err != nil { + log.Error(err.Error()) + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) + return nil +} -- cgit v1.2.3 From 21d54bcaac6c43b92e9bf86610d0952de454f2df Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 18 Jan 2019 19:22:05 +0700 Subject: cmd/swarm/swarm-snapshot: disable tests on windows (#18478) (cherry picked from commit 632135ce4c1d8d3d9a36771aab4137260018e84b) --- cmd/swarm/swarm-snapshot/create_test.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'cmd') diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go index dbd5b12cd..c9445168d 100644 --- a/cmd/swarm/swarm-snapshot/create_test.go +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "runtime" "sort" "strconv" "strings" @@ -33,6 +34,10 @@ import ( // It runs a few "create" commands with different flag values and loads generated // snapshot files to validate their content. func TestSnapshotCreate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + for _, v := range []struct { name string nodes int -- cgit v1.2.3 From 4625b1257f614646cdac49fcc0a76f2a1d58fb5c Mon Sep 17 00:00:00 2001 From: holisticode Date: Fri, 18 Jan 2019 12:14:06 -0500 Subject: cmd/swarm/swarm-smoke: use ResettingTimer instead of Counters for times (#18479) (cherry picked from commit 560957799a089042e471320d179ef2e96caf4f8d) --- cmd/swarm/swarm-smoke/upload_and_sync.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index d605f79a3..6b26b152d 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -86,9 +86,8 @@ func cliUploadAndSync(c *cli.Context) error { func uploadAndSync(c *cli.Context) error { defer func(now time.Time) { totalTime := time.Since(now) - log.Info("total time", "time", totalTime, "kb", filesize) - metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.total-time", nil).Update(totalTime) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) @@ -103,7 +102,7 @@ func uploadAndSync(c *cli.Context) error { log.Error(err.Error()) return err } - metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1))) + metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1) fhash, err := digest(bytes.NewReader(randomBytes)) if err != nil { @@ -166,8 +165,6 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") defer sp.Finish() - log.Trace("sleeping", "ruid", ruid) - time.Sleep(3 * time.Second) log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) var tn time.Time -- cgit v1.2.3 From 878aa58ec66ffc541a80aa65ee35261efa086c9c Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 18 Jan 2019 13:27:27 +0100 Subject: cmd/swarm: use resetting timer to measure fetch time (#18474) (cherry picked from commit a0b0db63055e1dd350215f9fe04b0abf19f3488a) --- cmd/swarm/swarm-smoke/upload_and_sync.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 6b26b152d..7babc8004 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -124,30 +124,28 @@ func uploadAndSync(c *cli.Context) error { for { start := time.Now() err := fetch(hash, endpoint, fhash, ruid) - fetchTime := time.Since(start) if err != nil { continue } - metrics.GetOrRegisterMeter("upload-and-sync.single.fetch-time", nil).Mark(int64(fetchTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).UpdateSince(start) wg.Done() return } }(endpoints[randIndex], ruid) } else { - for _, endpoint := range endpoints { + for _, endpoint := range endpoints[1:] { ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { start := time.Now() err := fetch(hash, endpoint, fhash, ruid) - fetchTime := time.Since(start) if err != nil { continue } - metrics.GetOrRegisterMeter("upload-and-sync.each.fetch-time", nil).Mark(int64(fetchTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).UpdateSince(start) wg.Done() return } -- cgit v1.2.3 From 4976fcc91a43b5c7047c51a03985887b694f0fbb Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 24 Jan 2019 12:02:18 +0100 Subject: swarm: bootnode-mode, new bootnodes and no p2p package discovery (#18498) (cherry picked from commit bbd120354a8d226b446591eeda9f9462cb9b690a) --- cmd/swarm/bootnodes.go | 61 +++----------------------------- cmd/swarm/config.go | 23 ++++++++---- cmd/swarm/flags.go | 4 +++ cmd/swarm/main.go | 25 +++++++++++-- cmd/swarm/run_test.go | 2 -- cmd/swarm/swarm-smoke/upload_and_sync.go | 4 +++ cmd/swarm/swarm-smoke/upload_speed.go | 2 ++ 7 files changed, 53 insertions(+), 68 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/bootnodes.go b/cmd/swarm/bootnodes.go index cbba9970d..ce3cd5288 100644 --- a/cmd/swarm/bootnodes.go +++ b/cmd/swarm/bootnodes.go @@ -17,61 +17,8 @@ package main var SwarmBootnodes = []string{ - // Foundation Swarm Gateway Cluster - "enode://e5c6f9215c919a5450a7b8c14c22535607b69f2c8e1e7f6f430cb25d7a2c27cd1df4c4f18ad7c1d7e5162e271ffcd3f20b1a1467fb6e790e7d727f3b2193de97@52.232.7.187:30399", - "enode://9b2fe07e69ccc7db5fef15793dab7d7d2e697ed92132d6e9548218e68a34613a8671ad03a6658d862b468ed693cae8a0f8f8d37274e4a657ffb59ca84676e45b@52.232.7.187:30400", - "enode://76c1059162c93ef9df0f01097c824d17c492634df211ef4c806935b349082233b63b90c23970254b3b7138d630400f7cf9b71e80355a446a8b733296cb04169a@52.232.7.187:30401", - "enode://ce46bbe2a8263145d65252d52da06e000ad350ed09c876a71ea9544efa42f63c1e1b6cc56307373aaad8f9dd069c90d0ed2dd1530106200e16f4ca681dd8ae2d@52.232.7.187:30402", - "enode://f431e0d6008a6c35c6e670373d828390c8323e53da8158e7bfc43cf07e632cc9e472188be8df01decadea2d4a068f1428caba769b632554a8fb0607bc296988f@52.232.7.187:30403", - "enode://174720abfff83d7392f121108ae50ea54e04889afe020df883655c0f6cb95414db945a0228d8982fe000d86fc9f4b7669161adc89cd7cd56f78f01489ab2b99b@52.232.7.187:30404", - "enode://2ae89be4be61a689b6f9ecee4360a59e185e010ab750f14b63b4ae43d4180e872e18e3437d4386ce44875dc7cc6eb761acba06412fe3178f3dac1dab3b65703e@52.232.7.187:30405", - "enode://24abebe1c0e6d75d6052ce3219a87be8573fd6397b4cb51f0773b83abba9b3d872bfb273cdc07389715b87adfac02f5235f5241442c5089802cbd8d42e310fce@52.232.7.187:30406", - "enode://d08dfa46bfbbdbcaafbb6e34abee4786610f6c91e0b76d7881f0334ac10dda41d8c1f2b6eedffb4493293c335c0ad46776443b2208d1fbbb9e1a90b25ee4eef2@52.232.7.187:30407", - "enode://8d95eb0f837d27581a43668ed3b8783d69dc4e84aa3edd7a0897e026155c8f59c8702fdc0375ee7bac15757c9c78e1315d9b73e4ce59c936db52ea4ae2f501c7@52.232.7.187:30408", - "enode://a5967cc804aebd422baaaba9f06f27c9e695ccab335b61088130f8cbe64e3cdf78793868c7051dfc06eecfe844fad54bc7f6dfaed9db3c7ecef279cb829c25fb@52.232.7.187:30409", - "enode://5f00134d81a8f2ebcc46f8766f627f492893eda48138f811b7de2168308171968f01710bca6da05764e74f14bae41652f554e6321f1aed85fa3461e89d075dbf@52.232.7.187:30410", - "enode://b2142b79b01a5aa66a5e23cc35e78219a8e97bc2412a6698cee24ae02e87078b725d71730711bd62e25ff1aa8658c6633778af8ac14c63814a337c3dd0ebda9f@52.232.7.187:30411", - "enode://1ffa7651094867d6486ce3ef46d27a052c2cb968b618346c6df7040322c7efc3337547ba85d4cbba32e8b31c42c867202554735c06d4c664b9afada2ed0c4b3c@52.232.7.187:30412", - "enode://129e0c3d5f5df12273754f6f703d2424409fa4baa599e0b758c55600169313887855e75b082028d2302ec034b303898cd697cc7ae8256ba924ce927510da2c8d@52.232.7.187:30413", - "enode://419e2dc0d2f5b022cf16b0e28842658284909fa027a0fbbb5e2b755e7f846ea02a8f0b66a7534981edf6a7bcf8a14855344c6668e2cd4476ccd35a11537c9144@52.232.7.187:30414", - "enode://23d55ad900583231b91f2f62e3f72eb498b342afd58b682be3af052eed62b5651094471065981de33d8786f075f05e3cca499503b0ac8ae84b2a06e99f5b0723@52.232.7.187:30415", - "enode://bc56e4158c00e9f616d7ea533def20a89bef959df4e62a768ff238ff4e1e9223f57ecff969941c20921bad98749baae311c0fbebce53bf7bbb9d3dc903640990@52.232.7.187:30416", - "enode://433ce15199c409875e7e72fffd69fdafe746f17b20f0d5555281722a65fde6c80328fab600d37d8624509adc072c445ce0dad4a1c01cff6acf3132c11d429d4d@52.232.7.187:30417", - "enode://632ee95b8f0eac51ef89ceb29313fef3a60050181d66a6b125583b1a225a7694b252edc016efb58aa3b251da756cb73280842a022c658ed405223b2f58626343@52.232.7.187:30418", - "enode://4a0f9bcff7a4b9ee453fb298d0fb222592efe121512e30cd72fef631beb8c6a15153a1456eb073ee18551c0e003c569651a101892dc4124e90b933733a498bb5@52.232.7.187:30419", - "enode://f0d80fbc72d16df30e19aac3051eb56a7aff0c8367686702e01ea132d8b0b3ee00cadd6a859d2cca98ec68d3d574f8a8a87dba2347ec1e2818dc84bc3fa34fae@52.232.7.187:30420", - "enode://a199146906e4f9f2b94b195a8308d9a59a3564b92efaab898a4243fe4c2ad918b7a8e4853d9d901d94fad878270a2669d644591299c3d43de1b298c00b92b4a7@52.232.7.187:30421", - "enode://052036ea8736b37adbfb684d90ce43e11b3591b51f31489d7c726b03618dea4f73b1e659deb928e6bf40564edcdcf08351643f42db3d4ca1c2b5db95dad59e94@52.232.7.187:30422", - "enode://460e2b8c6da8f12fac96c836e7d108f4b7ec55a1c64631bb8992339e117e1c28328fee83af863196e20af1487a655d13e5ceba90e980e92502d5bac5834c1f71@52.232.7.187:30423", - "enode://6d2cdd13741b2e72e9031e1b93c6d9a4e68de2844aa4e939f6a8a8498a7c1d7e2ee4c64217e92a6df08c9a32c6764d173552810ef1bd2ecb356532d389dd2136@52.232.7.187:30424", - "enode://62105fc25ce2cd5b299647f47eaa9211502dc76f0e9f461df915782df7242ac3223e3db04356ae6ed2977ccac20f0b16864406e9ca514a40a004cb6a5d0402aa@52.232.7.187:30425", - "enode://e0e388fc520fd493c33f0ce16685e6f98fb6aec28f2edc14ee6b179594ee519a896425b0025bb6f0e182dd3e468443f19c70885fbc66560d000093a668a86aa8@52.232.7.187:30426", - "enode://63f3353a72521ea10022127a4fe6b4acbef197c3fe668fd9f4805542d8a6fcf79f6335fbab62d180a35e19b739483e740858b113fdd7c13a26ad7b4e318a5aef@52.232.7.187:30427", - "enode://33a42b927085678d4aefd4e70b861cfca6ef5f6c143696c4f755973fd29e64c9e658cad57a66a687a7a156da1e3688b1fbdd17bececff2ee009fff038fa5666b@52.232.7.187:30428", - "enode://259ab5ab5c1daee3eab7e3819ab3177b82d25c29e6c2444fdd3f956e356afae79a72840ccf2d0665fe82c81ebc3b3734da1178ac9fd5d62c67e674b69f86b6be@52.232.7.187:30429", - "enode://558bccad7445ce3fd8db116ed6ab4aed1324fdbdac2348417340c1764dc46d46bffe0728e5b7d5c36f12e794c289f18f57f08f085d2c65c9910a5c7a65b6a66a@52.232.7.187:30430", - "enode://abe60937a0657ffded718e3f84a32987286983be257bdd6004775c4b525747c2b598f4fac49c8de324de5ce75b22673fa541a7ce2d555fb7f8ca325744ae3577@52.232.7.187:30431", - "enode://bce6f0aaa5b230742680084df71d4f026b3eff7f564265599216a1b06b765303fdc9325de30ffd5dfdaf302ce4b14322891d2faea50ce2ca298d7409f5858339@52.232.7.187:30432", - "enode://21b957c4e03277d42be6660730ec1b93f540764f26c6abdb54d006611139c7081248486206dfbf64fcaffd62589e9c6b8ea77a5297e4b21a605f1bcf49483ed0@52.232.7.187:30433", - "enode://ff104e30e64f24c3d7328acee8b13354e5551bc8d60bb25ecbd9632d955c7e34bb2d969482d173355baad91c8282f8b592624eb3929151090da3b4448d4d58fb@52.232.7.187:30434", - "enode://c76e2b5f81a521bceaec1518926a21380a345df9cf463461562c6845795512497fb67679e155fc96a74350f8b78de8f4c135dd52b106dbbb9795452021d09ea5@52.232.7.187:30435", - "enode://3288fd860105164f3e9b69934c4eb18f7146cfab31b5a671f994e21a36e9287766e5f9f075aefbc404538c77f7c2eb2a4495020a7633a1c3970d94e9fa770aeb@52.232.7.187:30436", - "enode://6cea859c7396d46b20cfcaa80f9a11cd112f8684f2f782f7b4c0e1e0af9212113429522075101923b9b957603e6c32095a6a07b5e5e35183c521952ee108dfaf@52.232.7.187:30437", - "enode://f628ec56e4ca8317cc24cc4ac9b27b95edcce7b96e1c7f3b53e30de4a8580fe44f2f0694a513bdb0a431acaf2824074d6ace4690247bbc34c14f426af8c056ea@52.232.7.187:30438", - "enode://055ec8b26fc105c4f97970a1cce9773a5e34c03f511b839db742198a1c571e292c54aa799e9afb991cc8a560529b8cdf3e0c344bc6c282aff2f68eec59361ddf@52.232.7.187:30439", - "enode://48cb0d430c328974226aa33a931d8446cd5a8d40f3ead8f4ce7ad60faa1278192eb6d58bed91258d63e81f255fc107eec2425ce2ae8b22350dd556076e160610@52.232.7.187:30440", - "enode://3fadb7af7f770d5ffc6b073b8d42834bebb18ce1fe8a4fe270d2b799e7051327093960dc61d9a18870db288f7746a0e6ea2a013cd6ab0e5f97ca08199473aace@52.232.7.187:30441", - "enode://a5d7168024c9992769cf380ffa559a64b4f39a29d468f579559863814eb0ae0ed689ac0871a3a2b4c78b03297485ec322d578281131ef5d5c09a4beb6200a97a@52.232.7.187:30442", - "enode://9c57744c5b2c2d71abcbe80512652f9234d4ab041b768a2a886ab390fe6f184860f40e113290698652d7e20a8ac74d27ac8671db23eb475b6c5e6253e4693bf8@52.232.7.187:30443", - "enode://daca9ff0c3176045a0e0ed228dee00ec86bc0939b135dc6b1caa23745d20fd0332e1ee74ad04020e89df56c7146d831a91b89d15ca3df05ba7618769fefab376@52.232.7.187:30444", - "enode://a3f6af59428cb4b9acb198db15ef5554fa43c2b0c18e468a269722d64a27218963a2975eaf82750b6262e42192b5e3669ea51337b4cda62b33987981bc5e0c1a@52.232.7.187:30445", - "enode://fe571422fa4651c3354c85dac61911a6a6520dd3c0332967a49d4133ca30e16a8a4946fa73ca2cb5de77917ea701a905e1c3015b2f4defcd53132b61cc84127a@52.232.7.187:30446", - - // Mainframe - "enode://ee9a5a571ea6c8a59f9a8bb2c569c865e922b41c91d09b942e8c1d4dd2e1725bd2c26149da14de1f6321a2c6fdf1e07c503c3e093fb61696daebf74d6acd916b@54.186.219.160:30399", - "enode://a03f0562ecb8a992ad5242345535e73483cdc18ab934d36bf24b567d43447c2cea68f89f1d51d504dd13acc30f24ebce5a150bea2ccb1b722122ce4271dc199d@52.67.248.147:30399", - "enode://e2cbf9eafd85903d3b1c56743035284320695e0072bc8d7396e0542aa5e1c321b236f67eab66b79c2f15d4447fa4bbe74dd67d0467da23e7eb829f60ec8a812b@13.58.169.1:30399", - "enode://8b8c6bda6047f1cad9fab2db4d3d02b7aa26279902c32879f7bcd4a7d189fee77fdc36ee151ce6b84279b4792e72578fd529d2274d014132465758fbfee51cee@13.209.13.15:30399", - "enode://63f6a8818927e429585287cf2ca0cb9b11fa990b7b9b331c2962cdc6f21807a2473b26e8256225c26caff70d7218e59586d704d49061452c6852e382c885d03c@35.154.106.174:30399", - "enode://ed4bd3b794ed73f18e6dcc70c6624dfec63b5654f6ab54e8f40b16eff8afbd342d4230e099ddea40e84423f81b2d2ea79799dc345257b1fec6f6c422c9d008f7@52.213.20.99:30399", + // EF Swarm Bootnode - AWS - eu-central-1 + "enode://4c113504601930bf2000c29bcd98d1716b6167749f58bad703bae338332fe93cc9d9204f08afb44100dc7bea479205f5d162df579f9a8f76f8b402d339709023@3.122.203.99:30301", + // EF Swarm Bootnode - AWS - us-west-2 + "enode://89f2ede3371bff1ad9f2088f2012984e280287a4e2b68007c2a6ad994909c51886b4a8e9e2ecc97f9910aca538398e0a5804b0ee80a187fde1ba4f32626322ba@52.35.212.179:30301", } diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 3eea3057b..0203a6798 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -79,6 +79,7 @@ const ( SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" GETH_ENV_DATADIR = "GETH_DATADIR" @@ -164,10 +165,9 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config return config, err } -//override the current config with whatever is provided through the command line -//most values are not allowed a zero value (empty string), if not otherwise noted +// cmdLineOverride overrides the current config with whatever is provided through the command line +// most values are not allowed a zero value (empty string), if not otherwise noted func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Config { - if keyid := ctx.GlobalString(SwarmAccountFlag.Name); keyid != "" { currentConfig.BzzAccount = keyid } @@ -258,14 +258,17 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity } + if ctx.GlobalIsSet(SwarmBootnodeModeFlag.Name) { + currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) + } + return currentConfig } -//override the current config with whatver is provided in environment variables -//most values are not allowed a zero value (empty string), if not otherwise noted +// envVarsOverride overrides the current config with whatver is provided in environment variables +// most values are not allowed a zero value (empty string), if not otherwise noted func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { - if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" { currentConfig.BzzAccount = keyid } @@ -364,6 +367,14 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.Cors = cors } + if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" { + bootnodeMode, err := strconv.ParseBool(bm) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err) + } + currentConfig.BootnodeMode = bootnodeMode + } + return currentConfig } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 12edc8cc9..4c186cc31 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -156,6 +156,10 @@ var ( Name: "compressed", Usage: "Prints encryption keys in compressed form", } + SwarmBootnodeModeFlag = cli.BoolFlag{ + Name: "bootnode-mode", + Usage: "Run Swarm in Bootnode mode", + } SwarmFeedNameFlag = cli.StringFlag{ Name: "name", Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index ccbb24eec..722dc4ff5 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -154,7 +154,6 @@ func init() { utils.BootnodesFlag, utils.KeyStoreDirFlag, utils.ListenPortFlag, - utils.NoDiscoverFlag, utils.DiscoveryV5Flag, utils.NetrestrictFlag, utils.NodeKeyFileFlag, @@ -187,6 +186,8 @@ func init() { SwarmUploadDefaultPath, SwarmUpFromStdinFlag, SwarmUploadMimeType, + // bootnode mode + SwarmBootnodeModeFlag, // storage flags SwarmStorePath, SwarmStoreCapacity, @@ -227,12 +228,17 @@ func main() { func keys(ctx *cli.Context) error { privateKey := getPrivKey(ctx) - pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey)) + pubkey := crypto.FromECDSAPub(&privateKey.PublicKey) + pubkeyhex := hex.EncodeToString(pubkey) pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + bzzkey := crypto.Keccak256Hash(pubkey).Hex() + if !ctx.Bool(SwarmCompressedFlag.Name) { - fmt.Println(fmt.Sprintf("publicKey=%s", pub)) + fmt.Println(fmt.Sprintf("bzzkey=%s", bzzkey[2:])) + fmt.Println(fmt.Sprintf("publicKey=%s", pubkeyhex)) } fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed)) + return nil } @@ -272,6 +278,10 @@ func bzzd(ctx *cli.Context) error { setSwarmBootstrapNodes(ctx, &cfg) //setup the ethereum node utils.SetNodeConfig(ctx, &cfg) + + //always disable discovery from p2p package - swarm discovery is done with the `hive` protocol + cfg.P2P.NoDiscovery = true + stack, err := node.New(&cfg) if err != nil { utils.Fatalf("can't create node: %v", err) @@ -294,6 +304,15 @@ func bzzd(ctx *cli.Context) error { stack.Stop() }() + // add swarm bootnodes, because swarm doesn't use p2p package's discovery discv5 + go func() { + s := stack.Server() + + for _, n := range cfg.P2P.BootstrapNodes { + s.AddPeer(n) + } + }() + stack.Wait() return nil } diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 680d238d0..4a6a56d9b 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -254,7 +254,6 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { node.Cmd = runSwarm(t, "--port", p2pPort, "--nat", "extip:127.0.0.1", - "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, "--ens-api", "", @@ -330,7 +329,6 @@ func newTestNode(t *testing.T, dir string) *testNode { node.Cmd = runSwarm(t, "--port", p2pPort, "--nat", "extip:127.0.0.1", - "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, "--ens-api", "", diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 7babc8004..0fc86c55d 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -49,6 +49,10 @@ func generateEndpoints(scheme string, cluster string, app string, from int, to i for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } + } else if cluster == "private-internal" { + for port := from; port < to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port)) + } } else { for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go index d55b5fe8e..4a355baf8 100644 --- a/cmd/swarm/swarm-smoke/upload_speed.go +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -35,6 +35,8 @@ var endpoint string func generateEndpoint(scheme string, cluster string, app string, from int) { if cluster == "prod" { endpoint = fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from) + } else if cluster == "private-internal" { + endpoint = fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from) } else { endpoint = fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster) } -- cgit v1.2.3 From e247dcc141a5d1c9129b987a5f164e9df29be951 Mon Sep 17 00:00:00 2001 From: gluk256 Date: Thu, 24 Jan 2019 15:35:10 +0400 Subject: swarm/version: commit version added (#18510) (cherry picked from commit ad13d2d407d2f614c39af92430fda0a926da2a8a) --- cmd/swarm/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 722dc4ff5..53888b615 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -66,9 +66,10 @@ OPTIONS: {{end}}{{end}} ` -var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) -) +// Git SHA1 commit hash of the release (set via linker flags) +// this variable will be assigned if corresponding parameter is passed with install, but not with test +// e.g.: go install -ldflags "-X main.gitCommit=ed1312d01b19e04ef578946226e5d8069d5dfd5a" ./cmd/swarm +var gitCommit string //declare a few constant error messages, useful for later error check comparisons in test var ( @@ -89,6 +90,7 @@ var defaultNodeConfig = node.DefaultConfig // This init function sets defaults so cmd/swarm can run alongside geth. func init() { + sv.GitCommit = gitCommit defaultNodeConfig.Name = clientIdentifier defaultNodeConfig.Version = sv.VersionWithCommit(gitCommit) defaultNodeConfig.P2P.ListenAddr = ":30399" -- cgit v1.2.3 From 1ecf2860cff6e4da75abbd1e6b30a9b1d69c5a12 Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 7 Feb 2019 07:51:24 -0500 Subject: cmd/swarm: hashes command (#19008) (cherry picked from commit 7f55b0cbd8618a1b0de8d7e37d2b0143ebae4abf) --- cmd/swarm/explore.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/swarm/main.go | 2 ++ 2 files changed, 61 insertions(+) create mode 100644 cmd/swarm/explore.go (limited to 'cmd') diff --git a/cmd/swarm/explore.go b/cmd/swarm/explore.go new file mode 100644 index 000000000..5b5b8bf41 --- /dev/null +++ b/cmd/swarm/explore.go @@ -0,0 +1,59 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Command bzzhash computes a swarm tree hash. +package main + +import ( + "context" + "fmt" + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/swarm/storage" + "gopkg.in/urfave/cli.v1" +) + +var hashesCommand = cli.Command{ + Action: hashes, + CustomHelpTemplate: helpTemplate, + Name: "hashes", + Usage: "print all hashes of a file to STDOUT", + ArgsUsage: "", + Description: "Prints all hashes of a file to STDOUT", +} + +func hashes(ctx *cli.Context) { + args := ctx.Args() + if len(args) < 1 { + utils.Fatalf("Usage: swarm hashes ") + } + f, err := os.Open(args[0]) + if err != nil { + utils.Fatalf("Error opening file " + args[1]) + } + defer f.Close() + + fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) + refs, err := fileStore.GetAllReferences(context.TODO(), f, false) + if err != nil { + utils.Fatalf("%v\n", err) + } else { + for _, r := range refs { + fmt.Println(r.String()) + } + } +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 53888b615..8af5d8c32 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -142,6 +142,8 @@ func init() { dbCommand, // See config.go DumpConfigCommand, + // hashesCommand + hashesCommand, } // append a hidden help subcommand to all commands that have subcommands -- cgit v1.2.3 From 7038b5734c9068fb67851b9a21d771ae93aef1fd Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 30 Jan 2019 15:46:44 +0700 Subject: cmd/swarm/swarm-smoke: sliding window test (#18967) (cherry picked from commit b91bf08876ca4da0c2a843a9ed3e88d64427cfb8) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 98 +--------- cmd/swarm/swarm-smoke/main.go | 12 +- cmd/swarm/swarm-smoke/sliding_window.go | 122 ++++++++++++ cmd/swarm/swarm-smoke/upload_and_sync.go | 160 +-------------- cmd/swarm/swarm-smoke/upload_speed.go | 55 +----- cmd/swarm/swarm-smoke/util.go | 267 ++++++++++++++++++++++++++ 6 files changed, 416 insertions(+), 298 deletions(-) create mode 100644 cmd/swarm/swarm-smoke/sliding_window.go create mode 100644 cmd/swarm/swarm-smoke/util.go (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 2c5e3fd23..a322ba89c 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -2,13 +2,11 @@ package main import ( "bytes" - "context" "crypto/md5" + crand "crypto/rand" "fmt" "io" "io/ioutil" - "net/http" - "net/http/httptrace" "os" "os/exec" "strings" @@ -18,13 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/ethereum/go-ethereum/swarm/testutil" - colorable "github.com/mattn/go-colorable" - opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -33,27 +25,6 @@ const ( feedRandomDataLength = 8 ) -func cliFeedUploadAndSync(c *cli.Context) error { - metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1) - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) - - errc := make(chan error) - go func() { - errc <- feedUploadAndSync(c) - }() - - select { - case err := <-errc: - if err != nil { - metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1) - } - return err - case <-time.After(time.Duration(timeout) * time.Second): - metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1) - return fmt.Errorf("timeout after %v sec", timeout) - } -} - // TODO: retrieve with manifest + extract repeating code func feedUploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) @@ -232,9 +203,10 @@ func feedUploadAndSync(c *cli.Context) error { seed := int(time.Now().UnixNano() / 1e6) log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed) - randomBytes := testutil.RandomBytes(seed, filesize*1000) + h = md5.New() + r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) - hash, err := upload(&randomBytes, endpoints[0]) + hash, err := upload(r, filesize*1000, endpoints[0]) if err != nil { return err } @@ -243,10 +215,7 @@ func feedUploadAndSync(c *cli.Context) error { return err } multihashHex := hexutil.Encode(hashBytes) - fileHash, err := digest(bytes.NewReader(randomBytes)) - if err != nil { - return err - } + fileHash := h.Sum(nil) log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) @@ -307,60 +276,3 @@ func feedUploadAndSync(c *cli.Context) error { return nil } - -func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { - ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") - defer sp.Finish() - - log.Trace("sleeping", "ruid", ruid) - time.Sleep(3 * time.Second) - - log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) - - var tn time.Time - reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user - req, _ := http.NewRequest("GET", reqUri, nil) - - opentracing.GlobalTracer().Inject( - sp.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)) - - trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) - - req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) - transport := http.DefaultTransport - - //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - - tn = time.Now() - res, err := transport.RoundTrip(req) - if err != nil { - log.Error(err.Error(), "ruid", ruid) - return err - } - - log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) - - if res.StatusCode != 200 { - return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) - } - - defer res.Body.Close() - - rdigest, err := digest(res.Body) - if err != nil { - log.Warn(err.Error(), "ruid", ruid) - return err - } - - if !bytes.Equal(rdigest, original) { - err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) - log.Warn(err.Error(), "ruid", ruid) - return err - } - - log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) - - return nil -} diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index f7f358ef1..c382591f8 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -140,19 +140,25 @@ func main() { Name: "upload_and_sync", Aliases: []string{"c"}, Usage: "upload and sync", - Action: cliUploadAndSync, + Action: wrapCliCommand("upload-and-sync", true, uploadAndSync), }, { Name: "feed_sync", Aliases: []string{"f"}, Usage: "feed update generate, upload and sync", - Action: cliFeedUploadAndSync, + Action: wrapCliCommand("feed-and-sync", true, feedUploadAndSync), }, { Name: "upload_speed", Aliases: []string{"u"}, Usage: "measure upload speed", - Action: cliUploadSpeed, + Action: wrapCliCommand("upload-speed", true, uploadSpeed), + }, + { + Name: "sliding_window", + Aliases: []string{"s"}, + Usage: "measure network aggregate capacity", + Action: wrapCliCommand("sliding-window", false, slidingWindow), }, } diff --git a/cmd/swarm/swarm-smoke/sliding_window.go b/cmd/swarm/swarm-smoke/sliding_window.go new file mode 100644 index 000000000..3dd404c50 --- /dev/null +++ b/cmd/swarm/swarm-smoke/sliding_window.go @@ -0,0 +1,122 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "crypto/md5" + crand "crypto/rand" + "fmt" + "io" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/pborman/uuid" + + cli "gopkg.in/urfave/cli.v1" +) + +var seed = time.Now().UTC().UnixNano() + +func init() { + rand.Seed(seed) +} + +type uploadResult struct { + hash string + digest []byte +} + +func slidingWindow(c *cli.Context) error { + defer func(now time.Time) { + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime) + metrics.GetOrRegisterCounter("sliding-window.total-time", nil).Inc(int64(totalTime)) + }(time.Now()) + + generateEndpoints(scheme, cluster, appName, from, to) + hashes := []uploadResult{} //swarm hashes of the uploads + nodes := to - from + const iterationTimeout = 30 * time.Second + log.Info("sliding window test started", "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout) + uploadedBytes := 0 + networkDepth := 0 + errored := false + +outer: + for { + log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) + + h := md5.New() + r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) + t1 := time.Now() + + hash, err := upload(r, filesize*1000, endpoints[0]) + if err != nil { + log.Error(err.Error()) + return err + } + + metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1) + + fhash := h.Sum(nil) + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay) + hashes = append(hashes, uploadResult{hash: hash, digest: fhash}) + time.Sleep(time.Duration(syncDelay) * time.Second) + uploadedBytes += filesize * 1000 + + for i, v := range hashes { + timeout := time.After(time.Duration(timeout) * time.Second) + errored = false + + inner: + for { + select { + case <-timeout: + errored = true + log.Error("error retrieving hash. timeout", "hash idx", i, "err", err) + metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1) + break inner + default: + randIndex := 1 + rand.Intn(len(endpoints)-1) + ruid := uuid.New()[:8] + start := time.Now() + err := fetch(v.hash, endpoints[randIndex], v.digest, ruid) + if err != nil { + continue inner + } + metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start) + break inner + } + } + + if errored { + break outer + } + networkDepth = i + metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth)) + } + } + + log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize) + log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize) + + return nil +} diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 0fc86c55d..b0e1f3ee4 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -17,76 +17,21 @@ package main import ( - "bytes" - "context" "crypto/md5" crand "crypto/rand" - "errors" "fmt" "io" - "io/ioutil" "math/rand" - "net/http" - "net/http/httptrace" - "os" "sync" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/spancontext" - "github.com/ethereum/go-ethereum/swarm/testutil" - opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) -func generateEndpoints(scheme string, cluster string, app string, from int, to int) { - if cluster == "prod" { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) - } - } else if cluster == "private-internal" { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port)) - } - } else { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) - } - } - - if includeLocalhost { - endpoints = append(endpoints, "http://localhost:8500") - } -} - -func cliUploadAndSync(c *cli.Context) error { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - - metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1) - - errc := make(chan error) - go func() { - errc <- uploadAndSync(c) - }() - - select { - case err := <-errc: - if err != nil { - metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1) - } - return err - case <-time.After(time.Duration(timeout) * time.Second): - metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1) - return fmt.Errorf("timeout after %v sec", timeout) - } -} - func uploadAndSync(c *cli.Context) error { defer func(now time.Time) { totalTime := time.Since(now) @@ -96,23 +41,21 @@ func uploadAndSync(c *cli.Context) error { generateEndpoints(scheme, cluster, appName, from, to) seed := int(time.Now().UnixNano() / 1e6) + log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) - randomBytes := testutil.RandomBytes(seed, filesize*1000) + h := md5.New() + r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) t1 := time.Now() - hash, err := upload(&randomBytes, endpoints[0]) + hash, err := upload(r, filesize*1000, endpoints[0]) if err != nil { log.Error(err.Error()) return err } metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1) - fhash, err := digest(bytes.NewReader(randomBytes)) - if err != nil { - log.Error(err.Error()) - return err - } + fhash := h.Sum(nil) log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) @@ -161,96 +104,3 @@ func uploadAndSync(c *cli.Context) error { return nil } - -// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file -func fetch(hash string, endpoint string, original []byte, ruid string) error { - ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") - defer sp.Finish() - - log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - - var tn time.Time - reqUri := endpoint + "/bzz:/" + hash + "/" - req, _ := http.NewRequest("GET", reqUri, nil) - - opentracing.GlobalTracer().Inject( - sp.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)) - - trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn) - - req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) - transport := http.DefaultTransport - - //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - - tn = time.Now() - res, err := transport.RoundTrip(req) - if err != nil { - log.Error(err.Error(), "ruid", ruid) - return err - } - log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) - - if res.StatusCode != 200 { - err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode) - log.Warn(err.Error(), "ruid", ruid) - return err - } - - defer res.Body.Close() - - rdigest, err := digest(res.Body) - if err != nil { - log.Warn(err.Error(), "ruid", ruid) - return err - } - - if !bytes.Equal(rdigest, original) { - err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) - log.Warn(err.Error(), "ruid", ruid) - return err - } - - log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) - - return nil -} - -// upload is uploading a file `f` to `endpoint` via the `swarm up` cmd -func upload(dataBytes *[]byte, endpoint string) (string, error) { - swarm := client.NewClient(endpoint) - f := &client.File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)), - ManifestEntry: api.ManifestEntry{ - ContentType: "text/plain", - Mode: 0660, - Size: int64(len(*dataBytes)), - }, - } - - // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. - return swarm.Upload(f, "", false) -} - -func digest(r io.Reader) ([]byte, error) { - h := md5.New() - _, err := io.Copy(h, r) - if err != nil { - return nil, err - } - return h.Sum(nil), nil -} - -// generates random data in heap buffer -func generateRandomData(datasize int) ([]byte, error) { - b := make([]byte, datasize) - c, err := crand.Read(b) - if err != nil { - return nil, err - } else if c != datasize { - return nil, errors.New("short read") - } - return b, nil -} diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go index 4a355baf8..943c93240 100644 --- a/cmd/swarm/swarm-smoke/upload_speed.go +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -17,54 +17,18 @@ package main import ( - "bytes" + "crypto/md5" + crand "crypto/rand" "fmt" - "os" + "io" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/testutil" cli "gopkg.in/urfave/cli.v1" ) -var endpoint string - -//just use the first endpoint -func generateEndpoint(scheme string, cluster string, app string, from int) { - if cluster == "prod" { - endpoint = fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from) - } else if cluster == "private-internal" { - endpoint = fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from) - } else { - endpoint = fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster) - } -} - -func cliUploadSpeed(c *cli.Context) error { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - - metrics.GetOrRegisterCounter("upload-speed", nil).Inc(1) - - errc := make(chan error) - go func() { - errc <- uploadSpeed(c) - }() - - select { - case err := <-errc: - if err != nil { - metrics.GetOrRegisterCounter("upload-speed.fail", nil).Inc(1) - } - return err - case <-time.After(time.Duration(timeout) * time.Second): - metrics.GetOrRegisterCounter("upload-speed.timeout", nil).Inc(1) - return fmt.Errorf("timeout after %v sec", timeout) - } -} - func uploadSpeed(c *cli.Context) error { defer func(now time.Time) { totalTime := time.Since(now) @@ -73,25 +37,22 @@ func uploadSpeed(c *cli.Context) error { metrics.GetOrRegisterCounter("upload-speed.total-time", nil).Inc(int64(totalTime)) }(time.Now()) - generateEndpoint(scheme, cluster, appName, from) + endpoint := generateEndpoint(scheme, cluster, appName, from) seed := int(time.Now().UnixNano() / 1e6) log.Info("uploading to "+endpoint, "seed", seed) - randomBytes := testutil.RandomBytes(seed, filesize*1000) + h := md5.New() + r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) t1 := time.Now() - hash, err := upload(&randomBytes, endpoint) + hash, err := upload(r, filesize*1000, endpoint) if err != nil { log.Error(err.Error()) return err } metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1))) - fhash, err := digest(bytes.NewReader(randomBytes)) - if err != nil { - log.Error(err.Error()) - return err - } + fhash := h.Sum(nil) log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) return nil diff --git a/cmd/swarm/swarm-smoke/util.go b/cmd/swarm/swarm-smoke/util.go new file mode 100644 index 000000000..2a3083bf5 --- /dev/null +++ b/cmd/swarm/swarm-smoke/util.go @@ -0,0 +1,267 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "context" + "crypto/md5" + crand "crypto/rand" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptrace" + "os" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/spancontext" + opentracing "github.com/opentracing/opentracing-go" + cli "gopkg.in/urfave/cli.v1" +) + +var ( + commandName = "" +) + +func wrapCliCommand(name string, killOnTimeout bool, command func(*cli.Context) error) func(*cli.Context) error { + return func(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + defer func(now time.Time) { + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime) + metrics.GetOrRegisterCounter(name+".total-time", nil).Inc(int64(totalTime)) + }(time.Now()) + + log.Info("smoke test starting", "task", name, "timeout", timeout) + commandName = name + metrics.GetOrRegisterCounter(name, nil).Inc(1) + + errc := make(chan error) + done := make(chan struct{}) + + if killOnTimeout { + go func() { + <-time.After(time.Duration(timeout) * time.Second) + close(done) + }() + } + + go func() { + errc <- command(ctx) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", name), nil).Inc(1) + } + return err + case <-done: + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", name), nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } + } +} + +func generateEndpoints(scheme string, cluster string, app string, from int, to int) { + if cluster == "prod" { + for port := from; port < to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) + } + } else if cluster == "private-internal" { + for port := from; port < to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port)) + } + } else { + for port := from; port < to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) + } + } + + if includeLocalhost { + endpoints = append(endpoints, "http://localhost:8500") + } +} + +//just use the first endpoint +func generateEndpoint(scheme string, cluster string, app string, from int) string { + if cluster == "prod" { + return fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from) + } else if cluster == "private-internal" { + return fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from) + } else { + return fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster) + } +} + +func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") + defer sp.Finish() + + log.Trace("sleeping", "ruid", ruid) + time.Sleep(3 * time.Second) + + log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) + + var tn time.Time + reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) + if err != nil { + log.Error(err.Error(), "ruid", ruid) + return err + } + + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} + +// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file +func fetch(hash string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") + defer sp.Finish() + + log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) + + var tn time.Time + reqUri := endpoint + "/bzz:/" + hash + "/" + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace(commandName+" - http get", commandName, ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) + if err != nil { + log.Error(err.Error(), "ruid", ruid) + return err + } + log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} + +// upload an arbitrary byte as a plaintext file to `endpoint` using the api client +func upload(r io.Reader, size int, endpoint string) (string, error) { + swarm := client.NewClient(endpoint) + f := &client.File{ + ReadCloser: ioutil.NopCloser(r), + ManifestEntry: api.ManifestEntry{ + ContentType: "text/plain", + Mode: 0660, + Size: int64(size), + }, + } + + // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. + return swarm.Upload(f, "", false) +} + +func digest(r io.Reader) ([]byte, error) { + h := md5.New() + _, err := io.Copy(h, r) + if err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +// generates random data in heap buffer +func generateRandomData(datasize int) ([]byte, error) { + b := make([]byte, datasize) + c, err := crand.Read(b) + if err != nil { + return nil, err + } else if c != datasize { + return nil, errors.New("short read") + } + return b, nil +} -- cgit v1.2.3 From 355d55bd349e7ca6360f626279858c31cfc59898 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 30 Jan 2019 14:02:15 +0100 Subject: cmd/swarm/swarm-smoke: remove wrong metrics (#18970) (cherry picked from commit c5c9cef5c0baf1652b6642858ad2426794823699) --- cmd/swarm/swarm-smoke/sliding_window.go | 7 ------- cmd/swarm/swarm-smoke/upload_and_sync.go | 6 ------ cmd/swarm/swarm-smoke/upload_speed.go | 7 ------- cmd/swarm/swarm-smoke/util.go | 8 ++++---- 4 files changed, 4 insertions(+), 24 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/sliding_window.go b/cmd/swarm/swarm-smoke/sliding_window.go index 3dd404c50..3c4b1c79a 100644 --- a/cmd/swarm/swarm-smoke/sliding_window.go +++ b/cmd/swarm/swarm-smoke/sliding_window.go @@ -43,13 +43,6 @@ type uploadResult struct { } func slidingWindow(c *cli.Context) error { - defer func(now time.Time) { - totalTime := time.Since(now) - - log.Info("total time", "time", totalTime) - metrics.GetOrRegisterCounter("sliding-window.total-time", nil).Inc(int64(totalTime)) - }(time.Now()) - generateEndpoints(scheme, cluster, appName, from, to) hashes := []uploadResult{} //swarm hashes of the uploads nodes := to - from diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index b0e1f3ee4..c67989651 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -33,12 +33,6 @@ import ( ) func uploadAndSync(c *cli.Context) error { - defer func(now time.Time) { - totalTime := time.Since(now) - log.Info("total time", "time", totalTime, "kb", filesize) - metrics.GetOrRegisterResettingTimer("upload-and-sync.total-time", nil).Update(totalTime) - }(time.Now()) - generateEndpoints(scheme, cluster, appName, from, to) seed := int(time.Now().UnixNano() / 1e6) diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go index 943c93240..1123f2449 100644 --- a/cmd/swarm/swarm-smoke/upload_speed.go +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -30,13 +30,6 @@ import ( ) func uploadSpeed(c *cli.Context) error { - defer func(now time.Time) { - totalTime := time.Since(now) - - log.Info("total time", "time", totalTime, "kb", filesize) - metrics.GetOrRegisterCounter("upload-speed.total-time", nil).Inc(int64(totalTime)) - }(time.Now()) - endpoint := generateEndpoint(scheme, cluster, appName, from) seed := int(time.Now().UnixNano() / 1e6) log.Info("uploading to "+endpoint, "seed", seed) diff --git a/cmd/swarm/swarm-smoke/util.go b/cmd/swarm/swarm-smoke/util.go index 2a3083bf5..003973d41 100644 --- a/cmd/swarm/swarm-smoke/util.go +++ b/cmd/swarm/swarm-smoke/util.go @@ -46,12 +46,12 @@ var ( func wrapCliCommand(name string, killOnTimeout bool, command func(*cli.Context) error) func(*cli.Context) error { return func(ctx *cli.Context) error { log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + defer func(now time.Time) { totalTime := time.Since(now) - - log.Info("total time", "time", totalTime) - metrics.GetOrRegisterCounter(name+".total-time", nil).Inc(int64(totalTime)) + log.Info("total time", "time", totalTime, "kb", filesize) + metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime) }(time.Now()) log.Info("smoke test starting", "task", name, "timeout", timeout) -- cgit v1.2.3 From 637a75d61a13ee8a89a702a2eadb5ace3c79e7da Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 7 Feb 2019 14:38:32 +0100 Subject: cmd/swarm/swarm-smoke: refactor generateEndpoints (#19006) (cherry picked from commit d212535ddd5bf63a0c0b194525246480ae46c537) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 67 ++++++++++-------- cmd/swarm/swarm-smoke/main.go | 70 +++++++------------ cmd/swarm/swarm-smoke/sliding_window.go | 56 +++++++++------ cmd/swarm/swarm-smoke/upload_and_sync.go | 71 ++++++++++++------- cmd/swarm/swarm-smoke/upload_speed.go | 43 +++++++++--- cmd/swarm/swarm-smoke/util.go | 98 +++++++++------------------ 6 files changed, 214 insertions(+), 191 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index a322ba89c..6b3fed0c7 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -3,7 +3,6 @@ package main import ( "bytes" "crypto/md5" - crand "crypto/rand" "fmt" "io" "io/ioutil" @@ -16,7 +15,9 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -25,13 +26,28 @@ const ( feedRandomDataLength = 8 ) -// TODO: retrieve with manifest + extract repeating code -func feedUploadAndSync(c *cli.Context) error { - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) +func feedUploadAndSyncCmd(ctx *cli.Context, tuid string) error { + errc := make(chan error) - generateEndpoints(scheme, cluster, appName, from, to) + go func() { + errc <- feedUploadAndSync(ctx, tuid) + }() - log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing") + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func feedUploadAndSync(c *cli.Context, tuid string) error { + log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing") // create a random private key to sign updates with and derive the address pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") @@ -85,7 +101,7 @@ func feedUploadAndSync(c *cli.Context) error { // create feed manifest, topic only var out bytes.Buffer - cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) + cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex) cmd.Stdout = &out log.Debug("create feed manifest topic cmd", "cmd", cmd) err = cmd.Run() @@ -100,7 +116,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // create feed manifest, subtopic only - cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) + cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex) cmd.Stdout = &out log.Debug("create feed manifest subtopic cmd", "cmd", cmd) err = cmd.Run() @@ -115,7 +131,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // create feed manifest, merged topic - cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) + cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) cmd.Stdout = &out log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) err = cmd.Run() @@ -141,7 +157,7 @@ func feedUploadAndSync(c *cli.Context) error { dataHex := hexutil.Encode(data) // update with topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex) cmd.Stdout = &out log.Debug("update feed manifest topic cmd", "cmd", cmd) err = cmd.Run() @@ -152,7 +168,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update with subtopic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex) cmd.Stdout = &out log.Debug("update feed manifest subtopic cmd", "cmd", cmd) err = cmd.Run() @@ -163,7 +179,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update with merged topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) cmd.Stdout = &out log.Debug("update feed manifest merged topic cmd", "cmd", cmd) err = cmd.Run() @@ -177,14 +193,14 @@ func feedUploadAndSync(c *cli.Context) error { // retrieve the data wg := sync.WaitGroup{} - for _, endpoint := range endpoints { + for _, host := range hosts { // raw retrieve, topic only for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { wg.Add(1) ruid := uuid.New()[:8] go func(hex string, endpoint string, ruid string) { for { - err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) + err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid) if err != nil { continue } @@ -192,21 +208,18 @@ func feedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(hex, endpoint, ruid) - + }(hex, httpEndpoint(host), ruid) } } wg.Wait() log.Info("all endpoints synced random data successfully") // upload test file - seed := int(time.Now().UnixNano() / 1e6) - log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed) + log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed) - h = md5.New() - r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) + randomBytes := testutil.RandomBytes(seed, filesize*1000) - hash, err := upload(r, filesize*1000, endpoints[0]) + hash, err := upload(randomBytes, httpEndpoint(hosts[0])) if err != nil { return err } @@ -220,7 +233,7 @@ func feedUploadAndSync(c *cli.Context) error { log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) // update file with topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex) cmd.Stdout = &out err = cmd.Run() if err != nil { @@ -230,7 +243,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update file with subtopic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex) cmd.Stdout = &out err = cmd.Run() if err != nil { @@ -240,7 +253,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update file with merged topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) cmd.Stdout = &out err = cmd.Run() if err != nil { @@ -251,7 +264,7 @@ func feedUploadAndSync(c *cli.Context) error { time.Sleep(3 * time.Second) - for _, endpoint := range endpoints { + for _, host := range hosts { // manifest retrieve, topic only for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { @@ -259,7 +272,7 @@ func feedUploadAndSync(c *cli.Context) error { ruid := uuid.New()[:8] go func(url string, endpoint string, ruid string) { for { - err := fetch(url, endpoint, fileHash, ruid) + err := fetch(url, endpoint, fileHash, ruid, "") if err != nil { continue } @@ -267,7 +280,7 @@ func feedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(url, endpoint, ruid) + }(url, httpEndpoint(host), ruid) } } diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index c382591f8..4e977c668 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -37,18 +37,15 @@ var ( ) var ( - endpoints []string - includeLocalhost bool - cluster string - appName string - scheme string - filesize int - syncDelay int - from int - to int - verbosity int - timeout int - single bool + allhosts string + hosts []string + filesize int + syncDelay int + httpPort int + wsPort int + verbosity int + timeout int + single bool ) func main() { @@ -59,39 +56,22 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ - Name: "cluster-endpoint", - Value: "prod", - Usage: "cluster to point to (prod or a given namespace)", - Destination: &cluster, - }, - cli.StringFlag{ - Name: "app", - Value: "swarm", - Usage: "application to point to (swarm or swarm-private)", - Destination: &appName, + Name: "hosts", + Value: "", + Usage: "comma-separated list of swarm hosts", + Destination: &allhosts, }, cli.IntFlag{ - Name: "cluster-from", - Value: 8501, - Usage: "swarm node (from)", - Destination: &from, + Name: "http-port", + Value: 80, + Usage: "http port", + Destination: &httpPort, }, cli.IntFlag{ - Name: "cluster-to", - Value: 8512, - Usage: "swarm node (to)", - Destination: &to, - }, - cli.StringFlag{ - Name: "cluster-scheme", - Value: "http", - Usage: "http or https", - Destination: &scheme, - }, - cli.BoolFlag{ - Name: "include-localhost", - Usage: "whether to include localhost:8500 as an endpoint", - Destination: &includeLocalhost, + Name: "ws-port", + Value: 8546, + Usage: "ws port", + Destination: &wsPort, }, cli.IntFlag{ Name: "filesize", @@ -140,25 +120,25 @@ func main() { Name: "upload_and_sync", Aliases: []string{"c"}, Usage: "upload and sync", - Action: wrapCliCommand("upload-and-sync", true, uploadAndSync), + Action: wrapCliCommand("upload-and-sync", uploadAndSyncCmd), }, { Name: "feed_sync", Aliases: []string{"f"}, Usage: "feed update generate, upload and sync", - Action: wrapCliCommand("feed-and-sync", true, feedUploadAndSync), + Action: wrapCliCommand("feed-and-sync", feedUploadAndSyncCmd), }, { Name: "upload_speed", Aliases: []string{"u"}, Usage: "measure upload speed", - Action: wrapCliCommand("upload-speed", true, uploadSpeed), + Action: wrapCliCommand("upload-speed", uploadSpeedCmd), }, { Name: "sliding_window", Aliases: []string{"s"}, Usage: "measure network aggregate capacity", - Action: wrapCliCommand("sliding-window", false, slidingWindow), + Action: wrapCliCommand("sliding-window", slidingWindowCmd), }, } diff --git a/cmd/swarm/swarm-smoke/sliding_window.go b/cmd/swarm/swarm-smoke/sliding_window.go index 3c4b1c79a..d313bbc37 100644 --- a/cmd/swarm/swarm-smoke/sliding_window.go +++ b/cmd/swarm/swarm-smoke/sliding_window.go @@ -17,50 +17,62 @@ package main import ( - "crypto/md5" - crand "crypto/rand" + "bytes" "fmt" - "io" "math/rand" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) -var seed = time.Now().UTC().UnixNano() - -func init() { - rand.Seed(seed) -} - type uploadResult struct { hash string digest []byte } -func slidingWindow(c *cli.Context) error { - generateEndpoints(scheme, cluster, appName, from, to) +func slidingWindowCmd(ctx *cli.Context, tuid string) error { + errc := make(chan error) + + go func() { + errc <- slidingWindow(ctx, tuid) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func slidingWindow(ctx *cli.Context, tuid string) error { hashes := []uploadResult{} //swarm hashes of the uploads - nodes := to - from + nodes := len(hosts) const iterationTimeout = 30 * time.Second - log.Info("sliding window test started", "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout) + log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout) uploadedBytes := 0 networkDepth := 0 errored := false outer: for { - log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) + log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed) - h := md5.New() - r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) t1 := time.Now() - hash, err := upload(r, filesize*1000, endpoints[0]) + randomBytes := testutil.RandomBytes(seed, filesize*1000) + + hash, err := upload(randomBytes, httpEndpoint(hosts[0])) if err != nil { log.Error(err.Error()) return err @@ -68,7 +80,11 @@ outer: metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1) - fhash := h.Sum(nil) + fhash, err := digest(bytes.NewReader(randomBytes)) + if err != nil { + log.Error(err.Error()) + return err + } log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay) hashes = append(hashes, uploadResult{hash: hash, digest: fhash}) @@ -88,10 +104,10 @@ outer: metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1) break inner default: - randIndex := 1 + rand.Intn(len(endpoints)-1) + idx := 1 + rand.Intn(len(hosts)-1) ruid := uuid.New()[:8] start := time.Now() - err := fetch(v.hash, endpoints[randIndex], v.digest, ruid) + err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid, "") if err != nil { continue inner } diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index c67989651..b2858e227 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -17,84 +17,109 @@ package main import ( - "crypto/md5" - crand "crypto/rand" + "bytes" "fmt" - "io" "math/rand" "sync" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) -func uploadAndSync(c *cli.Context) error { - generateEndpoints(scheme, cluster, appName, from, to) - seed := int(time.Now().UnixNano() / 1e6) +func uploadAndSyncCmd(ctx *cli.Context, tuid string) error { + randomBytes := testutil.RandomBytes(seed, filesize*1000) - log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) + errc := make(chan error) - h := md5.New() - r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) + go func() { + errc <- uplaodAndSync(ctx, randomBytes, tuid) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + // trigger debug functionality on randomBytes + + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error { + log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed) t1 := time.Now() - hash, err := upload(r, filesize*1000, endpoints[0]) + hash, err := upload(randomBytes, httpEndpoint(hosts[0])) if err != nil { log.Error(err.Error()) return err } - metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1) + t2 := time.Since(t1) + metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).Update(t2) - fhash := h.Sum(nil) + fhash, err := digest(bytes.NewReader(randomBytes)) + if err != nil { + log.Error(err.Error()) + return err + } - log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) + log.Info("uploaded successfully", "tuid", tuid, "hash", hash, "took", t2, "digest", fmt.Sprintf("%x", fhash)) time.Sleep(time.Duration(syncDelay) * time.Second) wg := sync.WaitGroup{} if single { - rand.Seed(time.Now().UTC().UnixNano()) - randIndex := 1 + rand.Intn(len(endpoints)-1) + randIndex := 1 + rand.Intn(len(hosts)-1) ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { start := time.Now() - err := fetch(hash, endpoint, fhash, ruid) + err := fetch(hash, endpoint, fhash, ruid, tuid) if err != nil { continue } + ended := time.Since(start) - metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).UpdateSince(start) + metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).Update(ended) + log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint) wg.Done() return } - }(endpoints[randIndex], ruid) + }(httpEndpoint(hosts[randIndex]), ruid) } else { - for _, endpoint := range endpoints[1:] { + for _, endpoint := range hosts[1:] { ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { start := time.Now() - err := fetch(hash, endpoint, fhash, ruid) + err := fetch(hash, endpoint, fhash, ruid, tuid) if err != nil { continue } + ended := time.Since(start) - metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).UpdateSince(start) + metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).Update(ended) + log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint) wg.Done() return } - }(endpoint, ruid) + }(httpEndpoint(endpoint), ruid) } } wg.Wait() - log.Info("all endpoints synced random file successfully") + log.Info("all hosts synced random file successfully") return nil } diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go index 1123f2449..20bf7b86c 100644 --- a/cmd/swarm/swarm-smoke/upload_speed.go +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -17,35 +17,56 @@ package main import ( - "crypto/md5" - crand "crypto/rand" + "bytes" "fmt" - "io" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" cli "gopkg.in/urfave/cli.v1" ) -func uploadSpeed(c *cli.Context) error { - endpoint := generateEndpoint(scheme, cluster, appName, from) - seed := int(time.Now().UnixNano() / 1e6) - log.Info("uploading to "+endpoint, "seed", seed) +func uploadSpeedCmd(ctx *cli.Context, tuid string) error { + log.Info("uploading to "+hosts[0], "tuid", tuid, "seed", seed) + randomBytes := testutil.RandomBytes(seed, filesize*1000) - h := md5.New() - r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h) + errc := make(chan error) + go func() { + errc <- uploadSpeed(ctx, tuid, randomBytes) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + // trigger debug functionality on randomBytes + + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uploadSpeed(c *cli.Context, tuid string, data []byte) error { t1 := time.Now() - hash, err := upload(r, filesize*1000, endpoint) + hash, err := upload(data, hosts[0]) if err != nil { log.Error(err.Error()) return err } metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1))) - fhash := h.Sum(nil) + fhash, err := digest(bytes.NewReader(data)) + if err != nil { + log.Error(err.Error()) + return err + } log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) return nil diff --git a/cmd/swarm/swarm-smoke/util.go b/cmd/swarm/swarm-smoke/util.go index 003973d41..87abb44b0 100644 --- a/cmd/swarm/swarm-smoke/util.go +++ b/cmd/swarm/swarm-smoke/util.go @@ -25,9 +25,11 @@ import ( "fmt" "io" "io/ioutil" + "math/rand" "net/http" "net/http/httptrace" "os" + "strings" "time" "github.com/ethereum/go-ethereum/log" @@ -36,83 +38,49 @@ import ( "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/spancontext" opentracing "github.com/opentracing/opentracing-go" + "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) var ( commandName = "" + seed = int(time.Now().UTC().UnixNano()) ) -func wrapCliCommand(name string, killOnTimeout bool, command func(*cli.Context) error) func(*cli.Context) error { +func init() { + rand.Seed(int64(seed)) +} + +func httpEndpoint(host string) string { + return fmt.Sprintf("http://%s:%d", host, httpPort) +} + +func wsEndpoint(host string) string { + return fmt.Sprintf("ws://%s:%d", host, wsPort) +} + +func wrapCliCommand(name string, command func(*cli.Context, string) error) func(*cli.Context) error { return func(ctx *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + // test uuid + tuid := uuid.New()[:8] + + commandName = name + + hosts = strings.Split(allhosts, ",") + defer func(now time.Time) { totalTime := time.Since(now) - log.Info("total time", "time", totalTime, "kb", filesize) + log.Info("total time", "tuid", tuid, "time", totalTime, "kb", filesize) metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime) }(time.Now()) - log.Info("smoke test starting", "task", name, "timeout", timeout) - commandName = name + log.Info("smoke test starting", "tuid", tuid, "task", name, "timeout", timeout) metrics.GetOrRegisterCounter(name, nil).Inc(1) - errc := make(chan error) - done := make(chan struct{}) - - if killOnTimeout { - go func() { - <-time.After(time.Duration(timeout) * time.Second) - close(done) - }() - } - - go func() { - errc <- command(ctx) - }() - - select { - case err := <-errc: - if err != nil { - metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", name), nil).Inc(1) - } - return err - case <-done: - metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", name), nil).Inc(1) - return fmt.Errorf("timeout after %v sec", timeout) - } - } -} - -func generateEndpoints(scheme string, cluster string, app string, from int, to int) { - if cluster == "prod" { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) - } - } else if cluster == "private-internal" { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port)) - } - } else { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) - } - } - - if includeLocalhost { - endpoints = append(endpoints, "http://localhost:8500") - } -} - -//just use the first endpoint -func generateEndpoint(scheme string, cluster string, app string, from int) string { - if cluster == "prod" { - return fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from) - } else if cluster == "private-internal" { - return fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from) - } else { - return fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster) + return command(ctx, tuid) } } @@ -174,11 +142,11 @@ func fetchFeed(topic string, user string, endpoint string, original []byte, ruid } // fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file -func fetch(hash string, endpoint string, original []byte, ruid string) error { +func fetch(hash string, endpoint string, original []byte, ruid string, tuid string) error { ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") defer sp.Finish() - log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) + log.Info("http get request", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash) var tn time.Time reqUri := endpoint + "/bzz:/" + hash + "/" @@ -202,7 +170,7 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { log.Error(err.Error(), "ruid", ruid) return err } - log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) + log.Info("http get response", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) if res.StatusCode != 200 { err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode) @@ -230,14 +198,14 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { } // upload an arbitrary byte as a plaintext file to `endpoint` using the api client -func upload(r io.Reader, size int, endpoint string) (string, error) { +func upload(data []byte, endpoint string) (string, error) { swarm := client.NewClient(endpoint) f := &client.File{ - ReadCloser: ioutil.NopCloser(r), + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), ManifestEntry: api.ManifestEntry{ ContentType: "text/plain", Mode: 0660, - Size: int64(size), + Size: int64(len(data)), }, } -- cgit v1.2.3 From d1ace4f344616fb6fa8643872c1f9cac89f8549e Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 7 Feb 2019 09:49:19 -0500 Subject: swarm: Debug API and HasChunks() API endpoint (#18980) (cherry picked from commit 41597c2856d6ac7328baca1340c3e36ab0edd382) --- cmd/swarm/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmd') diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 8af5d8c32..1e96d1294 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -451,5 +451,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) { } cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node) } - log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes)) + } -- cgit v1.2.3 From 85217b08bde168c0fd41d7265afe401454bb0f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 7 Feb 2019 15:46:58 +0100 Subject: cmd/swarm/global-store: global store cmd (#19014) (cherry picked from commit 33d0a0efa61fed2b16797fd12161519943943282) --- cmd/swarm/config.go | 9 ++ cmd/swarm/flags.go | 5 + cmd/swarm/global-store/global_store.go | 100 +++++++++++++++ cmd/swarm/global-store/global_store_test.go | 191 ++++++++++++++++++++++++++++ cmd/swarm/global-store/main.go | 104 +++++++++++++++ cmd/swarm/global-store/run_test.go | 49 +++++++ cmd/swarm/main.go | 20 ++- 7 files changed, 475 insertions(+), 3 deletions(-) create mode 100644 cmd/swarm/global-store/global_store.go create mode 100644 cmd/swarm/global-store/global_store_test.go create mode 100644 cmd/swarm/global-store/main.go create mode 100644 cmd/swarm/global-store/run_test.go (limited to 'cmd') diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 0203a6798..98d4dee7b 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -82,6 +82,7 @@ const ( SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" + SWARM_GLOBALSTORE_API = "SWARM_GLOBALSTORE_API" GETH_ENV_DATADIR = "GETH_DATADIR" ) @@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) } + if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) { + currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name) + } + return currentConfig } @@ -375,6 +380,10 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.BootnodeMode = bootnodeMode } + if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" { + currentConfig.GlobalStoreAPI = api + } + return currentConfig } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 4c186cc31..b092a7747 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -176,4 +176,9 @@ var ( Name: "user", Usage: "Indicates the user who updates the feed", } + SwarmGlobalStoreAPIFlag = cli.StringFlag{ + Name: "globalstore-api", + Usage: "URL of the Global Store API provider (only for testing)", + EnvVar: SWARM_GLOBALSTORE_API, + } ) diff --git a/cmd/swarm/global-store/global_store.go b/cmd/swarm/global-store/global_store.go new file mode 100644 index 000000000..a55756e1c --- /dev/null +++ b/cmd/swarm/global-store/global_store.go @@ -0,0 +1,100 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "net" + "net/http" + "os" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" + cli "gopkg.in/urfave/cli.v1" +) + +// startHTTP starts a global store with HTTP RPC server. +// It is used for "http" cli command. +func startHTTP(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + log.Info("http", "address", listener.Addr().String()) + + return http.Serve(listener, server) +} + +// startWS starts a global store with WebSocket RPC server. +// It is used for "websocket" cli command. +func startWS(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + origins := ctx.StringSlice("origins") + log.Info("websocket", "address", listener.Addr().String(), "origins", origins) + + return http.Serve(listener, server.WebsocketHandler(origins)) +} + +// newServer creates a global store and returns its RPC server. +// Returned cleanup function should be called only if err is nil. +func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + + cleanup = func() {} + var globalStore mock.GlobalStorer + dir := ctx.String("dir") + if dir != "" { + dbStore, err := db.NewGlobalStore(dir) + if err != nil { + return nil, nil, err + } + cleanup = func() { + dbStore.Close() + } + globalStore = dbStore + log.Info("database global store", "dir", dir) + } else { + globalStore = mem.NewGlobalStore() + log.Info("in-memory global store") + } + + server = rpc.NewServer() + if err := server.RegisterName("mockStore", globalStore); err != nil { + cleanup() + return nil, nil, err + } + + return server, cleanup, nil +} diff --git a/cmd/swarm/global-store/global_store_test.go b/cmd/swarm/global-store/global_store_test.go new file mode 100644 index 000000000..85f361ed4 --- /dev/null +++ b/cmd/swarm/global-store/global_store_test.go @@ -0,0 +1,191 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" +) + +// TestHTTP_InMemory tests in-memory global store that exposes +// HTTP server. +func TestHTTP_InMemory(t *testing.T) { + testHTTP(t, true) +} + +// TestHTTP_Database tests global store with persisted database +// that exposes HTTP server. +func TestHTTP_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testHTTP(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testHTTP(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with HTTP server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testHTTP(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + client, err := rpc.DialHTTP("http://" + addr) + if err != nil { + t.Fatal(err) + } + + // wait until global store process is started as + // rpc.DialHTTP is actually not connecting + for i := 0; i < 1000; i++ { + _, err = http.DefaultClient.Get("http://" + addr) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// TestWebsocket_InMemory tests in-memory global store that exposes +// WebSocket server. +func TestWebsocket_InMemory(t *testing.T) { + testWebsocket(t, true) +} + +// TestWebsocket_Database tests global store with persisted database +// that exposes HTTP server. +func TestWebsocket_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testWebsocket(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testWebsocket(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with WebSocket server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testWebsocket(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + var client *rpc.Client + var err error + // wait until global store process is started + for i := 0; i < 1000; i++ { + client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "") + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// findFreeTCPAddress returns a local address (IP:Port) to which +// global store can listen on. +func findFreeTCPAddress(t *testing.T) (addr string) { + t.Helper() + + listener, err := net.Listen("tcp", "") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + return listener.Addr().String() +} diff --git a/cmd/swarm/global-store/main.go b/cmd/swarm/global-store/main.go new file mode 100644 index 000000000..51df0099a --- /dev/null +++ b/cmd/swarm/global-store/main.go @@ -0,0 +1,104 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Global Store. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Global Store") + + app.Name = "global-store" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 3, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "http", + Aliases: []string{"h"}, + Usage: "start swarm global store with http server", + Action: startHTTP, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for http connection", + }, + ), + }, + { + Name: "websocket", + Aliases: []string{"ws"}, + Usage: "start swarm global store with websocket server", + Action: startWS, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for websocket connection", + }, + cli.StringSliceFlag{ + Name: "origins", + Value: &cli.StringSlice{"*"}, + Usage: "websocket origins", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/global-store/run_test.go b/cmd/swarm/global-store/run_test.go new file mode 100644 index 000000000..d7ef626e5 --- /dev/null +++ b/cmd/swarm/global-store/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-global-store", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-global-store", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 1e96d1294..3053ea1b3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -39,13 +39,16 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" "github.com/ethereum/go-ethereum/swarm/tracing" sv "github.com/ethereum/go-ethereum/swarm/version" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const clientIdentifier = "swarm" @@ -196,6 +199,7 @@ func init() { SwarmStorePath, SwarmStoreCapacity, SwarmStoreCacheCapacity, + SwarmGlobalStoreAPIFlag, } rpcFlags := []cli.Flag{ utils.WSEnabledFlag, @@ -324,8 +328,18 @@ func bzzd(ctx *cli.Context) error { func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) { //define the swarm service boot function boot := func(_ *node.ServiceContext) (node.Service, error) { - // In production, mockStore must be always nil. - return swarm.NewSwarm(bzzconfig, nil) + var nodeStore *mock.NodeStore + if bzzconfig.GlobalStoreAPI != "" { + // connect to global store + client, err := rpc.Dial(bzzconfig.GlobalStoreAPI) + if err != nil { + return nil, fmt.Errorf("global store: %v", err) + } + globalStore := mockrpc.NewGlobalStore(client) + // create a node store for this swarm key on global store + nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey)) + } + return swarm.NewSwarm(bzzconfig, nodeStore) } //register within the ethereum node if err := stack.Register(boot); err != nil { -- cgit v1.2.3 From 996230174ce203f328e622b82ffc0b9e0fc1c99b Mon Sep 17 00:00:00 2001 From: holisticode Date: Mon, 18 Feb 2019 06:05:22 -0500 Subject: cmd/swarm/swarm-smoke: Trigger chunk debug on timeout (#19101) * cmd/swarm/swarm-smoke: first version trigger has-chunks on timeout * cmd/swarm/swarm-smoke: finalize trigger to chunk debug * cmd/swarm/swarm-smoke: fixed httpEndpoint for trigger * cmd/swarm/swarm-smoke: port * cmd/swarm/swarm-smoke: ws not rpc * cmd/swarm/swarm-smoke: added debug output * cmd/swarm/swarm-smoke: addressed PR comments * cmd/swarm/swarm-smoke: renamed track-timeout and track-chunks (cherry picked from commit 62d7688d0a7ddbdb5d7167b264e0ea617578b60d) --- cmd/swarm/swarm-smoke/main.go | 25 +++++++---- cmd/swarm/swarm-smoke/upload_and_sync.go | 71 +++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 10 deletions(-) (limited to 'cmd') diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 4e977c668..4f28e86b1 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -37,15 +37,16 @@ var ( ) var ( - allhosts string - hosts []string - filesize int - syncDelay int - httpPort int - wsPort int - verbosity int - timeout int - single bool + allhosts string + hosts []string + filesize int + syncDelay int + httpPort int + wsPort int + verbosity int + timeout int + single bool + trackTimeout int ) func main() { @@ -102,6 +103,12 @@ func main() { Usage: "whether to fetch content from a single node or from all nodes", Destination: &single, }, + cli.IntFlag{ + Name: "track-timeout", + Value: 5, + Usage: "timeout in seconds to wait for GetAllReferences to return", + Destination: &trackTimeout, + }, } app.Flags = append(app.Flags, []cli.Flag{ diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index b2858e227..90230df25 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -18,13 +18,19 @@ package main import ( "bytes" + "context" "fmt" + "io/ioutil" "math/rand" + "os" "sync" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/pborman/uuid" @@ -49,12 +55,75 @@ func uploadAndSyncCmd(ctx *cli.Context, tuid string) error { case <-time.After(time.Duration(timeout) * time.Second): metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + e := fmt.Errorf("timeout after %v sec", timeout) // trigger debug functionality on randomBytes + err := trackChunks(randomBytes[:]) + if err != nil { + e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err) + } - return fmt.Errorf("timeout after %v sec", timeout) + return e } } +func trackChunks(testData []byte) error { + log.Warn("Test timed out; running chunk debug sequence") + + addrs, err := getAllRefs(testData) + if err != nil { + return err + } + log.Trace("All references retrieved") + + // has-chunks + for _, host := range hosts { + httpHost := fmt.Sprintf("ws://%s:%d", host, 8546) + log.Trace("Calling `Has` on host", "httpHost", httpHost) + rpcClient, err := rpc.Dial(httpHost) + if err != nil { + log.Trace("Error dialing host", "err", err) + return err + } + log.Trace("rpc dial ok") + var hasInfo []api.HasInfo + err = rpcClient.Call(&hasInfo, "bzz_has", addrs) + if err != nil { + log.Trace("Error calling host", "err", err) + return err + } + log.Trace("rpc call ok") + count := 0 + for _, info := range hasInfo { + if !info.Has { + count++ + log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr) + } + } + if count == 0 { + log.Info("Host reported to have all chunks", "host", httpHost) + } + } + return nil +} + +func getAllRefs(testData []byte) (storage.AddressCollection, error) { + log.Trace("Getting all references for given root hash") + datadir, err := ioutil.TempDir("", "chunk-debug") + if err != nil { + return nil, fmt.Errorf("unable to create temp dir: %v", err) + } + defer os.RemoveAll(datadir) + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32)) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second) + defer cancel() + + reader := bytes.NewReader(testData) + return fileStore.GetAllReferences(ctx, reader, false) +} + func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error { log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed) -- cgit v1.2.3 From 4f908db69e8cb69cdf45775c7c75e74244e91653 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 29 Jan 2019 09:14:24 +0100 Subject: cmd/utils: allow for multiple influxdb tags (#18520) This PR is replacing the metrics.influxdb.host.tag cmd-line flag with metrics.influxdb.tags - a comma-separated key/value tags, that are passed to the InfluxDB reporter, so that we can index measurements with multiple tags, and not just one host tag. This will be useful for Swarm, where we want to index measurements not just with the host tag, but also with bzzkey and git commit version (for long-running deployments). (cherry picked from commit 21acf0bc8d4f179397bb7d06d6f36df3cbee4a8e) --- cmd/geth/main.go | 4 +-- cmd/geth/usage.go | 4 +-- cmd/swarm/swarm-smoke/main.go | 15 +++++----- cmd/utils/flags.go | 41 +++++++++++++++++++-------- cmd/utils/flags_test.go | 64 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 23 deletions(-) create mode 100644 cmd/utils/flags_test.go (limited to 'cmd') diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c62fd87f2..e60a27e43 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const ( @@ -165,7 +165,7 @@ var ( utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, - utils.MetricsInfluxDBHostTagFlag, + utils.MetricsInfluxDBTagsFlag, } ) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index ed7e98bba..6823aa36c 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/debug" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) // AppHelpTemplate is the test template for the default, global app help topic. @@ -230,7 +230,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, - utils.MetricsInfluxDBHostTagFlag, + utils.MetricsInfluxDBTagsFlag, }, }, { diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 4f28e86b1..43d2c1ff5 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -117,7 +117,7 @@ func main() { swarmmetrics.MetricsInfluxDBDatabaseFlag, swarmmetrics.MetricsInfluxDBUsernameFlag, swarmmetrics.MetricsInfluxDBPasswordFlag, - swarmmetrics.MetricsInfluxDBHostTagFlag, + swarmmetrics.MetricsInfluxDBTagsFlag, }...) app.Flags = append(app.Flags, tracing.Flags...) @@ -176,13 +176,14 @@ func emitMetrics(ctx *cli.Context) error { database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name) username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name) password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) + tags = ctx.GlobalString(swarmmetrics.MetricsInfluxDBTagsFlag.Name) ) - return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ - "host": hosttag, - "version": gitCommit, - "filesize": fmt.Sprintf("%v", filesize), - }) + + tagsMap := utils.SplitTagsFlag(tags) + tagsMap["version"] = gitCommit + tagsMap["filesize"] = fmt.Sprintf("%v", filesize) + + return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", tagsMap) } return nil diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a65a36204..55e84b876 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -57,7 +57,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -618,14 +618,14 @@ var ( Usage: "Password to authorize access to the database", Value: "test", } - // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. - // It is used so that we can group all nodes and average a measurement across all of them, but also so - // that we can select a specific node and inspect its measurements. + // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. + // For example `host` tag could be used so that we can group all nodes and average a measurement + // across all of them, but also so that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - MetricsInfluxDBHostTagFlag = cli.StringFlag{ - Name: "metrics.influxdb.host.tag", - Usage: "InfluxDB `host` tag attached to all measurements", - Value: "localhost", + MetricsInfluxDBTagsFlag = cli.StringFlag{ + Name: "metrics.influxdb.tags", + Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", + Value: "host=localhost", } EWASMInterpreterFlag = cli.StringFlag{ @@ -1378,16 +1378,33 @@ func SetupMetrics(ctx *cli.Context) { database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) ) if enableExport { + tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + log.Info("Enabling metrics export to InfluxDB") - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", map[string]string{ - "host": hosttag, - }) + + go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) + } + } +} + +func SplitTagsFlag(tagsFlag string) map[string]string { + tags := strings.Split(tagsFlag, ",") + tagsMap := map[string]string{} + + for _, t := range tags { + if t != "" { + kv := strings.Split(t, "=") + + if len(kv) == 2 { + tagsMap[kv[0]] = kv[1] + } } } + + return tagsMap } // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. diff --git a/cmd/utils/flags_test.go b/cmd/utils/flags_test.go new file mode 100644 index 000000000..adfdd0903 --- /dev/null +++ b/cmd/utils/flags_test.go @@ -0,0 +1,64 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Package utils contains internal helper functions for go-ethereum commands. +package utils + +import ( + "reflect" + "testing" +) + +func Test_SplitTagsFlag(t *testing.T) { + tests := []struct { + name string + args string + want map[string]string + }{ + { + "2 tags case", + "host=localhost,bzzkey=123", + map[string]string{ + "host": "localhost", + "bzzkey": "123", + }, + }, + { + "1 tag case", + "host=localhost123", + map[string]string{ + "host": "localhost123", + }, + }, + { + "empty case", + "", + map[string]string{}, + }, + { + "garbage", + "smth=smthelse=123", + map[string]string{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SplitTagsFlag(tt.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitTagsFlag() = %v, want %v", got, tt.want) + } + }) + } +} -- cgit v1.2.3