diff options
| -rw-r--r-- | cmd/swarm/config.go | 9 | ||||
| -rw-r--r-- | cmd/swarm/flags.go | 5 | ||||
| -rw-r--r-- | cmd/swarm/global-store/global_store.go | 100 | ||||
| -rw-r--r-- | cmd/swarm/global-store/global_store_test.go | 191 | ||||
| -rw-r--r-- | cmd/swarm/global-store/main.go | 104 | ||||
| -rw-r--r-- | cmd/swarm/global-store/run_test.go | 49 | ||||
| -rw-r--r-- | cmd/swarm/main.go | 20 | ||||
| -rw-r--r-- | swarm/api/config.go | 1 |
8 files changed, 476 insertions, 3 deletions
diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 0203a6798..98d4dee7b 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -82,6 +82,7 @@ const ( SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" + SWARM_GLOBALSTORE_API = "SWARM_GLOBALSTORE_API" GETH_ENV_DATADIR = "GETH_DATADIR" ) @@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) } + if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) { + currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name) + } + return currentConfig } @@ -375,6 +380,10 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.BootnodeMode = bootnodeMode } + if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" { + currentConfig.GlobalStoreAPI = api + } + return currentConfig } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 4c186cc31..b092a7747 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -176,4 +176,9 @@ var ( Name: "user", Usage: "Indicates the user who updates the feed", } + SwarmGlobalStoreAPIFlag = cli.StringFlag{ + Name: "globalstore-api", + Usage: "URL of the Global Store API provider (only for testing)", + EnvVar: SWARM_GLOBALSTORE_API, + } ) diff --git a/cmd/swarm/global-store/global_store.go b/cmd/swarm/global-store/global_store.go new file mode 100644 index 000000000..a55756e1c --- /dev/null +++ b/cmd/swarm/global-store/global_store.go @@ -0,0 +1,100 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "net" + "net/http" + "os" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" + cli "gopkg.in/urfave/cli.v1" +) + +// startHTTP starts a global store with HTTP RPC server. +// It is used for "http" cli command. +func startHTTP(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + log.Info("http", "address", listener.Addr().String()) + + return http.Serve(listener, server) +} + +// startWS starts a global store with WebSocket RPC server. +// It is used for "websocket" cli command. +func startWS(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + origins := ctx.StringSlice("origins") + log.Info("websocket", "address", listener.Addr().String(), "origins", origins) + + return http.Serve(listener, server.WebsocketHandler(origins)) +} + +// newServer creates a global store and returns its RPC server. +// Returned cleanup function should be called only if err is nil. +func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + + cleanup = func() {} + var globalStore mock.GlobalStorer + dir := ctx.String("dir") + if dir != "" { + dbStore, err := db.NewGlobalStore(dir) + if err != nil { + return nil, nil, err + } + cleanup = func() { + dbStore.Close() + } + globalStore = dbStore + log.Info("database global store", "dir", dir) + } else { + globalStore = mem.NewGlobalStore() + log.Info("in-memory global store") + } + + server = rpc.NewServer() + if err := server.RegisterName("mockStore", globalStore); err != nil { + cleanup() + return nil, nil, err + } + + return server, cleanup, nil +} diff --git a/cmd/swarm/global-store/global_store_test.go b/cmd/swarm/global-store/global_store_test.go new file mode 100644 index 000000000..85f361ed4 --- /dev/null +++ b/cmd/swarm/global-store/global_store_test.go @@ -0,0 +1,191 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "context" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" +) + +// TestHTTP_InMemory tests in-memory global store that exposes +// HTTP server. +func TestHTTP_InMemory(t *testing.T) { + testHTTP(t, true) +} + +// TestHTTP_Database tests global store with persisted database +// that exposes HTTP server. +func TestHTTP_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testHTTP(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testHTTP(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with HTTP server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testHTTP(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + client, err := rpc.DialHTTP("http://" + addr) + if err != nil { + t.Fatal(err) + } + + // wait until global store process is started as + // rpc.DialHTTP is actually not connecting + for i := 0; i < 1000; i++ { + _, err = http.DefaultClient.Get("http://" + addr) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// TestWebsocket_InMemory tests in-memory global store that exposes +// WebSocket server. +func TestWebsocket_InMemory(t *testing.T) { + testWebsocket(t, true) +} + +// TestWebsocket_Database tests global store with persisted database +// that exposes HTTP server. +func TestWebsocket_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testWebsocket(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testWebsocket(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with WebSocket server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testWebsocket(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + var client *rpc.Client + var err error + // wait until global store process is started + for i := 0; i < 1000; i++ { + client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "") + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// findFreeTCPAddress returns a local address (IP:Port) to which +// global store can listen on. +func findFreeTCPAddress(t *testing.T) (addr string) { + t.Helper() + + listener, err := net.Listen("tcp", "") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + return listener.Addr().String() +} diff --git a/cmd/swarm/global-store/main.go b/cmd/swarm/global-store/main.go new file mode 100644 index 000000000..51df0099a --- /dev/null +++ b/cmd/swarm/global-store/main.go @@ -0,0 +1,104 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Global Store. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Global Store") + + app.Name = "global-store" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 3, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "http", + Aliases: []string{"h"}, + Usage: "start swarm global store with http server", + Action: startHTTP, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for http connection", + }, + ), + }, + { + Name: "websocket", + Aliases: []string{"ws"}, + Usage: "start swarm global store with websocket server", + Action: startWS, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for websocket connection", + }, + cli.StringSliceFlag{ + Name: "origins", + Value: &cli.StringSlice{"*"}, + Usage: "websocket origins", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/global-store/run_test.go b/cmd/swarm/global-store/run_test.go new file mode 100644 index 000000000..d7ef626e5 --- /dev/null +++ b/cmd/swarm/global-store/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-global-store", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-global-store", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 1e96d1294..3053ea1b3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -39,13 +39,16 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" "github.com/ethereum/go-ethereum/swarm/tracing" sv "github.com/ethereum/go-ethereum/swarm/version" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const clientIdentifier = "swarm" @@ -196,6 +199,7 @@ func init() { SwarmStorePath, SwarmStoreCapacity, SwarmStoreCacheCapacity, + SwarmGlobalStoreAPIFlag, } rpcFlags := []cli.Flag{ utils.WSEnabledFlag, @@ -324,8 +328,18 @@ func bzzd(ctx *cli.Context) error { func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) { //define the swarm service boot function boot := func(_ *node.ServiceContext) (node.Service, error) { - // In production, mockStore must be always nil. - return swarm.NewSwarm(bzzconfig, nil) + var nodeStore *mock.NodeStore + if bzzconfig.GlobalStoreAPI != "" { + // connect to global store + client, err := rpc.Dial(bzzconfig.GlobalStoreAPI) + if err != nil { + return nil, fmt.Errorf("global store: %v", err) + } + globalStore := mockrpc.NewGlobalStore(client) + // create a node store for this swarm key on global store + nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey)) + } + return swarm.NewSwarm(bzzconfig, nodeStore) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/swarm/api/config.go b/swarm/api/config.go index 54dd67ba8..b8de16f5f 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -71,6 +71,7 @@ type Config struct { SwapAPI string Cors string BzzAccount string + GlobalStoreAPI string privateKey *ecdsa.PrivateKey } |