From 32516c768ec09e2a71cab5983d2c8b8ae5d92fc7 Mon Sep 17 00:00:00 2001 From: holisticode Date: Mon, 11 Dec 2017 16:56:06 -0500 Subject: cmd/swarm: add config file (#15548) This commit adds a TOML configuration option to swarm. It reuses the TOML configuration structure used in geth with swarm customized items. The commit: * Adds a "dumpconfig" command to the swarm executable which allows printing the (default) configuration to stdout, which then can be redirected to a file in order to customize it. * Adds a "--config " option to the swarm executable which will allow to load a configuration file in TOML format from the specified location in order to initialize the Swarm node The override priorities are like follows: environment variables override command line arguments override config file override default config. --- swarm/api/config.go | 131 ++++++++++++-------------------- swarm/api/config_test.go | 116 ++++++++-------------------- swarm/network/hive.go | 12 ++- swarm/network/kademlia/kademlia.go | 2 +- swarm/network/kademlia/kademlia_test.go | 10 +-- swarm/network/syncer.go | 9 ++- swarm/services/swap/swap.go | 28 ++++--- swarm/storage/netstore.go | 10 ++- swarm/swarm.go | 12 ++- 9 files changed, 132 insertions(+), 198 deletions(-) (limited to 'swarm') diff --git a/swarm/api/config.go b/swarm/api/config.go index d8d25b1c8..140c938ae 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -18,15 +18,15 @@ package api import ( "crypto/ecdsa" - "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/services/swap" "github.com/ethereum/go-ethereum/swarm/storage" @@ -46,101 +46,68 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams *network.SyncParams - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - EnsRoot common.Address - NetworkId uint64 + Contract common.Address + EnsRoot common.Address + EnsApi string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + SwapApi string + Cors string + BzzAccount string + BootNodes string } -// config is agnostic to where private key is coming from -// so managing accounts is outside swarm and left to wrappers -func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, networkId uint64) (self *Config, err error) { - address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address - dirpath := filepath.Join(path, "bzz-"+common.Bytes2Hex(address.Bytes())) - err = os.MkdirAll(dirpath, os.ModePerm) - if err != nil { - return - } - confpath := filepath.Join(dirpath, "config.json") - var data []byte - pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) - pubkeyhex := common.ToHex(pubkey) - keyhex := crypto.Keccak256Hash(pubkey).Hex() +//create a default config with all parameters to set to defaults +func NewDefaultConfig() (self *Config) { self = &Config{ - SyncParams: network.NewSyncParams(dirpath), - HiveParams: network.NewHiveParams(dirpath), + StoreParams: storage.NewDefaultStoreParams(), ChunkerParams: storage.NewChunkerParams(), - StoreParams: storage.NewStoreParams(dirpath), + HiveParams: network.NewDefaultHiveParams(), + SyncParams: network.NewDefaultSyncParams(), + Swap: swap.NewDefaultSwapParams(), ListenAddr: DefaultHTTPListenAddr, Port: DefaultHTTPPort, - Path: dirpath, - Swap: swap.DefaultSwapParams(contract, prvKey), - PublicKey: pubkeyhex, - BzzKey: keyhex, + Path: node.DefaultDataDir(), + EnsApi: node.DefaultIPCEndpoint("geth"), EnsRoot: ens.TestNetAddress, - NetworkId: networkId, - } - data, err = ioutil.ReadFile(confpath) - - // if not set in function param, then set default for swarm network, will be overwritten by config file if present - if networkId == 0 { - self.NetworkId = network.NetworkId + NetworkId: network.NetworkId, + SwapEnabled: false, + SyncEnabled: true, + SwapApi: "", + BootNodes: "", } - if err != nil { - if !os.IsNotExist(err) { - return - } + return +} - // file does not exist - // write out config file - err = self.Save() - if err != nil { - err = fmt.Errorf("error writing config: %v", err) - } - return - } +//some config params need to be initialized after the complete +//config building phase is completed (e.g. due to overriding flags) +func (self *Config) Init(prvKey *ecdsa.PrivateKey) { - // file exists, deserialise - err = json.Unmarshal(data, self) + address := crypto.PubkeyToAddress(prvKey.PublicKey) + self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes())) + err := os.MkdirAll(self.Path, os.ModePerm) if err != nil { - return nil, fmt.Errorf("unable to parse config: %v", err) - } - // check public key - if pubkeyhex != self.PublicKey { - return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey) - } - if keyhex != self.BzzKey { - return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) - } - - // if set in function param, replace id set from config file - if networkId != 0 { - self.NetworkId = networkId + log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err)) + return } - self.Swap.SetKey(prvKey) - - if (self.EnsRoot == common.Address{}) { - self.EnsRoot = ens.TestNetAddress - } + pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) + pubkeyhex := common.ToHex(pubkey) + keyhex := crypto.Keccak256Hash(pubkey).Hex() - return -} + self.PublicKey = pubkeyhex + self.BzzKey = keyhex -func (self *Config) Save() error { - data, err := json.MarshalIndent(self, "", " ") - if err != nil { - return err - } - err = os.MkdirAll(self.Path, os.ModePerm) - if err != nil { - return err - } - confpath := filepath.Join(self.Path, "config.json") - return ioutil.WriteFile(confpath, data, os.ModePerm) + self.Swap.Init(self.Contract, prvKey) + self.SyncParams.Init(self.Path) + self.HiveParams.Init(self.Path) + self.StoreParams.Init(self.Path) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 85d056270..2e03a610e 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -17,109 +17,53 @@ package api import ( - "io/ioutil" - "os" - "path/filepath" - "strings" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) -var ( - hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" - defaultConfig = `{ - "ChunkDbPath": "` + filepath.Join("TMPDIR", "chunks") + `", - "DbCapacity": 5000000, - "CacheCapacity": 5000, - "Radius": 0, - "Branches": 128, - "Hash": "SHA3", - "CallInterval": 3000000000, - "KadDbPath": "` + filepath.Join("TMPDIR", "bzz-peers.json") + `", - "MaxProx": 8, - "ProxBinSize": 2, - "BucketSize": 4, - "PurgeInterval": 151200000000000, - "InitialRetryInterval": 42000000, - "MaxIdleInterval": 42000000000, - "ConnRetryExp": 2, - "Swap": { - "BuyAt": 20000000000, - "SellAt": 20000000000, - "PayAt": 100, - "DropAt": 10000, - "AutoCashInterval": 300000000000, - "AutoCashThreshold": 50000000000000, - "AutoDepositInterval": 300000000000, - "AutoDepositThreshold": 50000000000000, - "AutoDepositBuffer": 100000000000000, - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "Contract": "0x0000000000000000000000000000000000000000", - "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e" - }, - "RequestDbPath": "` + filepath.Join("TMPDIR", "requests") + `", - "RequestDbBatchSize": 512, - "KeyBufferSize": 1024, - "SyncBatchSize": 128, - "SyncBufferSize": 128, - "SyncCacheSize": 1024, - "SyncPriorities": [ - 2, - 1, - 1, - 0, - 0 - ], - "SyncModes": [ - true, - true, - true, - true, - false - ], - "Path": "TMPDIR", - "ListenAddr": "127.0.0.1", - "Port": "8500", - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", - "EnsRoot": "0x112234455c3a32fd11230c42e7bccd4a84e02010", - "NetworkId": 323 -}` -) +func TestConfig(t *testing.T) { -func TestConfigWriteRead(t *testing.T) { - tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) + var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" prvkey, err := crypto.HexToECDSA(hexprvkey) if err != nil { t.Fatalf("failed to load private key: %v", err) } - orig, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + + one := NewDefaultConfig() + two := NewDefaultConfig() + + if equal := reflect.DeepEqual(one, two); equal == false { + t.Fatal("Two default configs are not equal") } - data, err := ioutil.ReadFile(filepath.Join(orig.Path, "config.json")) - if err != nil { - t.Fatalf("default config file cannot be read: %v", err) + + one.Init(prvkey) + + //the init function should set the following fields + if one.BzzKey == "" { + t.Fatal("Expected BzzKey to be set") } - exp := strings.Replace(defaultConfig, "TMPDIR", orig.Path, -1) - exp = strings.Replace(exp, "\\", "\\\\", -1) - if string(data) != exp { - t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data)) + if one.PublicKey == "" { + t.Fatal("Expected PublicKey to be set") } - conf, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + //the Init function should append subdirs to the given path + if one.Swap.PayProfile.Beneficiary == (common.Address{}) { + t.Fatal("Failed to correctly initialize SwapParams") } - if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() { - t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex()) + + if one.SyncParams.RequestDbPath == one.Path { + t.Fatal("Failed to correctly initialize SyncParams") } + if one.HiveParams.KadDbPath == one.Path { + t.Fatal("Failed to correctly initialize HiveParams") + } + + if one.StoreParams.ChunkDbPath == one.Path { + t.Fatal("Failed to correctly initialize StoreParams") + } } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index d37b7e400..2504a4610 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -70,19 +70,25 @@ type HiveParams struct { *kademlia.KadParams } -func NewHiveParams(path string) *HiveParams { - kad := kademlia.NewKadParams() +//create default params +func NewDefaultHiveParams() *HiveParams { + kad := kademlia.NewDefaultKadParams() // kad.BucketSize = bucketSize // kad.MaxProx = maxProx // kad.ProxBinSize = proxBinSize return &HiveParams{ CallInterval: callInterval, - KadDbPath: filepath.Join(path, "bzz-peers.json"), KadParams: kad, } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *HiveParams) Init(path string) { + self.KadDbPath = filepath.Join(path, "bzz-peers.json") +} + func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive { kad := kademlia.New(kademlia.Address(addr), params.KadParams) return &Hive{ diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index bf976a3e1..0abc42a19 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -52,7 +52,7 @@ type KadParams struct { ConnRetryExp int } -func NewKadParams() *KadParams { +func NewDefaultKadParams() *KadParams { return &KadParams{ MaxProx: maxProx, ProxBinSize: proxBinSize, diff --git a/swarm/network/kademlia/kademlia_test.go b/swarm/network/kademlia/kademlia_test.go index 417ccecae..88858908a 100644 --- a/swarm/network/kademlia/kademlia_test.go +++ b/swarm/network/kademlia/kademlia_test.go @@ -63,7 +63,7 @@ func TestOn(t *testing.T) { if !ok1 || !ok2 { t.Errorf("oops") } - kad := New(addr, NewKadParams()) + kad := New(addr, NewDefaultKadParams()) err := kad.On(&testNode{addr: other}, nil) _ = err } @@ -72,7 +72,7 @@ func TestBootstrap(t *testing.T) { test := func(test *bootstrapTest) bool { // for any node kad.le, Target and N - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = test.MaxProx params.BucketSize = test.BucketSize params.ProxBinSize = test.BucketSize @@ -127,7 +127,7 @@ func TestFindClosest(t *testing.T) { test := func(test *FindClosestTest) bool { // for any node kad.le, Target and N - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(test.Self, params) var err error @@ -198,7 +198,7 @@ var ( func TestProxAdjust(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) self := gen(Address{}, r).(Address) - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(self, params) @@ -232,7 +232,7 @@ func TestSaveLoad(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) addresses := gen([]Address{}, r).([]Address) self := RandomAddress() - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(self, params) diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index d76af022c..6d729fcb9 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -131,9 +131,8 @@ type SyncParams struct { } // constructor with default values -func NewSyncParams(bzzdir string) *SyncParams { +func NewDefaultSyncParams() *SyncParams { return &SyncParams{ - RequestDbPath: filepath.Join(bzzdir, "requests"), RequestDbBatchSize: requestDbBatchSize, KeyBufferSize: keyBufferSize, SyncBufferSize: syncBufferSize, @@ -144,6 +143,12 @@ func NewSyncParams(bzzdir string) *SyncParams { } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *SyncParams) Init(path string) { + self.RequestDbPath = filepath.Join(path, "requests") +} + // syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding type syncer struct { *SyncParams // sync parameters diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index 093892e8d..1f9b22b90 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -80,17 +80,10 @@ type PayProfile struct { lock sync.RWMutex } -func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapParams { - pubkey := &prvkey.PublicKey +//create params with default values +func NewDefaultSwapParams() *SwapParams { return &SwapParams{ - PayProfile: &PayProfile{ - PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)), - Contract: contract, - Beneficiary: crypto.PubkeyToAddress(*pubkey), - privateKey: prvkey, - publicKey: pubkey, - owner: crypto.PubkeyToAddress(*pubkey), - }, + PayProfile: &PayProfile{}, Params: &swap.Params{ Profile: &swap.Profile{ BuyAt: buyAt, @@ -109,6 +102,21 @@ func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapP } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) { + pubkey := &prvkey.PublicKey + + self.PayProfile = &PayProfile{ + PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)), + Contract: contract, + Beneficiary: crypto.PubkeyToAddress(*pubkey), + privateKey: prvkey, + publicKey: pubkey, + owner: crypto.PubkeyToAddress(*pubkey), + } +} + // swap constructor, parameters // * global chequebook, assume deployed service and // * the balance is at buffer. diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 7b0612edc..5d4f17deb 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -57,15 +57,21 @@ type StoreParams struct { Radius int } -func NewStoreParams(path string) (self *StoreParams) { +//create params with default values +func NewDefaultStoreParams() (self *StoreParams) { return &StoreParams{ - ChunkDbPath: filepath.Join(path, "chunks"), DbCapacity: defaultDbCapacity, CacheCapacity: defaultCacheCapacity, Radius: defaultRadius, } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *StoreParams) Init(path string) { + self.ChunkDbPath = filepath.Join(path, "chunks") +} + // netstore contructor, takes path argument that is used to initialise dbStore, // the persistent (disk) storage component of LocalStore // the second argument is the hive, the connection/logistics manager for the node diff --git a/swarm/swarm.go b/swarm/swarm.go index 9db15325a..3be3660b5 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -220,7 +220,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { // stops all component services. func (self *Swarm) Stop() error { self.dpa.Stop() - self.hive.Stop() + err := self.hive.Stop() if ch := self.config.Swap.Chequebook(); ch != nil { ch.Stop() ch.Save() @@ -230,7 +230,7 @@ func (self *Swarm) Stop() error { self.lstore.DbStore.Close() } self.sfs.Stop() - return self.config.Save() + return err } // implements the node.Service interface @@ -301,7 +301,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error { return err } log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex())) - self.config.Save() self.hive.DropAll() return nil } @@ -314,10 +313,9 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { return } - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) - if err != nil { - return - } + config := api.NewDefaultConfig() + config.Path = datadir + config.Init(prvKey) config.Port = port dpa, err := storage.NewLocalDPA(datadir) -- cgit v1.2.3