diff options
41 files changed, 1039 insertions, 917 deletions
@@ -4,8 +4,8 @@ Official golang implementation of the Ethereum protocol | Linux | OSX | ARM | Windows | Tests ----------|---------|-----|-----|---------|------ -develop | [](https://build.ethdev.com/builders/Linux%20Go%20develop%20branch/builds/-1) | [](https://build.ethdev.com/builders/OSX%20Go%20develop%20branch/builds/-1) | [](https://build.ethdev.com/builders/ARM%20Go%20develop%20branch/builds/-1) | [](https://build.ethdev.com/builders/Windows%20Go%20develop%20branch/builds/-1) | [](https://travis-ci.org/ethereum/go-ethereum) [](http://codecov.io/github/ethereum/go-ethereum?branch=develop) -master | [](https://build.ethdev.com/builders/Linux%20Go%20master%20branch/builds/-1) | [](https://build.ethdev.com/builders/OSX%20Go%20master%20branch/builds/-1) | [](https://build.ethdev.com/builders/ARM%20Go%20master%20branch/builds/-1) | [](https://build.ethdev.com/builders/Windows%20Go%20master%20branch/builds/-1) | [](https://travis-ci.org/ethereum/go-ethereum) [](http://codecov.io/github/ethereum/go-ethereum?branch=master) +develop | [](https://build.ethdev.com/builders/Linux%20Go%20develop%20branch/builds/-1) | [](https://build.ethdev.com/builders/OSX%20Go%20develop%20branch/builds/-1) | [](https://build.ethdev.com/builders/ARM%20Go%20develop%20branch/builds/-1) | [](https://build.ethdev.com/builders/Windows%20Go%20develop%20branch/builds/-1) | [](https://travis-ci.org/ethereum/go-ethereum) [](https://codecov.io/github/ethereum/go-ethereum?branch=develop) +master | [](https://build.ethdev.com/builders/Linux%20Go%20master%20branch/builds/-1) | [](https://build.ethdev.com/builders/OSX%20Go%20master%20branch/builds/-1) | [](https://build.ethdev.com/builders/ARM%20Go%20master%20branch/builds/-1) | [](https://build.ethdev.com/builders/Windows%20Go%20master%20branch/builds/-1) | [](https://travis-ci.org/ethereum/go-ethereum) [](https://codecov.io/github/ethereum/go-ethereum?branch=master) [ branch. * [Docker](https://registry.hub.docker.com/u/ethereum/client-go/) -* [OS X](http://build.ethdev.com/builds/OSX%20Go%20develop%20branch/Mist-OSX-latest.dmg) +* [OS X](https://build.ethdev.com/builds/OSX%20Go%20develop%20branch/Mist-OSX-latest.dmg) * Ubuntu [trusty](https://build.ethdev.com/builds/Linux%20Go%20develop%20deb%20i386-trusty/latest/) | [utopic](https://build.ethdev.com/builds/Linux%20Go%20develop%20deb%20i386-utopic/latest/) @@ -283,9 +283,9 @@ for more details on configuring your environment, managing project dependencies ## License The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the -[GNU Lesser General Public License v3.0](http://www.gnu.org/licenses/lgpl-3.0.en.html), also +[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), also included in our repository in the `COPYING.LESSER` file. The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the -[GNU General Public License v3.0](http://www.gnu.org/licenses/gpl-3.0.en.html), also included +[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also included in our repository in the `COPYING` file. diff --git a/build/update-license.go b/build/update-license.go index f83a2b34b..96667be15 100644 --- a/build/update-license.go +++ b/build/update-license.go @@ -49,7 +49,6 @@ var ( // don't relicense vendored sources "crypto/sha3/", "crypto/ecies/", "logger/glog/", "crypto/secp256k1/curve.go", - "trie/arc.go", // don't license generated files "contracts/chequebook/contract/", "contracts/ens/contract/", diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 54984d6e0..553e5367c 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -79,7 +79,8 @@ func importChain(ctx *cli.Context) error { if ctx.GlobalBool(utils.TestNetFlag.Name) { state.StartingNonce = 1048576 // (2**20) } - chain, chainDb := utils.MakeChain(ctx) + stack := makeFullNode(ctx) + chain, chainDb := utils.MakeChain(ctx, stack) start := time.Now() err := utils.ImportChain(chain, ctx.Args().First()) chainDb.Close() @@ -94,7 +95,8 @@ func exportChain(ctx *cli.Context) error { if len(ctx.Args()) < 1 { utils.Fatalf("This command requires an argument.") } - chain, _ := utils.MakeChain(ctx) + stack := makeFullNode(ctx) + chain, _ := utils.MakeChain(ctx, stack) start := time.Now() var err error @@ -122,20 +124,25 @@ func exportChain(ctx *cli.Context) error { } func removeDB(ctx *cli.Context) error { - confirm, err := console.Stdin.PromptConfirm("Remove local database?") - if err != nil { - utils.Fatalf("%v", err) + stack := utils.MakeNode(ctx, clientIdentifier, gitCommit) + dbdir := stack.ResolvePath("chaindata") + if !common.FileExist(dbdir) { + fmt.Println(dbdir, "does not exist") + return nil } - if confirm { - fmt.Println("Removing chaindata...") + fmt.Println(dbdir) + confirm, err := console.Stdin.PromptConfirm("Remove this database?") + switch { + case err != nil: + utils.Fatalf("%v", err) + case !confirm: + fmt.Println("Operation aborted") + default: + fmt.Println("Removing...") start := time.Now() - - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "chaindata")) - + os.RemoveAll(dbdir) fmt.Printf("Removed in %v\n", time.Since(start)) - } else { - fmt.Println("Operation aborted") } return nil } @@ -143,7 +150,8 @@ func removeDB(ctx *cli.Context) error { func upgradeDB(ctx *cli.Context) error { glog.Infoln("Upgrading blockchain database") - chain, chainDb := utils.MakeChain(ctx) + stack := utils.MakeNode(ctx, clientIdentifier, gitCommit) + chain, chainDb := utils.MakeChain(ctx, stack) bcVersion := core.GetBlockChainVersion(chainDb) if bcVersion == 0 { bcVersion = core.BlockChainVersion @@ -156,10 +164,12 @@ func upgradeDB(ctx *cli.Context) error { utils.Fatalf("Unable to export chain for reimport %s", err) } chainDb.Close() - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "chaindata")) + if dir := dbDirectory(chainDb); dir != "" { + os.RemoveAll(dir) + } // Import the chain file. - chain, chainDb = utils.MakeChain(ctx) + chain, chainDb = utils.MakeChain(ctx, stack) core.WriteBlockChainVersion(chainDb, core.BlockChainVersion) err := utils.ImportChain(chain, exportFile) chainDb.Close() @@ -172,8 +182,17 @@ func upgradeDB(ctx *cli.Context) error { return nil } +func dbDirectory(db ethdb.Database) string { + ldb, ok := db.(*ethdb.LDBDatabase) + if !ok { + return "" + } + return ldb.Path() +} + func dump(ctx *cli.Context) error { - chain, chainDb := utils.MakeChain(ctx) + stack := makeFullNode(ctx) + chain, chainDb := utils.MakeChain(ctx, stack) for _, arg := range ctx.Args() { var block *types.Block if hashish(arg) { diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 92d6f7f86..066247303 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -107,7 +107,7 @@ func remoteConsole(ctx *cli.Context) error { utils.Fatalf("Unable to attach to remote geth: %v", err) } config := console.Config{ - DataDir: utils.MustMakeDataDir(ctx), + DataDir: utils.MakeDataDir(ctx), DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), @@ -135,7 +135,7 @@ func remoteConsole(ctx *cli.Context) error { // for "geth attach" and "geth monitor" with no argument. func dialRPC(endpoint string) (*rpc.Client, error) { if endpoint == "" { - endpoint = node.DefaultIPCEndpoint() + endpoint = node.DefaultIPCEndpoint(clientIdentifier) } else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { // Backwards compatibility with geth < 1.5 which required // these prefixes. diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 7058fb385..59730b17f 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -195,9 +195,9 @@ func testDAOForkBlockNewChain(t *testing.T, testnet bool, genesis string, votes geth.cmd.Wait() } // Retrieve the DAO config flag from the database - path := filepath.Join(datadir, "chaindata") + path := filepath.Join(datadir, "geth", "chaindata") if testnet && genesis == "" { - path = filepath.Join(datadir, "testnet", "chaindata") + path = filepath.Join(datadir, "testnet", "geth", "chaindata") } db, err := ethdb.NewLDBDatabase(path, 0, 0) if err != nil { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a7b332d0f..65311ca41 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -46,7 +45,7 @@ import ( ) const ( - clientIdentifier = "Geth" // Client identifier to advertise over the network + clientIdentifier = "geth" // Client identifier to advertise over the network ) var ( @@ -245,17 +244,15 @@ func initGenesis(ctx *cli.Context) error { state.StartingNonce = 1048576 // (2**20) } - chainDb, err := ethdb.NewLDBDatabase(filepath.Join(utils.MustMakeDataDir(ctx), "chaindata"), 0, 0) - if err != nil { - utils.Fatalf("could not open database: %v", err) - } + stack := makeFullNode(ctx) + chaindb := utils.MakeChainDatabase(ctx, stack) genesisFile, err := os.Open(genesisPath) if err != nil { utils.Fatalf("failed to read genesis file: %v", err) } - block, err := core.WriteGenesisBlock(chainDb, genesisFile) + block, err := core.WriteGenesisBlock(chaindb, genesisFile) if err != nil { utils.Fatalf("failed to write genesis block: %v", err) } @@ -296,9 +293,6 @@ func makeFullNode(ctx *cli.Context) *node.Node { // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the // miner. func startNode(ctx *cli.Context, stack *node.Node) { - // Report geth version - glog.V(logger.Info).Infof("instance: Geth/%s/%s/%s\n", utils.Version, runtime.Version(), runtime.GOOS) - // Start up the node itself utils.StartNode(stack) @@ -379,7 +373,7 @@ func gpubench(ctx *cli.Context) error { } func version(c *cli.Context) error { - fmt.Println(clientIdentifier) + fmt.Println(strings.Title(clientIdentifier)) fmt.Println("Version:", utils.Version) if gitCommit != "" { fmt.Println("Git Commit:", gitCommit) diff --git a/cmd/geth/monitorcmd.go b/cmd/geth/monitorcmd.go index d1490dce2..b74315dab 100644 --- a/cmd/geth/monitorcmd.go +++ b/cmd/geth/monitorcmd.go @@ -35,7 +35,7 @@ import ( var ( monitorCommandAttachFlag = cli.StringFlag{ Name: "attach", - Value: node.DefaultIPCEndpoint(), + Value: node.DefaultIPCEndpoint(clientIdentifier), Usage: "API endpoint to attach to", } monitorCommandRowsFlag = cli.IntFlag{ diff --git a/cmd/gethrpctest/main.go b/cmd/gethrpctest/main.go index d267dbf58..d0d6e1618 100644 --- a/cmd/gethrpctest/main.go +++ b/cmd/gethrpctest/main.go @@ -23,7 +23,6 @@ import ( "os" "os/signal" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -88,12 +87,12 @@ func MakeSystemNode(privkey string, test *tests.BlockTest) (*node.Node, error) { // Create a networkless protocol stack stack, err := node.New(&node.Config{ UseLightweightKDF: true, - IPCPath: node.DefaultIPCEndpoint(), - HTTPHost: common.DefaultHTTPHost, - HTTPPort: common.DefaultHTTPPort, + IPCPath: node.DefaultIPCEndpoint(""), + HTTPHost: node.DefaultHTTPHost, + HTTPPort: node.DefaultHTTPPort, HTTPModules: []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"}, - WSHost: common.DefaultWSHost, - WSPort: common.DefaultWSPort, + WSHost: node.DefaultWSHost, + WSPort: node.DefaultWSPort, WSModules: []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"}, NoDiscovery: true, }) diff --git a/cmd/utils/customflags.go b/cmd/utils/customflags.go index 5cbccfe98..11c92d451 100644 --- a/cmd/utils/customflags.go +++ b/cmd/utils/customflags.go @@ -137,9 +137,19 @@ func (self *DirectoryFlag) Set(value string) { // Note, it has limitations, e.g. ~someuser/tmp will not be expanded func expandPath(p string) string { if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { - if user, err := user.Current(); err == nil { - p = user.HomeDir + p[1:] + if home := homeDir(); home != "" { + p = home + p[1:] } } return path.Clean(os.ExpandEnv(p)) } + +func homeDir() string { + if home := os.Getenv("HOME"); home != "" { + return home + } + if usr, err := user.Current(); err == nil { + return usr.HomeDir + } + return "" +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3ab556a8f..0be499c5b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -105,7 +105,7 @@ var ( DataDirFlag = DirectoryFlag{ Name: "datadir", Usage: "Data directory for the databases and keystore", - Value: DirectoryString{common.DefaultDataDir()}, + Value: DirectoryString{node.DefaultDataDir()}, } KeyStoreDirFlag = DirectoryFlag{ Name: "keystore", @@ -139,7 +139,7 @@ var ( DocRootFlag = DirectoryFlag{ Name: "docroot", Usage: "Document Root for HTTPClient file scheme", - Value: DirectoryString{common.HomeDir()}, + Value: DirectoryString{homeDir()}, } CacheFlag = cli.IntFlag{ Name: "cache", @@ -245,12 +245,12 @@ var ( RPCListenAddrFlag = cli.StringFlag{ Name: "rpcaddr", Usage: "HTTP-RPC server listening interface", - Value: common.DefaultHTTPHost, + Value: node.DefaultHTTPHost, } RPCPortFlag = cli.IntFlag{ Name: "rpcport", Usage: "HTTP-RPC server listening port", - Value: common.DefaultHTTPPort, + Value: node.DefaultHTTPPort, } RPCCORSDomainFlag = cli.StringFlag{ Name: "rpccorsdomain", @@ -268,13 +268,13 @@ var ( } IPCApiFlag = cli.StringFlag{ Name: "ipcapi", - Usage: "API's offered over the IPC-RPC interface", + Usage: "APIs offered over the IPC-RPC interface", Value: rpc.DefaultIPCApis, } IPCPathFlag = DirectoryFlag{ Name: "ipcpath", Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)", - Value: DirectoryString{common.DefaultIPCSocket}, + Value: DirectoryString{"geth.ipc"}, } WSEnabledFlag = cli.BoolFlag{ Name: "ws", @@ -283,12 +283,12 @@ var ( WSListenAddrFlag = cli.StringFlag{ Name: "wsaddr", Usage: "WS-RPC server listening interface", - Value: common.DefaultWSHost, + Value: node.DefaultWSHost, } WSPortFlag = cli.IntFlag{ Name: "wsport", Usage: "WS-RPC server listening port", - Value: common.DefaultWSPort, + Value: node.DefaultWSPort, } WSApiFlag = cli.StringFlag{ Name: "wsapi", @@ -396,13 +396,14 @@ var ( } ) -// MustMakeDataDir retrieves the currently requested data directory, terminating +// MakeDataDir retrieves the currently requested data directory, terminating // if none (or the empty string) is specified. If the node is starting a testnet, // the a subdirectory of the specified datadir will be used. -func MustMakeDataDir(ctx *cli.Context) string { +func MakeDataDir(ctx *cli.Context) string { if path := ctx.GlobalString(DataDirFlag.Name); path != "" { + // TODO: choose a different location outside of the regular datadir. if ctx.GlobalBool(TestNetFlag.Name) { - return filepath.Join(path, "/testnet") + return filepath.Join(path, "testnet") } return path } @@ -447,16 +448,16 @@ func MakeNodeKey(ctx *cli.Context) *ecdsa.PrivateKey { return key } -// MakeNodeName creates a node name from a base set and the command line flags. -func MakeNodeName(client, version string, ctx *cli.Context) string { - name := common.MakeName(client, version) +// makeNodeUserIdent creates the user identifier from CLI flags. +func makeNodeUserIdent(ctx *cli.Context) string { + var comps []string if identity := ctx.GlobalString(IdentityFlag.Name); len(identity) > 0 { - name += "/" + identity + comps = append(comps, identity) } if ctx.GlobalBool(VMEnableJitFlag.Name) { - name += "/JIT" + comps = append(comps, "JIT") } - return name + return strings.Join(comps, "/") } // MakeBootstrapNodes creates a list of bootstrap nodes from the command line @@ -612,11 +613,13 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node { } config := &node.Config{ - DataDir: MustMakeDataDir(ctx), + DataDir: MakeDataDir(ctx), KeyStoreDir: ctx.GlobalString(KeyStoreDirFlag.Name), UseLightweightKDF: ctx.GlobalBool(LightKDFFlag.Name), PrivateKey: MakeNodeKey(ctx), - Name: MakeNodeName(name, vsn, ctx), + Name: name, + Version: vsn, + UserIdent: makeNodeUserIdent(ctx), NoDiscovery: ctx.GlobalBool(NoDiscoverFlag.Name), BootstrapNodes: MakeBootstrapNodes(ctx), ListenAddr: MakeListenAddress(ctx), @@ -674,7 +677,7 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) { ethConf := ð.Config{ Etherbase: MakeEtherbase(stack.AccountManager(), ctx), - ChainConfig: MustMakeChainConfig(ctx), + ChainConfig: MakeChainConfig(ctx, stack), FastSync: ctx.GlobalBool(FastSyncFlag.Name), DatabaseCache: ctx.GlobalInt(CacheFlag.Name), DatabaseHandles: MakeDatabaseHandles(), @@ -748,16 +751,16 @@ func SetupNetwork(ctx *cli.Context) { params.TargetGasLimit = common.String2Big(ctx.GlobalString(TargetGasLimitFlag.Name)) } -// MustMakeChainConfig reads the chain configuration from the database in ctx.Datadir. -func MustMakeChainConfig(ctx *cli.Context) *core.ChainConfig { - db := MakeChainDatabase(ctx) +// MakeChainConfig reads the chain configuration from the database in ctx.Datadir. +func MakeChainConfig(ctx *cli.Context, stack *node.Node) *core.ChainConfig { + db := MakeChainDatabase(ctx, stack) defer db.Close() - return MustMakeChainConfigFromDb(ctx, db) + return MakeChainConfigFromDb(ctx, db) } -// MustMakeChainConfigFromDb reads the chain configuration from the given database. -func MustMakeChainConfigFromDb(ctx *cli.Context, db ethdb.Database) *core.ChainConfig { +// MakeChainConfigFromDb reads the chain configuration from the given database. +func MakeChainConfigFromDb(ctx *cli.Context, db ethdb.Database) *core.ChainConfig { // If the chain is already initialized, use any existing chain configs config := new(core.ChainConfig) @@ -800,14 +803,13 @@ func MustMakeChainConfigFromDb(ctx *cli.Context, db ethdb.Database) *core.ChainC } // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. -func MakeChainDatabase(ctx *cli.Context) ethdb.Database { +func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { var ( - datadir = MustMakeDataDir(ctx) cache = ctx.GlobalInt(CacheFlag.Name) handles = MakeDatabaseHandles() ) - chainDb, err := ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache, handles) + chainDb, err := stack.OpenDatabase("chaindata", cache, handles) if err != nil { Fatalf("Could not open database: %v", err) } @@ -815,9 +817,9 @@ func MakeChainDatabase(ctx *cli.Context) ethdb.Database { } // MakeChain creates a chain manager from set command line flags. -func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) { +func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) { var err error - chainDb = MakeChainDatabase(ctx) + chainDb = MakeChainDatabase(ctx, stack) if ctx.GlobalBool(OlympicFlag.Name) { _, err := core.WriteTestNetGenesisBlock(chainDb) @@ -825,7 +827,7 @@ func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database glog.Fatalln(err) } } - chainConfig := MustMakeChainConfigFromDb(ctx, chainDb) + chainConfig := MakeChainConfigFromDb(ctx, chainDb) pow := pow.PoW(core.FakePow{}) if !ctx.GlobalBool(FakePoWFlag.Name) { diff --git a/common/path.go b/common/path.go index cbcd13c4f..bd8da86e7 100644 --- a/common/path.go +++ b/common/path.go @@ -19,10 +19,8 @@ package common import ( "fmt" "os" - "os/user" "path/filepath" "runtime" - "strings" ) // MakeName creates a node name that follows the ethereum convention @@ -32,21 +30,6 @@ func MakeName(name, version string) string { return fmt.Sprintf("%s/v%s/%s/%s", name, version, runtime.GOOS, runtime.Version()) } -func ExpandHomePath(p string) (path string) { - path = p - sep := string(os.PathSeparator) - - // Check in case of paths like "/something/~/something/" - if len(p) > 1 && p[:1+len(sep)] == "~"+sep { - usr, _ := user.Current() - dir := usr.HomeDir - - path = strings.Replace(p, "~", dir, 1) - } - - return -} - func FileExist(filePath string) bool { _, err := os.Stat(filePath) if err != nil && os.IsNotExist(err) { @@ -62,13 +45,3 @@ func AbsolutePath(Datadir string, filename string) string { } return filepath.Join(Datadir, filename) } - -func HomeDir() string { - if home := os.Getenv("HOME"); home != "" { - return home - } - if usr, err := user.Current(); err == nil { - return usr.HomeDir - } - return "" -} diff --git a/core/blockchain.go b/core/blockchain.go index a5f146a2d..1fbcdfc6f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -357,7 +357,12 @@ func (self *BlockChain) AuxValidator() pow.PoW { return self.pow } // State returns a new mutable state based on the current HEAD block. func (self *BlockChain) State() (*state.StateDB, error) { - return state.New(self.CurrentBlock().Root(), self.chainDb) + return self.StateAt(self.CurrentBlock().Root()) +} + +// StateAt returns a new mutable state based on a particular point in time. +func (self *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { + return self.stateCache.New(root) } // Reset purges the entire blockchain, restoring it to its genesis state. diff --git a/core/state/iterator.go b/core/state/iterator.go index 9d8a69b7c..14265b277 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -76,7 +76,7 @@ func (it *NodeIterator) step() error { } // Initialize the iterator if we've just started if it.stateIt == nil { - it.stateIt = trie.NewNodeIterator(it.state.trie.Trie) + it.stateIt = it.state.trie.NodeIterator() } // If we had data nodes previously, we surely have at least state nodes if it.dataIt != nil { diff --git a/core/state/state_object.go b/core/state/state_object.go index 3496008a6..a54620d55 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -95,8 +95,6 @@ type Account struct { Balance *big.Int Root common.Hash // merkle root of the storage trie CodeHash []byte - - codeSize *int } // NewObject creates a state object. @@ -275,20 +273,9 @@ func (self *StateObject) Code(db trie.Database) []byte { return code } -// CodeSize returns the size of the contract code associated with this object. -func (self *StateObject) CodeSize(db trie.Database) int { - if self.data.codeSize == nil { - self.data.codeSize = new(int) - *self.data.codeSize = len(self.Code(db)) - } - return *self.data.codeSize -} - func (self *StateObject) SetCode(code []byte) { self.code = code self.data.CodeHash = crypto.Keccak256(code) - self.data.codeSize = new(int) - *self.data.codeSize = len(code) self.dirtyCode = true if self.onDirty != nil { self.onDirty(self.Address()) diff --git a/core/state/statedb.go b/core/state/statedb.go index 10f3f4652..5c51e3b59 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -20,6 +20,7 @@ package state import ( "fmt" "math/big" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/vm" @@ -28,23 +29,32 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + lru "github.com/hashicorp/golang-lru" ) // The starting nonce determines the default nonce when new accounts are being // created. var StartingNonce uint64 +const ( + // Number of past tries to keep. The arbitrarily chosen value here + // is max uncle depth + 1. + maxJournalLength = 8 + + // Number of codehash->size associations to keep. + codeSizeCacheSize = 100000 +) + // StateDBs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: // * Contracts // * Accounts type StateDB struct { - db ethdb.Database - trie *trie.SecureTrie - - // This map caches canon state accounts. - all map[common.Address]Account + db ethdb.Database + trie *trie.SecureTrie + pastTries []*trie.SecureTrie + codeSizeCache *lru.Cache // This map holds 'live' objects, which will get modified while processing a state transition. stateObjects map[common.Address]*StateObject @@ -57,6 +67,8 @@ type StateDB struct { txIndex int logs map[common.Hash]vm.Logs logSize uint + + lock sync.Mutex } // Create a new state from a given trie @@ -65,10 +77,11 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { if err != nil { return nil, err } + csc, _ := lru.New(codeSizeCacheSize) return &StateDB{ db: db, trie: tr, - all: make(map[common.Address]Account), + codeSizeCache: csc, stateObjects: make(map[common.Address]*StateObject), stateObjectsDirty: make(map[common.Address]struct{}), refund: new(big.Int), @@ -76,30 +89,74 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { }, nil } -// Reset clears out all emphemeral state objects from the state db, but keeps -// the underlying state trie to avoid reloading data for the next operations. -func (self *StateDB) Reset(root common.Hash) error { - tr, err := trie.NewSecure(root, self.db) +// New creates a new statedb by reusing any journalled tries to avoid costly +// disk io. +func (self *StateDB) New(root common.Hash) (*StateDB, error) { + self.lock.Lock() + defer self.lock.Unlock() + + tr, err := self.openTrie(root) if err != nil { - return err - } - all := self.all - if self.trie.Hash() != root { - // The root has changed, invalidate canon state. - all = make(map[common.Address]Account) + return nil, err } - *self = StateDB{ + return &StateDB{ db: self.db, trie: tr, - all: all, + codeSizeCache: self.codeSizeCache, stateObjects: make(map[common.Address]*StateObject), stateObjectsDirty: make(map[common.Address]struct{}), refund: new(big.Int), logs: make(map[common.Hash]vm.Logs), + }, nil +} + +// Reset clears out all emphemeral state objects from the state db, but keeps +// the underlying state trie to avoid reloading data for the next operations. +func (self *StateDB) Reset(root common.Hash) error { + self.lock.Lock() + defer self.lock.Unlock() + + tr, err := self.openTrie(root) + if err != nil { + return err } + self.trie = tr + self.stateObjects = make(map[common.Address]*StateObject) + self.stateObjectsDirty = make(map[common.Address]struct{}) + self.refund = new(big.Int) + self.thash = common.Hash{} + self.bhash = common.Hash{} + self.txIndex = 0 + self.logs = make(map[common.Hash]vm.Logs) + self.logSize = 0 + return nil } +// openTrie creates a trie. It uses an existing trie if one is available +// from the journal if available. +func (self *StateDB) openTrie(root common.Hash) (*trie.SecureTrie, error) { + for i := len(self.pastTries) - 1; i >= 0; i-- { + if self.pastTries[i].Hash() == root { + tr := *self.pastTries[i] + return &tr, nil + } + } + return trie.NewSecure(root, self.db) +} + +func (self *StateDB) pushTrie(t *trie.SecureTrie) { + self.lock.Lock() + defer self.lock.Unlock() + + if len(self.pastTries) >= maxJournalLength { + copy(self.pastTries, self.pastTries[1:]) + self.pastTries[len(self.pastTries)-1] = t + } else { + self.pastTries = append(self.pastTries, t) + } +} + func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) { self.thash = thash self.bhash = bhash @@ -165,17 +222,28 @@ func (self *StateDB) GetNonce(addr common.Address) uint64 { func (self *StateDB) GetCode(addr common.Address) []byte { stateObject := self.GetStateObject(addr) if stateObject != nil { - return stateObject.Code(self.db) + code := stateObject.Code(self.db) + key := common.BytesToHash(stateObject.CodeHash()) + self.codeSizeCache.Add(key, len(code)) + return code } return nil } func (self *StateDB) GetCodeSize(addr common.Address) int { stateObject := self.GetStateObject(addr) - if stateObject != nil { - return stateObject.CodeSize(self.db) + if stateObject == nil { + return 0 } - return 0 + key := common.BytesToHash(stateObject.CodeHash()) + if cached, ok := self.codeSizeCache.Get(key); ok { + return cached.(int) + } + size := len(stateObject.Code(self.db)) + if stateObject.dbErr == nil { + self.codeSizeCache.Add(key, size) + } + return size } func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash { @@ -269,13 +337,6 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje return obj } - // Use cached account data from the canon state if possible. - if data, ok := self.all[addr]; ok { - obj := NewObject(addr, data, self.MarkStateObjectDirty) - self.SetStateObject(obj) - return obj - } - // Load the object from the database. enc := self.trie.Get(addr[:]) if len(enc) == 0 { @@ -286,10 +347,6 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje glog.Errorf("can't decode object at %x: %v", addr[:], err) return nil } - // Update the all cache. Content in DB always corresponds - // to the current head state so this is ok to do here. - // The object we just loaded has no storage trie and code yet. - self.all[addr] = data // Insert into the live set. obj := NewObject(addr, data, self.MarkStateObjectDirty) self.SetStateObject(obj) @@ -351,11 +408,15 @@ func (self *StateDB) CreateAccount(addr common.Address) vm.Account { // func (self *StateDB) Copy() *StateDB { + self.lock.Lock() + defer self.lock.Unlock() + // Copy all the basic fields, initialize the memory ones state := &StateDB{ db: self.db, trie: self.trie, - all: self.all, + pastTries: self.pastTries, + codeSizeCache: self.codeSizeCache, stateObjects: make(map[common.Address]*StateObject, len(self.stateObjectsDirty)), stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)), refund: new(big.Int).Set(self.refund), @@ -375,11 +436,15 @@ func (self *StateDB) Copy() *StateDB { } func (self *StateDB) Set(state *StateDB) { + self.lock.Lock() + defer self.lock.Unlock() + + self.db = state.db self.trie = state.trie + self.pastTries = state.pastTries self.stateObjects = state.stateObjects self.stateObjectsDirty = state.stateObjectsDirty - self.all = state.all - + self.codeSizeCache = state.codeSizeCache self.refund = state.refund self.logs = state.logs self.logSize = state.logSize @@ -444,12 +509,6 @@ func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) { func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) { s.refund = new(big.Int) - defer func() { - if err != nil { - // Committing failed, any updates to the canon state are invalid. - s.all = make(map[common.Address]Account) - } - }() // Commit objects to the trie. for addr, stateObject := range s.stateObjects { @@ -457,7 +516,6 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) // If the object has been removed, don't bother syncing it // and just mark it for deletion in the trie. s.DeleteStateObject(stateObject) - delete(s.all, addr) } else if _, ok := s.stateObjectsDirty[addr]; ok { // Write any contract code associated with the state object if stateObject.code != nil && stateObject.dirtyCode { @@ -472,12 +530,15 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) } // Update the object in the main account trie. s.UpdateStateObject(stateObject) - s.all[addr] = stateObject.data } delete(s.stateObjectsDirty, addr) } // Write trie changes. - return s.trie.CommitTo(dbw) + root, err = s.trie.CommitTo(dbw) + if err == nil { + s.pushTrie(s.trie) + } + return root, err } func (self *StateDB) Refunds() *big.Int { diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 715645c6c..c768781a4 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -62,9 +62,6 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { } root, _ := state.Commit() - // Remove any potentially cached data from the test state creation - trie.ClearGlobalCache() - // Return the generated state return db, root, accounts } @@ -72,9 +69,6 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { // checkStateAccounts cross references a reconstructed state with an expected // account array. func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { - // Remove any potentially cached data from the state synchronisation - trie.ClearGlobalCache() - // Check root availability and state contents state, err := New(root, db) if err != nil { @@ -98,9 +92,6 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou // checkStateConsistency checks that all nodes in a state trie are indeed present. func checkStateConsistency(db ethdb.Database, root common.Hash) error { - // Remove any potentially cached data from the test state creation or previous checks - trie.ClearGlobalCache() - // Create and iterate a state trie rooted in a sub-node if _, err := db.Get(root.Bytes()); err != nil { return nil // Consider a non existent state consistent diff --git a/eth/api.go b/eth/api.go index d6c0826ed..c2fdbe99c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -293,7 +293,7 @@ func (api *PublicDebugAPI) DumpBlock(number uint64) (state.Dump, error) { if block == nil { return state.Dump{}, fmt.Errorf("block #%d not found", number) } - stateDb, err := state.New(block.Root(), api.eth.ChainDb()) + stateDb, err := api.eth.BlockChain().StateAt(block.Root()) if err != nil { return state.Dump{}, err } @@ -406,7 +406,7 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, logConfig *vm.LogConf if err := core.ValidateHeader(api.config, blockchain.AuxValidator(), block.Header(), blockchain.GetHeader(block.ParentHash(), block.NumberU64()-1), true, false); err != nil { return false, structLogger.StructLogs(), err } - statedb, err := state.New(blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), api.eth.ChainDb()) + statedb, err := blockchain.StateAt(blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1).Root()) if err != nil { return false, structLogger.StructLogs(), err } @@ -501,7 +501,7 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common. if parent == nil { return nil, fmt.Errorf("block parent %x not found", block.ParentHash()) } - stateDb, err := state.New(parent.Root(), api.eth.ChainDb()) + stateDb, err := api.eth.BlockChain().StateAt(parent.Root()) if err != nil { return nil, err } diff --git a/eth/api_backend.go b/eth/api_backend.go index 4f8f06529..4adeb0aa0 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -81,7 +81,7 @@ func (b *EthApiBackend) StateAndHeaderByNumber(blockNr rpc.BlockNumber) (ethapi. if header == nil { return nil, nil, nil } - stateDb, err := state.New(header.Root, b.eth.chainDb) + stateDb, err := b.eth.BlockChain().StateAt(header.Root) return EthApiState{stateDb}, header, err } diff --git a/ethdb/database.go b/ethdb/database.go index f93731cfe..479c54b60 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" @@ -84,6 +85,7 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { OpenFilesCacheCapacity: handles, BlockCacheCapacity: cache / 2 * opt.MiB, WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally + Filter: filter.NewBloomFilter(10), }) if _, corrupted := err.(*errors.ErrCorrupted); corrupted { db, err = leveldb.RecoverFile(file, nil) @@ -98,6 +100,11 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { }, nil } +// Path returns the path to the database directory. +func (db *LDBDatabase) Path() string { + return db.fn +} + // Put puts the given key / value to the queue func (self *LDBDatabase) Put(key []byte, value []byte) error { // Measure the database put latency, if requested diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 6480085dd..9a97be25f 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -454,6 +454,8 @@ type CallArgs struct { } func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (string, *big.Int, error) { + defer func(start time.Time) { glog.V(logger.Debug).Infof("call took %v", time.Since(start)) }(time.Now()) + state, header, err := s.b.StateAndHeaderByNumber(blockNr) if state == nil || err != nil { return "0x", common.Big0, err diff --git a/light/state_test.go b/light/state_test.go index 90c38604a..d7014a2dc 100644 --- a/light/state_test.go +++ b/light/state_test.go @@ -42,7 +42,6 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { case *TrieRequest: t, _ := trie.New(req.root, odr.sdb) req.proof = t.Prove(req.key) - trie.ClearGlobalCache() case *NodeDataRequest: req.data, _ = odr.sdb.Get(req.hash[:]) } @@ -75,7 +74,6 @@ func TestLightStateOdr(t *testing.T) { odr := &testOdr{sdb: sdb, ldb: ldb} ls := NewLightState(root, odr) ctx := context.Background() - trie.ClearGlobalCache() for i := byte(0); i < 100; i++ { addr := common.Address{i} @@ -160,7 +158,6 @@ func TestLightStateSetCopy(t *testing.T) { odr := &testOdr{sdb: sdb, ldb: ldb} ls := NewLightState(root, odr) ctx := context.Background() - trie.ClearGlobalCache() for i := byte(0); i < 100; i++ { addr := common.Address{i} @@ -237,7 +234,6 @@ func TestLightStateDelete(t *testing.T) { odr := &testOdr{sdb: sdb, ldb: ldb} ls := NewLightState(root, odr) ctx := context.Background() - trie.ClearGlobalCache() addr := common.Address{42} diff --git a/miner/worker.go b/miner/worker.go index 1676036d8..ac1ef5ba3 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -361,7 +361,7 @@ func (self *worker) push(work *Work) { // makeCurrent creates a new environment for the current cycle. func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error { - state, err := state.New(parent.Root(), self.eth.ChainDb()) + state, err := self.chain.StateAt(parent.Root()) if err != nil { return err } diff --git a/node/api.go b/node/api.go index 3523874ab..631e92c8e 100644 --- a/node/api.go +++ b/node/api.go @@ -84,17 +84,17 @@ func (api *PrivateAdminAPI) StartRPC(host *string, port *rpc.HexNumber, cors *st } if host == nil { - h := common.DefaultHTTPHost - if api.node.httpHost != "" { - h = api.node.httpHost + h := DefaultHTTPHost + if api.node.config.HTTPHost != "" { + h = api.node.config.HTTPHost } host = &h } if port == nil { - port = rpc.NewHexNumber(api.node.httpPort) + port = rpc.NewHexNumber(api.node.config.HTTPPort) } if cors == nil { - cors = &api.node.httpCors + cors = &api.node.config.HTTPCors } modules := api.node.httpWhitelist @@ -133,20 +133,20 @@ func (api *PrivateAdminAPI) StartWS(host *string, port *rpc.HexNumber, allowedOr } if host == nil { - h := common.DefaultWSHost - if api.node.wsHost != "" { - h = api.node.wsHost + h := DefaultWSHost + if api.node.config.WSHost != "" { + h = api.node.config.WSHost } host = &h } if port == nil { - port = rpc.NewHexNumber(api.node.wsPort) + port = rpc.NewHexNumber(api.node.config.WSPort) } if allowedOrigins == nil { - allowedOrigins = &api.node.wsOrigins + allowedOrigins = &api.node.config.WSOrigins } - modules := api.node.wsWhitelist + modules := api.node.config.WSModules if apis != nil { modules = nil for _, m := range strings.Split(*apis, ",") { diff --git a/node/config.go b/node/config.go index 432da7015..15884a12e 100644 --- a/node/config.go +++ b/node/config.go @@ -18,7 +18,6 @@ package node import ( "crypto/ecdsa" - "encoding/json" "fmt" "io/ioutil" "net" @@ -48,6 +47,18 @@ var ( // P2P network layer of a protocol stack. These values can be further extended by // all registered services. type Config struct { + // Name sets the instance name of the node. It must not contain the / character and is + // used in the devp2p node identifier. The instance name of geth is "geth". If no + // value is specified, the basename of the current executable is used. + Name string + + // UserIdent, if set, is used as an additional component in the devp2p node identifier. + UserIdent string + + // Version should be set to the version number of the program. It is used + // in the devp2p node identifier. + Version string + // DataDir is the file system folder the node should use for any data storage // requirements. The configured data directory will not be directly shared with // registered services, instead those can use utility methods to create/access @@ -80,10 +91,6 @@ type Config struct { // needed. PrivateKey *ecdsa.PrivateKey - // Name sets the node name of this server. Use common.MakeName to create a name - // that follows existing conventions. - Name string - // NoDiscovery specifies whether the peer discovery mechanism should be started // or not. Disabling is usually useful for protocol debugging (manual topology). NoDiscovery bool @@ -178,9 +185,23 @@ func (c *Config) IPCEndpoint() string { return c.IPCPath } +// NodeDB returns the path to the discovery node database. +func (c *Config) NodeDB() string { + if c.DataDir == "" { + return "" // ephemeral + } + return c.resolvePath("nodes") +} + // DefaultIPCEndpoint returns the IPC path used by default. -func DefaultIPCEndpoint() string { - config := &Config{DataDir: common.DefaultDataDir(), IPCPath: common.DefaultIPCSocket} +func DefaultIPCEndpoint(clientIdentifier string) string { + if clientIdentifier == "" { + clientIdentifier = strings.TrimSuffix(filepath.Base(os.Args[0]), ".exe") + if clientIdentifier == "" { + panic("empty executable name") + } + } + config := &Config{DataDir: DefaultDataDir(), IPCPath: clientIdentifier + ".ipc"} return config.IPCEndpoint() } @@ -195,7 +216,7 @@ func (c *Config) HTTPEndpoint() string { // DefaultHTTPEndpoint returns the HTTP endpoint used by default. func DefaultHTTPEndpoint() string { - config := &Config{HTTPHost: common.DefaultHTTPHost, HTTPPort: common.DefaultHTTPPort} + config := &Config{HTTPHost: DefaultHTTPHost, HTTPPort: DefaultHTTPPort} return config.HTTPEndpoint() } @@ -210,19 +231,80 @@ func (c *Config) WSEndpoint() string { // DefaultWSEndpoint returns the websocket endpoint used by default. func DefaultWSEndpoint() string { - config := &Config{WSHost: common.DefaultWSHost, WSPort: common.DefaultWSPort} + config := &Config{WSHost: DefaultWSHost, WSPort: DefaultWSPort} return config.WSEndpoint() } +// NodeName returns the devp2p node identifier. +func (c *Config) NodeName() string { + name := c.name() + // Backwards compatibility: previous versions used title-cased "Geth", keep that. + if name == "geth" || name == "geth-testnet" { + name = "Geth" + } + if c.UserIdent != "" { + name += "/" + c.UserIdent + } + if c.Version != "" { + name += "/v" + c.Version + } + name += "/" + runtime.GOOS + name += "/" + runtime.Version() + return name +} + +func (c *Config) name() string { + if c.Name == "" { + progname := strings.TrimSuffix(filepath.Base(os.Args[0]), ".exe") + if progname == "" { + panic("empty executable name, set Config.Name") + } + return progname + } + return c.Name +} + +// These resources are resolved differently for the "geth" and "geth-testnet" instances. +var isOldGethResource = map[string]bool{ + "chaindata": true, + "nodes": true, + "nodekey": true, + "static-nodes.json": true, + "trusted-nodes.json": true, +} + +// resolvePath resolves path in the instance directory. +func (c *Config) resolvePath(path string) string { + if filepath.IsAbs(path) { + return path + } + if c.DataDir == "" { + return "" + } + // Backwards-compatibility: ensure that data directory files created + // by geth 1.4 are used if they exist. + if c.name() == "geth" && isOldGethResource[path] { + oldpath := "" + if c.Name == "geth" { + oldpath = filepath.Join(c.DataDir, path) + } + if oldpath != "" && common.FileExist(oldpath) { + // TODO: print warning + return oldpath + } + } + return filepath.Join(c.DataDir, c.name(), path) +} + // NodeKey retrieves the currently configured private key of the node, checking // first any manually set key, falling back to the one found in the configured // data folder. If no key can be found, a new one is generated. func (c *Config) NodeKey() *ecdsa.PrivateKey { - // Use any specifically configured key + // Use any specifically configured key. if c.PrivateKey != nil { return c.PrivateKey } - // Generate ephemeral key if no datadir is being used + // Generate ephemeral key if no datadir is being used. if c.DataDir == "" { key, err := crypto.GenerateKey() if err != nil { @@ -230,16 +312,22 @@ func (c *Config) NodeKey() *ecdsa.PrivateKey { } return key } - // Fall back to persistent key from the data directory - keyfile := filepath.Join(c.DataDir, datadirPrivateKey) + + keyfile := c.resolvePath(datadirPrivateKey) if key, err := crypto.LoadECDSA(keyfile); err == nil { return key } - // No persistent key found, generate and store a new one + // No persistent key found, generate and store a new one. key, err := crypto.GenerateKey() if err != nil { glog.Fatalf("Failed to generate node key: %v", err) } + instanceDir := filepath.Join(c.DataDir, c.name()) + if err := os.MkdirAll(instanceDir, 0700); err != nil { + glog.V(logger.Error).Infof("Failed to persist node key: %v", err) + return key + } + keyfile = filepath.Join(instanceDir, datadirPrivateKey) if err := crypto.SaveECDSA(keyfile, key); err != nil { glog.V(logger.Error).Infof("Failed to persist node key: %v", err) } @@ -248,12 +336,12 @@ func (c *Config) NodeKey() *ecdsa.PrivateKey { // StaticNodes returns a list of node enode URLs configured as static nodes. func (c *Config) StaticNodes() []*discover.Node { - return c.parsePersistentNodes(datadirStaticNodes) + return c.parsePersistentNodes(c.resolvePath(datadirStaticNodes)) } // TrusterNodes returns a list of node enode URLs configured as trusted nodes. func (c *Config) TrusterNodes() []*discover.Node { - return c.parsePersistentNodes(datadirTrustedNodes) + return c.parsePersistentNodes(c.resolvePath(datadirTrustedNodes)) } // parsePersistentNodes parses a list of discovery node URLs loaded from a .json @@ -267,15 +355,10 @@ func (c *Config) parsePersistentNodes(file string) []*discover.Node { if _, err := os.Stat(path); err != nil { return nil } - // Load the nodes from the config file - blob, err := ioutil.ReadFile(path) - if err != nil { - glog.V(logger.Error).Infof("Failed to access nodes: %v", err) - return nil - } - nodelist := []string{} - if err := json.Unmarshal(blob, &nodelist); err != nil { - glog.V(logger.Error).Infof("Failed to load nodes: %v", err) + // Load the nodes from the config file. + var nodelist []string + if err := common.LoadJSON(path, &nodelist); err != nil { + glog.V(logger.Error).Infof("Can't load node file %s: %v", path, err) return nil } // Interpret the list as a discovery node array diff --git a/node/config_test.go b/node/config_test.go index 45a54d184..b258d2a8b 100644 --- a/node/config_test.go +++ b/node/config_test.go @@ -96,57 +96,55 @@ func TestIPCPathResolution(t *testing.T) { // ephemeral. func TestNodeKeyPersistency(t *testing.T) { // Create a temporary folder and make sure no key is present - dir, err := ioutil.TempDir("", "") + dir, err := ioutil.TempDir("", "node-test") if err != nil { t.Fatalf("failed to create temporary data directory: %v", err) } defer os.RemoveAll(dir) - if _, err := os.Stat(filepath.Join(dir, datadirPrivateKey)); err == nil { - t.Fatalf("non-created node key already exists") - } + keyfile := filepath.Join(dir, "unit-test", datadirPrivateKey) + // Configure a node with a preset key and ensure it's not persisted key, err := crypto.GenerateKey() if err != nil { t.Fatalf("failed to generate one-shot node key: %v", err) } - if _, err := New(&Config{DataDir: dir, PrivateKey: key}); err != nil { - t.Fatalf("failed to create empty stack: %v", err) - } - if _, err := os.Stat(filepath.Join(dir, datadirPrivateKey)); err == nil { + config := &Config{Name: "unit-test", DataDir: dir, PrivateKey: key} + config.NodeKey() + if _, err := os.Stat(filepath.Join(keyfile)); err == nil { t.Fatalf("one-shot node key persisted to data directory") } + // Configure a node with no preset key and ensure it is persisted this time - if _, err := New(&Config{DataDir: dir}); err != nil { - t.Fatalf("failed to create newly keyed stack: %v", err) - } - if _, err := os.Stat(filepath.Join(dir, datadirPrivateKey)); err != nil { + config = &Config{Name: "unit-test", DataDir: dir} + config.NodeKey() + if _, err := os.Stat(keyfile); err != nil { t.Fatalf("node key not persisted to data directory: %v", err) } - key, err = crypto.LoadECDSA(filepath.Join(dir, datadirPrivateKey)) + key, err = crypto.LoadECDSA(keyfile) if err != nil { t.Fatalf("failed to load freshly persisted node key: %v", err) } - blob1, err := ioutil.ReadFile(filepath.Join(dir, datadirPrivateKey)) + blob1, err := ioutil.ReadFile(keyfile) if err != nil { t.Fatalf("failed to read freshly persisted node key: %v", err) } + // Configure a new node and ensure the previously persisted key is loaded - if _, err := New(&Config{DataDir: dir}); err != nil { - t.Fatalf("failed to create previously keyed stack: %v", err) - } - blob2, err := ioutil.ReadFile(filepath.Join(dir, datadirPrivateKey)) + config = &Config{Name: "unit-test", DataDir: dir} + config.NodeKey() + blob2, err := ioutil.ReadFile(filepath.Join(keyfile)) if err != nil { t.Fatalf("failed to read previously persisted node key: %v", err) } if bytes.Compare(blob1, blob2) != 0 { t.Fatalf("persisted node key mismatch: have %x, want %x", blob2, blob1) } + // Configure ephemeral node and ensure no key is dumped locally - if _, err := New(&Config{DataDir: ""}); err != nil { - t.Fatalf("failed to create ephemeral stack: %v", err) - } - if _, err := os.Stat(filepath.Join(".", datadirPrivateKey)); err == nil { + config = &Config{Name: "unit-test", DataDir: ""} + config.NodeKey() + if _, err := os.Stat(filepath.Join(".", "unit-test", datadirPrivateKey)); err == nil { t.Fatalf("ephemeral node key persisted to disk") } } diff --git a/common/defaults.go b/node/defaults.go index 8a136fa80..bfe257c8e 100644 --- a/common/defaults.go +++ b/node/defaults.go @@ -14,9 +14,11 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package common +package node import ( + "os" + "os/user" "path/filepath" "runtime" ) @@ -33,7 +35,7 @@ const ( // persistence requirements. func DefaultDataDir() string { // Try to place the data folder in the user's home dir - home := HomeDir() + home := homeDir() if home != "" { if runtime.GOOS == "darwin" { return filepath.Join(home, "Library", "Ethereum") @@ -46,3 +48,13 @@ func DefaultDataDir() string { // As we cannot guess a stable location, return empty and handle later return "" } + +func homeDir() string { + if home := os.Getenv("HOME"); home != "" { + return home + } + if usr, err := user.Current(); err == nil { + return usr.HomeDir + } + return "" +} diff --git a/node/doc.go b/node/doc.go new file mode 100644 index 000000000..f009e6f85 --- /dev/null +++ b/node/doc.go @@ -0,0 +1,90 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +/* +Package node sets up multi-protocol Ethereum nodes. + +In the model exposed by this package, a node is a collection of services which use shared +resources to provide RPC APIs. Services can also offer devp2p protocols, which are wired +up to the devp2p network when the node instance is started. + + +Resources Managed By Node + +All file-system resources used by a node instance are located in a directory called the +data directory. The location of each resource can be overridden through additional node +configuration. The data directory is optional. If it is not set and the location of a +resource is otherwise unspecified, package node will create the resource in memory. + +To access to the devp2p network, Node configures and starts p2p.Server. Each host on the +devp2p network has a unique identifier, the node key. The Node instance persists this key +across restarts. Node also loads static and trusted node lists and ensures that knowledge +about other hosts is persisted. + +JSON-RPC servers which run HTTP, WebSocket or IPC can be started on a Node. RPC modules +offered by registered services will be offered on those endpoints. Users can restrict any +endpoint to a subset of RPC modules. Node itself offers the "debug", "admin" and "web3" +modules. + +Service implementations can open LevelDB databases through the service context. Package +node chooses the file system location of each database. If the node is configured to run +without a data directory, databases are opened in memory instead. + +Node also creates the shared store of encrypted Ethereum account keys. Services can access +the account manager through the service context. + + +Sharing Data Directory Among Instances + +Multiple node instances can share a single data directory if they have distinct instance +names (set through the Name config option). Sharing behaviour depends on the type of +resource. + +devp2p-related resources (node key, static/trusted node lists, known hosts database) are +stored in a directory with the same name as the instance. Thus, multiple node instances +using the same data directory will store this information in different subdirectories of +the data directory. + +LevelDB databases are also stored within the instance subdirectory. If multiple node +instances use the same data directory, openening the databases with identical names will +create one database for each instance. + +The account key store is shared among all node instances using the same data directory +unless its location is changed through the KeyStoreDir configuration option. + + +Data Directory Sharing Example + +In this exanple, two node instances named A and B are started with the same data +directory. Mode instance A opens the database "db", node instance B opens the databases +"db" and "db-2". The following files will be created in the data directory: + + data-directory/ + A/ + nodekey -- devp2p node key of instance A + nodes/ -- devp2p discovery knowledge database of instance A + db/ -- LevelDB content for "db" + A.ipc -- JSON-RPC UNIX domain socket endpoint of instance A + B/ + nodekey -- devp2p node key of node B + nodes/ -- devp2p discovery knowledge database of instance B + static-nodes.json -- devp2p static node list of instance B + db/ -- LevelDB content for "db" + db-2/ -- LevelDB content for "db-2" + B.ipc -- JSON-RPC UNIX domain socket endpoint of instance A + keystore/ -- account key store, used by both instances +*/ +package node diff --git a/node/node.go b/node/node.go index f3be2f763..41c9eb27f 100644 --- a/node/node.go +++ b/node/node.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -// Package node represents the Ethereum protocol stack container. package node import ( @@ -23,16 +22,19 @@ import ( "os" "path/filepath" "reflect" + "strings" "sync" "syscall" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" + "github.com/syndtr/goleveldb/leveldb/storage" ) var ( @@ -44,14 +46,14 @@ var ( datadirInUseErrnos = map[uint]bool{11: true, 32: true, 35: true} ) -// Node represents a P2P node into which arbitrary (uniquely typed) services might -// be registered. +// Node is a container on which services can be registered. type Node struct { - datadir string // Path to the currently used data directory eventmux *event.TypeMux // Event multiplexer used between the services of a stack + config *Config + accman *accounts.Manager - accman *accounts.Manager - ephemeralKeystore string // if non-empty, the key directory that will be removed by Stop + ephemeralKeystore string // if non-empty, the key directory that will be removed by Stop + instanceDirLock storage.Storage // prevents concurrent use of instance directory serverConfig p2p.Config server *p2p.Server // Currently running P2P networking layer @@ -66,21 +68,14 @@ type Node struct { ipcListener net.Listener // IPC RPC listener socket to serve API requests ipcHandler *rpc.Server // IPC RPC request handler to process the API requests - httpHost string // HTTP hostname - httpPort int // HTTP post httpEndpoint string // HTTP endpoint (interface + port) to listen at (empty = HTTP disabled) httpWhitelist []string // HTTP RPC modules to allow through this endpoint - httpCors string // HTTP RPC Cross-Origin Resource Sharing header httpListener net.Listener // HTTP RPC listener socket to server API requests httpHandler *rpc.Server // HTTP RPC request handler to process the API requests - wsHost string // Websocket host - wsPort int // Websocket post - wsEndpoint string // Websocket endpoint (interface + port) to listen at (empty = websocket disabled) - wsWhitelist []string // Websocket RPC modules to allow through this endpoint - wsOrigins string // Websocket RPC allowed origin domains - wsListener net.Listener // Websocket RPC listener socket to server API requests - wsHandler *rpc.Server // Websocket RPC request handler to process the API requests + wsEndpoint string // Websocket endpoint (interface + port) to listen at (empty = websocket disabled) + wsListener net.Listener // Websocket RPC listener socket to server API requests + wsHandler *rpc.Server // Websocket RPC request handler to process the API requests stop chan struct{} // Channel to wait for termination notifications lock sync.RWMutex @@ -88,54 +83,45 @@ type Node struct { // New creates a new P2P node, ready for protocol registration. func New(conf *Config) (*Node, error) { - // Ensure the data directory exists, failing if it cannot be created + // Copy config and resolve the datadir so future changes to the current + // working directory don't affect the node. + confCopy := *conf + conf = &confCopy if conf.DataDir != "" { - if err := os.MkdirAll(conf.DataDir, 0700); err != nil { + absdatadir, err := filepath.Abs(conf.DataDir) + if err != nil { return nil, err } + conf.DataDir = absdatadir + } + // Ensure that the instance name doesn't cause weird conflicts with + // other files in the data directory. + if strings.ContainsAny(conf.Name, `/\`) { + return nil, errors.New(`Config.Name must not contain '/' or '\'`) + } + if conf.Name == datadirDefaultKeyStore { + return nil, errors.New(`Config.Name cannot be "` + datadirDefaultKeyStore + `"`) } + if strings.HasSuffix(conf.Name, ".ipc") { + return nil, errors.New(`Config.Name cannot end in ".ipc"`) + } + // Ensure that the AccountManager method works before the node has started. + // We rely on this in cmd/geth. am, ephemeralKeystore, err := makeAccountManager(conf) if err != nil { return nil, err } - - // Assemble the networking layer and the node itself - nodeDbPath := "" - if conf.DataDir != "" { - nodeDbPath = filepath.Join(conf.DataDir, datadirNodeDatabase) - } + // Note: any interaction with Config that would create/touch files + // in the data directory or instance directory is delayed until Start. return &Node{ - datadir: conf.DataDir, accman: am, ephemeralKeystore: ephemeralKeystore, - serverConfig: p2p.Config{ - PrivateKey: conf.NodeKey(), - Name: conf.Name, - Discovery: !conf.NoDiscovery, - BootstrapNodes: conf.BootstrapNodes, - StaticNodes: conf.StaticNodes(), - TrustedNodes: conf.TrusterNodes(), - NodeDatabase: nodeDbPath, - ListenAddr: conf.ListenAddr, - NAT: conf.NAT, - Dialer: conf.Dialer, - NoDial: conf.NoDial, - MaxPeers: conf.MaxPeers, - MaxPendingPeers: conf.MaxPendingPeers, - }, - serviceFuncs: []ServiceConstructor{}, - ipcEndpoint: conf.IPCEndpoint(), - httpHost: conf.HTTPHost, - httpPort: conf.HTTPPort, - httpEndpoint: conf.HTTPEndpoint(), - httpWhitelist: conf.HTTPModules, - httpCors: conf.HTTPCors, - wsHost: conf.WSHost, - wsPort: conf.WSPort, - wsEndpoint: conf.WSEndpoint(), - wsWhitelist: conf.WSModules, - wsOrigins: conf.WSOrigins, - eventmux: new(event.TypeMux), + config: conf, + serviceFuncs: []ServiceConstructor{}, + ipcEndpoint: conf.IPCEndpoint(), + httpEndpoint: conf.HTTPEndpoint(), + wsEndpoint: conf.WSEndpoint(), + eventmux: new(event.TypeMux), }, nil } @@ -161,13 +147,36 @@ func (n *Node) Start() error { if n.server != nil { return ErrNodeRunning } - // Otherwise copy and specialize the P2P configuration + if err := n.openDataDir(); err != nil { + return err + } + + // Initialize the p2p server. This creates the node key and + // discovery databases. + n.serverConfig = p2p.Config{ + PrivateKey: n.config.NodeKey(), + Name: n.config.NodeName(), + Discovery: !n.config.NoDiscovery, + BootstrapNodes: n.config.BootstrapNodes, + StaticNodes: n.config.StaticNodes(), + TrustedNodes: n.config.TrusterNodes(), + NodeDatabase: n.config.NodeDB(), + ListenAddr: n.config.ListenAddr, + NAT: n.config.NAT, + Dialer: n.config.Dialer, + NoDial: n.config.NoDial, + MaxPeers: n.config.MaxPeers, + MaxPendingPeers: n.config.MaxPendingPeers, + } running := &p2p.Server{Config: n.serverConfig} + glog.V(logger.Info).Infoln("instance:", n.serverConfig.Name) + + // Otherwise copy and specialize the P2P configuration services := make(map[reflect.Type]Service) for _, constructor := range n.serviceFuncs { // Create a new context for the particular service ctx := &ServiceContext{ - datadir: n.datadir, + config: n.config, services: make(map[reflect.Type]Service), EventMux: n.eventmux, AccountManager: n.accman, @@ -227,6 +236,26 @@ func (n *Node) Start() error { return nil } +func (n *Node) openDataDir() error { + if n.config.DataDir == "" { + return nil // ephemeral + } + + instdir := filepath.Join(n.config.DataDir, n.config.name()) + if err := os.MkdirAll(instdir, 0700); err != nil { + return err + } + // Try to open the instance directory as LevelDB storage. This creates a lock file + // which prevents concurrent use by another instance as well as accidental use of the + // instance directory as a database. + storage, err := storage.OpenFile(instdir, true) + if err != nil { + return err + } + n.instanceDirLock = storage + return nil +} + // startRPC is a helper method to start all the various RPC endpoint during node // startup. It's not meant to be called at any time afterwards as it makes certain // assumptions about the state of the node. @@ -244,12 +273,12 @@ func (n *Node) startRPC(services map[reflect.Type]Service) error { n.stopInProc() return err } - if err := n.startHTTP(n.httpEndpoint, apis, n.httpWhitelist, n.httpCors); err != nil { + if err := n.startHTTP(n.httpEndpoint, apis, n.config.HTTPModules, n.config.HTTPCors); err != nil { n.stopIPC() n.stopInProc() return err } - if err := n.startWS(n.wsEndpoint, apis, n.wsWhitelist, n.wsOrigins); err != nil { + if err := n.startWS(n.wsEndpoint, apis, n.config.WSModules, n.config.WSOrigins); err != nil { n.stopHTTP() n.stopIPC() n.stopInProc() @@ -381,7 +410,6 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors n.httpEndpoint = endpoint n.httpListener = listener n.httpHandler = handler - n.httpCors = cors return nil } @@ -436,7 +464,6 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig n.wsEndpoint = endpoint n.wsListener = listener n.wsHandler = handler - n.wsOrigins = wsOrigins return nil } @@ -465,12 +492,12 @@ func (n *Node) Stop() error { if n.server == nil { return ErrNodeStopped } - // Otherwise terminate the API, all services and the P2P server too + + // Terminate the API, services and the p2p server. n.stopWS() n.stopHTTP() n.stopIPC() n.rpcAPIs = nil - failure := &StopError{ Services: make(map[reflect.Type]error), } @@ -480,9 +507,16 @@ func (n *Node) Stop() error { } } n.server.Stop() - n.services = nil n.server = nil + + // Release instance directory lock. + if n.instanceDirLock != nil { + n.instanceDirLock.Close() + n.instanceDirLock = nil + } + + // unblock n.Wait close(n.stop) // Remove the keystore if it was created ephemerally. @@ -566,7 +600,7 @@ func (n *Node) Service(service interface{}) error { // DataDir retrieves the current datadir used by the protocol stack. func (n *Node) DataDir() string { - return n.datadir + return n.config.DataDir } // AccountManager retrieves the account manager used by the protocol stack. @@ -595,6 +629,21 @@ func (n *Node) EventMux() *event.TypeMux { return n.eventmux } +// OpenDatabase opens an existing database with the given name (or creates one if no +// previous can be found) from within the node's instance directory. If the node is +// ephemeral, a memory database is returned. +func (n *Node) OpenDatabase(name string, cache, handles int) (ethdb.Database, error) { + if n.config.DataDir == "" { + return ethdb.NewMemDatabase() + } + return ethdb.NewLDBDatabase(n.config.resolvePath(name), cache, handles) +} + +// ResolvePath returns the absolute path of a resource in the instance directory. +func (n *Node) ResolvePath(x string) string { + return n.config.resolvePath(x) +} + // apis returns the collection of RPC descriptors this node offers. func (n *Node) apis() []rpc.API { return []rpc.API{ diff --git a/node/service.go b/node/service.go index 51531466b..1cd1fe808 100644 --- a/node/service.go +++ b/node/service.go @@ -17,7 +17,6 @@ package node import ( - "path/filepath" "reflect" "github.com/ethereum/go-ethereum/accounts" @@ -31,7 +30,7 @@ import ( // the protocol stack, that is passed to all constructors to be optionally used; // as well as utility methods to operate on the service environment. type ServiceContext struct { - datadir string // Data directory for protocol persistence + config *Config services map[reflect.Type]Service // Index of the already constructed services EventMux *event.TypeMux // Event multiplexer used for decoupled notifications AccountManager *accounts.Manager // Account manager created by the node. @@ -41,10 +40,10 @@ type ServiceContext struct { // if no previous can be found) from within the node's data directory. If the // node is an ephemeral one, a memory database is returned. func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) { - if ctx.datadir == "" { + if ctx.config.DataDir == "" { return ethdb.NewMemDatabase() } - return ethdb.NewLDBDatabase(filepath.Join(ctx.datadir, name), cache, handles) + return ethdb.NewLDBDatabase(ctx.config.resolvePath(name), cache, handles) } // Service retrieves a currently running service registered of a specific type. @@ -64,11 +63,13 @@ type ServiceConstructor func(ctx *ServiceContext) (Service, error) // Service is an individual protocol that can be registered into a node. // // Notes: -// - Service life-cycle management is delegated to the node. The service is -// allowed to initialize itself upon creation, but no goroutines should be -// spun up outside of the Start method. -// - Restart logic is not required as the node will create a fresh instance -// every time a service is started. +// +// • Service life-cycle management is delegated to the node. The service is allowed to +// initialize itself upon creation, but no goroutines should be spun up outside of the +// Start method. +// +// • Restart logic is not required as the node will create a fresh instance +// every time a service is started. type Service interface { // Protocols retrieves the P2P protocols the service wishes to start. Protocols() []p2p.Protocol diff --git a/node/service_test.go b/node/service_test.go index 7bd94a52e..a7ae439e0 100644 --- a/node/service_test.go +++ b/node/service_test.go @@ -38,18 +38,18 @@ func TestContextDatabases(t *testing.T) { t.Fatalf("non-created database already exists") } // Request the opening/creation of a database and ensure it persists to disk - ctx := &ServiceContext{datadir: dir} + ctx := &ServiceContext{config: &Config{Name: "unit-test", DataDir: dir}} db, err := ctx.OpenDatabase("persistent", 0, 0) if err != nil { t.Fatalf("failed to open persistent database: %v", err) } db.Close() - if _, err := os.Stat(filepath.Join(dir, "persistent")); err != nil { + if _, err := os.Stat(filepath.Join(dir, "unit-test", "persistent")); err != nil { t.Fatalf("persistent database doesn't exists: %v", err) } // Request th opening/creation of an ephemeral database and ensure it's not persisted - ctx = &ServiceContext{datadir: ""} + ctx = &ServiceContext{config: &Config{DataDir: ""}} db, err = ctx.OpenDatabase("ephemeral", 0, 0) if err != nil { t.Fatalf("failed to open ephemeral database: %v", err) diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index 505a1fc77..f9ba93613 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -105,7 +105,7 @@ func Map(m Interface, c chan struct{}, protocol string, extport, intport int, na glog.V(logger.Debug).Infof("deleting port mapping: %s %d -> %d (%s) using %s\n", protocol, extport, intport, name, m) m.DeleteMapping(protocol, extport, intport) }() - if err := m.AddMapping(protocol, intport, extport, name, mapTimeout); err != nil { + if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { glog.V(logger.Debug).Infof("network port %s:%d could not be mapped: %v\n", protocol, intport, err) } else { glog.V(logger.Info).Infof("mapped network port %s:%d -> %d (%s) using %s\n", protocol, extport, intport, name, m) @@ -118,7 +118,7 @@ func Map(m Interface, c chan struct{}, protocol string, extport, intport int, na } case <-refresh.C: glog.V(logger.Detail).Infof("refresh port mapping %s:%d -> %d (%s) using %s\n", protocol, extport, intport, name, m) - if err := m.AddMapping(protocol, intport, extport, name, mapTimeout); err != nil { + if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { glog.V(logger.Debug).Infof("network port %s:%d could not be mapped: %v\n", protocol, intport, err) } refresh.Reset(mapUpdateInterval) @@ -197,11 +197,7 @@ type autodisc struct { func startautodisc(what string, doit func() Interface) Interface { // TODO: monitor network configuration and rerun doit when it changes. - ad := &autodisc{what: what, doit: doit} - // Start the auto discovery as early as possible so it is already - // in progress when the rest of the stack calls the methods. - go ad.wait() - return ad + return &autodisc{what: what, doit: doit} } func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { diff --git a/trie/arc.go b/trie/arc.go deleted file mode 100644 index fc7a3259f..000000000 --- a/trie/arc.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (c) 2015 Hans Alexander Gugel <alexander.gugel@gmail.com> -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -// This file contains a modified version of package arc from -// https://github.com/alexanderGugel/arc -// -// It implements the ARC (Adaptive Replacement Cache) algorithm as detailed in -// https://www.usenix.org/legacy/event/fast03/tech/full_papers/megiddo/megiddo.pdf - -package trie - -import ( - "container/list" - "sync" -) - -type arc struct { - p int - c int - t1 *list.List - b1 *list.List - t2 *list.List - b2 *list.List - cache map[string]*entry - mutex sync.Mutex -} - -type entry struct { - key hashNode - value node - ll *list.List - el *list.Element -} - -// newARC returns a new Adaptive Replacement Cache with the -// given capacity. -func newARC(c int) *arc { - return &arc{ - c: c, - t1: list.New(), - b1: list.New(), - t2: list.New(), - b2: list.New(), - cache: make(map[string]*entry, c), - } -} - -// Clear clears the cache -func (a *arc) Clear() { - a.mutex.Lock() - defer a.mutex.Unlock() - a.p = 0 - a.t1 = list.New() - a.b1 = list.New() - a.t2 = list.New() - a.b2 = list.New() - a.cache = make(map[string]*entry, a.c) -} - -// Put inserts a new key-value pair into the cache. -// This optimizes future access to this entry (side effect). -func (a *arc) Put(key hashNode, value node) bool { - a.mutex.Lock() - defer a.mutex.Unlock() - ent, ok := a.cache[string(key)] - if ok != true { - ent = &entry{key: key, value: value} - a.req(ent) - a.cache[string(key)] = ent - } else { - ent.value = value - a.req(ent) - } - return ok -} - -// Get retrieves a previously via Set inserted entry. -// This optimizes future access to this entry (side effect). -func (a *arc) Get(key hashNode) (value node, ok bool) { - a.mutex.Lock() - defer a.mutex.Unlock() - ent, ok := a.cache[string(key)] - if ok { - a.req(ent) - return ent.value, ent.value != nil - } - return nil, false -} - -func (a *arc) req(ent *entry) { - if ent.ll == a.t1 || ent.ll == a.t2 { - // Case I - ent.setMRU(a.t2) - } else if ent.ll == a.b1 { - // Case II - // Cache Miss in t1 and t2 - - // Adaptation - var d int - if a.b1.Len() >= a.b2.Len() { - d = 1 - } else { - d = a.b2.Len() / a.b1.Len() - } - a.p = a.p + d - if a.p > a.c { - a.p = a.c - } - - a.replace(ent) - ent.setMRU(a.t2) - } else if ent.ll == a.b2 { - // Case III - // Cache Miss in t1 and t2 - - // Adaptation - var d int - if a.b2.Len() >= a.b1.Len() { - d = 1 - } else { - d = a.b1.Len() / a.b2.Len() - } - a.p = a.p - d - if a.p < 0 { - a.p = 0 - } - - a.replace(ent) - ent.setMRU(a.t2) - } else if ent.ll == nil { - // Case IV - - if a.t1.Len()+a.b1.Len() == a.c { - // Case A - if a.t1.Len() < a.c { - a.delLRU(a.b1) - a.replace(ent) - } else { - a.delLRU(a.t1) - } - } else if a.t1.Len()+a.b1.Len() < a.c { - // Case B - if a.t1.Len()+a.t2.Len()+a.b1.Len()+a.b2.Len() >= a.c { - if a.t1.Len()+a.t2.Len()+a.b1.Len()+a.b2.Len() == 2*a.c { - a.delLRU(a.b2) - } - a.replace(ent) - } - } - - ent.setMRU(a.t1) - } -} - -func (a *arc) delLRU(list *list.List) { - lru := list.Back() - list.Remove(lru) - delete(a.cache, string(lru.Value.(*entry).key)) -} - -func (a *arc) replace(ent *entry) { - if a.t1.Len() > 0 && ((a.t1.Len() > a.p) || (ent.ll == a.b2 && a.t1.Len() == a.p)) { - lru := a.t1.Back().Value.(*entry) - lru.value = nil - lru.setMRU(a.b1) - } else { - lru := a.t2.Back().Value.(*entry) - lru.value = nil - lru.setMRU(a.b2) - } -} - -func (e *entry) setLRU(list *list.List) { - e.detach() - e.ll = list - e.el = e.ll.PushBack(e) -} - -func (e *entry) setMRU(list *list.List) { - e.detach() - e.ll = list - e.el = e.ll.PushFront(e) -} - -func (e *entry) detach() { - if e.ll != nil { - e.ll.Remove(e.el) - } -} diff --git a/trie/hasher.go b/trie/hasher.go new file mode 100644 index 000000000..87e02fb85 --- /dev/null +++ b/trie/hasher.go @@ -0,0 +1,157 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package trie + +import ( + "bytes" + "hash" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/rlp" +) + +type hasher struct { + tmp *bytes.Buffer + sha hash.Hash +} + +// hashers live in a global pool. +var hasherPool = sync.Pool{ + New: func() interface{} { + return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()} + }, +} + +func newHasher() *hasher { + return hasherPool.Get().(*hasher) +} + +func returnHasherToPool(h *hasher) { + hasherPool.Put(h) +} + +// hash collapses a node down into a hash node, also returning a copy of the +// original node initialzied with the computed hash to replace the original one. +func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) { + // If we're not storing the node, just hashing, use avaialble cached data + if hash, dirty := n.cache(); hash != nil && (db == nil || !dirty) { + return hash, n, nil + } + // Trie not processed yet or needs storage, walk the children + collapsed, cached, err := h.hashChildren(n, db) + if err != nil { + return hashNode{}, n, err + } + hashed, err := h.store(collapsed, db, force) + if err != nil { + return hashNode{}, n, err + } + // Cache the hash and RLP blob of the ndoe for later reuse + if hash, ok := hashed.(hashNode); ok && !force { + switch cached := cached.(type) { + case shortNode: + cached.hash = hash + if db != nil { + cached.dirty = false + } + return hashed, cached, nil + case fullNode: + cached.hash = hash + if db != nil { + cached.dirty = false + } + return hashed, cached, nil + } + } + return hashed, cached, nil +} + +// hashChildren replaces the children of a node with their hashes if the encoded +// size of the child is larger than a hash, returning the collapsed node as well +// as a replacement for the original node with the child hashes cached in. +func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, error) { + var err error + + switch n := original.(type) { + case shortNode: + // Hash the short node's child, caching the newly hashed subtree + cached := n + cached.Key = common.CopyBytes(cached.Key) + + n.Key = compactEncode(n.Key) + if _, ok := n.Val.(valueNode); !ok { + if n.Val, cached.Val, err = h.hash(n.Val, db, false); err != nil { + return n, original, err + } + } + if n.Val == nil { + n.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings. + } + return n, cached, nil + + case fullNode: + // Hash the full node's children, caching the newly hashed subtrees + cached := fullNode{dirty: n.dirty} + + for i := 0; i < 16; i++ { + if n.Children[i] != nil { + if n.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false); err != nil { + return n, original, err + } + } else { + n.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings. + } + } + cached.Children[16] = n.Children[16] + if n.Children[16] == nil { + n.Children[16] = valueNode(nil) + } + return n, cached, nil + + default: + // Value and hash nodes don't have children so they're left as were + return n, original, nil + } +} + +func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { + // Don't store hashes or empty nodes. + if _, isHash := n.(hashNode); n == nil || isHash { + return n, nil + } + // Generate the RLP encoding of the node + h.tmp.Reset() + if err := rlp.Encode(h.tmp, n); err != nil { + panic("encode error: " + err.Error()) + } + if h.tmp.Len() < 32 && !force { + return n, nil // Nodes smaller than 32 bytes are stored inside their parent + } + // Larger nodes are replaced by their hash and stored in the database. + hash, _ := n.cache() + if hash == nil { + h.sha.Reset() + h.sha.Write(h.tmp.Bytes()) + hash = hashNode(h.sha.Sum(nil)) + } + if db != nil { + return hash, db.Put(hash, h.tmp.Bytes()) + } + return hash, nil +} diff --git a/trie/iterator.go b/trie/iterator.go index 88c4cee7f..8cad51aff 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -16,18 +16,13 @@ package trie -import ( - "bytes" - "fmt" +import "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/logger/glog" -) - -// Iterator is a key-value trie iterator to traverse the data contents. +// Iterator is a key-value trie iterator that traverses a Trie. type Iterator struct { - trie *Trie + trie *Trie + nodeIt *NodeIterator + keyBuf []byte Key []byte // Current data key on which the iterator is positioned on Value []byte // Current data value on which the iterator is positioned on @@ -35,119 +30,45 @@ type Iterator struct { // NewIterator creates a new key-value iterator. func NewIterator(trie *Trie) *Iterator { - return &Iterator{trie: trie, Key: nil} -} - -// Next moves the iterator forward with one key-value entry. -func (self *Iterator) Next() bool { - isIterStart := false - if self.Key == nil { - isIterStart = true - self.Key = make([]byte, 32) + return &Iterator{ + trie: trie, + nodeIt: NewNodeIterator(trie), + keyBuf: make([]byte, 0, 64), + Key: nil, } - - key := remTerm(compactHexDecode(self.Key)) - k := self.next(self.trie.root, key, isIterStart) - - self.Key = []byte(decodeCompact(k)) - - return len(k) > 0 } -func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byte { - if node == nil { - return nil - } - - switch node := node.(type) { - case fullNode: - if len(key) > 0 { - k := self.next(node.Children[key[0]], key[1:], isIterStart) - if k != nil { - return append([]byte{key[0]}, k...) - } - } - - var r byte - if len(key) > 0 { - r = key[0] + 1 - } - - for i := r; i < 16; i++ { - k := self.key(node.Children[i]) - if k != nil { - return append([]byte{i}, k...) - } +// Next moves the iterator forward one key-value entry. +func (it *Iterator) Next() bool { + for it.nodeIt.Next() { + if it.nodeIt.Leaf { + it.Key = it.makeKey() + it.Value = it.nodeIt.LeafBlob + return true } - - case shortNode: - k := remTerm(node.Key) - if vnode, ok := node.Val.(valueNode); ok { - switch bytes.Compare([]byte(k), key) { - case 0: - if isIterStart { - self.Value = vnode - return k - } - case 1: - self.Value = vnode - return k - } - } else { - cnode := node.Val - - var ret []byte - skey := key[len(k):] - if bytes.HasPrefix(key, k) { - ret = self.next(cnode, skey, isIterStart) - } else if bytes.Compare(k, key[:len(k)]) > 0 { - return self.key(node) - } - - if ret != nil { - return append(k, ret...) - } - } - - case hashNode: - rn, err := self.trie.resolveHash(node, nil, nil) - if err != nil && glog.V(logger.Error) { - glog.Errorf("Unhandled trie error: %v", err) - } - return self.next(rn, key, isIterStart) } - return nil + it.Key = nil + it.Value = nil + return false } -func (self *Iterator) key(node interface{}) []byte { - switch node := node.(type) { - case shortNode: - // Leaf node - k := remTerm(node.Key) - if vnode, ok := node.Val.(valueNode); ok { - self.Value = vnode - return k - } - return append(k, self.key(node.Val)...) - case fullNode: - if node.Children[16] != nil { - self.Value = node.Children[16].(valueNode) - return []byte{16} - } - for i := 0; i < 16; i++ { - k := self.key(node.Children[i]) - if k != nil { - return append([]byte{byte(i)}, k...) +func (it *Iterator) makeKey() []byte { + key := it.keyBuf[:0] + for _, se := range it.nodeIt.stack { + switch node := se.node.(type) { + case fullNode: + if se.child <= 16 { + key = append(key, byte(se.child)) + } + case shortNode: + if hasTerm(node.Key) { + key = append(key, node.Key[:len(node.Key)-1]...) + } else { + key = append(key, node.Key...) } } - case hashNode: - rn, err := self.trie.resolveHash(node, nil, nil) - if err != nil && glog.V(logger.Error) { - glog.Errorf("Unhandled trie error: %v", err) - } - return self.key(rn) } - return nil + return decodeCompact(key) } // nodeIteratorState represents the iteration state at one particular node of the @@ -199,25 +120,27 @@ func (it *NodeIterator) Next() bool { // step moves the iterator to the next node of the trie. func (it *NodeIterator) step() error { - // Abort if we reached the end of the iteration if it.trie == nil { + // Abort if we reached the end of the iteration return nil } - // Initialize the iterator if we've just started, or pop off the old node otherwise if len(it.stack) == 0 { - // Always start with a collapsed root + // Initialize the iterator if we've just started. root := it.trie.Hash() - it.stack = append(it.stack, &nodeIteratorState{node: hashNode(root[:]), child: -1}) - if it.stack[0].node == nil { - return fmt.Errorf("root node missing: %x", it.trie.Hash()) + state := &nodeIteratorState{node: it.trie.root, child: -1} + if root != emptyRoot { + state.hash = root } + it.stack = append(it.stack, state) } else { + // Continue iterating at the previous node otherwise. it.stack = it.stack[:len(it.stack)-1] if len(it.stack) == 0 { it.trie = nil return nil } } + // Continue iteration to the next child for { parent := it.stack[len(it.stack)-1] @@ -232,7 +155,12 @@ func (it *NodeIterator) step() error { } for parent.child++; parent.child < len(node.Children); parent.child++ { if current := node.Children[parent.child]; current != nil { - it.stack = append(it.stack, &nodeIteratorState{node: current, parent: ancestor, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{ + hash: common.BytesToHash(node.hash), + node: current, + parent: ancestor, + child: -1, + }) break } } @@ -242,7 +170,12 @@ func (it *NodeIterator) step() error { break } parent.child++ - it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{ + hash: common.BytesToHash(node.hash), + node: node.Val, + parent: ancestor, + child: -1, + }) } else if hash, ok := parent.node.(hashNode); ok { // Hash node, resolve the hash child from the database, then the node itself if parent.child >= 0 { @@ -254,7 +187,12 @@ func (it *NodeIterator) step() error { if err != nil { return err } - it.stack = append(it.stack, &nodeIteratorState{hash: common.BytesToHash(hash), node: node, parent: ancestor, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{ + hash: common.BytesToHash(hash), + node: node, + parent: ancestor, + child: -1, + }) } else { break } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index dc8276116..2bcc3700e 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -34,21 +34,60 @@ func TestIterator(t *testing.T) { {"dog", "puppy"}, {"somethingveryoddindeedthis is", "myothernodedata"}, } - v := make(map[string]bool) + all := make(map[string]string) for _, val := range vals { - v[val.k] = false + all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } trie.Commit() + found := make(map[string]string) it := NewIterator(trie) for it.Next() { - v[string(it.Key)] = true + found[string(it.Key)] = string(it.Value) } - for k, found := range v { - if !found { - t.Error("iterator didn't find", k) + for k, v := range all { + if found[k] != v { + t.Errorf("iterator value mismatch for %s: got %q want %q", k, found[k], v) + } + } +} + +type kv struct { + k, v []byte + t bool +} + +func TestIteratorLargeData(t *testing.T) { + trie := newEmpty() + vals := make(map[string]*kv) + + for i := byte(0); i < 255; i++ { + value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} + value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false} + trie.Update(value.k, value.v) + trie.Update(value2.k, value2.v) + vals[string(value.k)] = value + vals[string(value2.k)] = value2 + } + + it := NewIterator(trie) + for it.Next() { + vals[string(it.Key)].t = true + } + + var untouched []*kv + for _, value := range vals { + if !value.t { + untouched = append(untouched, value) + } + } + + if len(untouched) > 0 { + t.Errorf("Missed %d nodes", len(untouched)) + for _, value := range untouched { + t.Error(value) } } } diff --git a/trie/proof.go b/trie/proof.go index 5135de047..116c13a1b 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -70,15 +70,13 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue { panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - if t.hasher == nil { - t.hasher = newHasher() - } + hasher := newHasher() proof := make([]rlp.RawValue, 0, len(nodes)) for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. - n, _, _ = t.hasher.hashChildren(n, nil) - hn, _ := t.hasher.store(n, nil, false) + n, _, _ = hasher.hashChildren(n, nil) + hn, _ := hasher.store(n, nil, false) if _, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 1d027c102..2a8b57214 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -17,16 +17,15 @@ package trie import ( - "hash" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" ) var secureKeyPrefix = []byte("secure-key-") +const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash + // SecureTrie wraps a trie with key hashing. In a secure trie, all // access operations hash the key using keccak256. This prevents // calling code from creating long chains of nodes that @@ -38,12 +37,11 @@ var secureKeyPrefix = []byte("secure-key-") // // SecureTrie is not safe for concurrent use. type SecureTrie struct { - *Trie - - hash hash.Hash - hashKeyBuf []byte - secKeyBuf []byte - secKeyCache map[string][]byte + trie Trie + hashKeyBuf [secureKeyLength]byte + secKeyBuf [200]byte + secKeyCache map[string][]byte + secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch } // NewSecure creates a trie with an existing root node from db. @@ -61,8 +59,7 @@ func NewSecure(root common.Hash, db Database) (*SecureTrie, error) { return nil, err } return &SecureTrie{ - Trie: trie, - secKeyCache: make(map[string][]byte), + trie: *trie, }, nil } @@ -80,7 +77,7 @@ func (t *SecureTrie) Get(key []byte) []byte { // The value bytes must not be modified by the caller. // If a node was not found in the database, a MissingNodeError is returned. func (t *SecureTrie) TryGet(key []byte) ([]byte, error) { - return t.Trie.TryGet(t.hashKey(key)) + return t.trie.TryGet(t.hashKey(key)) } // Update associates key with value in the trie. Subsequent calls to @@ -105,11 +102,11 @@ func (t *SecureTrie) Update(key, value []byte) { // If a node was not found in the database, a MissingNodeError is returned. func (t *SecureTrie) TryUpdate(key, value []byte) error { hk := t.hashKey(key) - err := t.Trie.TryUpdate(hk, value) + err := t.trie.TryUpdate(hk, value) if err != nil { return err } - t.secKeyCache[string(hk)] = common.CopyBytes(key) + t.getSecKeyCache()[string(hk)] = common.CopyBytes(key) return nil } @@ -124,17 +121,17 @@ func (t *SecureTrie) Delete(key []byte) { // If a node was not found in the database, a MissingNodeError is returned. func (t *SecureTrie) TryDelete(key []byte) error { hk := t.hashKey(key) - delete(t.secKeyCache, string(hk)) - return t.Trie.TryDelete(hk) + delete(t.getSecKeyCache(), string(hk)) + return t.trie.TryDelete(hk) } // GetKey returns the sha3 preimage of a hashed key that was // previously used to store a value. func (t *SecureTrie) GetKey(shaKey []byte) []byte { - if key, ok := t.secKeyCache[string(shaKey)]; ok { + if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - key, _ := t.Trie.db.Get(t.secKey(shaKey)) + key, _ := t.trie.db.Get(t.secKey(shaKey)) return key } @@ -144,7 +141,23 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { // Committing flushes nodes from memory. Subsequent Get calls will load nodes // from the database. func (t *SecureTrie) Commit() (root common.Hash, err error) { - return t.CommitTo(t.db) + return t.CommitTo(t.trie.db) +} + +func (t *SecureTrie) Hash() common.Hash { + return t.trie.Hash() +} + +func (t *SecureTrie) Root() []byte { + return t.trie.Root() +} + +func (t *SecureTrie) Iterator() *Iterator { + return t.trie.Iterator() +} + +func (t *SecureTrie) NodeIterator() *NodeIterator { + return NewNodeIterator(&t.trie) } // CommitTo writes all nodes and the secure hash pre-images to the given database. @@ -154,7 +167,7 @@ func (t *SecureTrie) Commit() (root common.Hash, err error) { // the trie's database. Calling code must ensure that the changes made to db are // written back to the trie's attached database before using the trie. func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) { - if len(t.secKeyCache) > 0 { + if len(t.getSecKeyCache()) > 0 { for hk, key := range t.secKeyCache { if err := db.Put(t.secKey([]byte(hk)), key); err != nil { return common.Hash{}, err @@ -162,27 +175,37 @@ func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) { } t.secKeyCache = make(map[string][]byte) } - n, clean, err := t.hashRoot(db) - if err != nil { - return (common.Hash{}), err - } - t.root = clean - return common.BytesToHash(n.(hashNode)), nil + return t.trie.CommitTo(db) } +// secKey returns the database key for the preimage of key, as an ephemeral buffer. +// The caller must not hold onto the return value because it will become +// invalid on the next call to hashKey or secKey. func (t *SecureTrie) secKey(key []byte) []byte { - t.secKeyBuf = append(t.secKeyBuf[:0], secureKeyPrefix...) - t.secKeyBuf = append(t.secKeyBuf, key...) - return t.secKeyBuf + buf := append(t.secKeyBuf[:0], secureKeyPrefix...) + buf = append(buf, key...) + return buf } +// hashKey returns the hash of key as an ephemeral buffer. +// The caller must not hold onto the return value because it will become +// invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { - if t.hash == nil { - t.hash = sha3.NewKeccak256() - t.hashKeyBuf = make([]byte, 32) + h := newHasher() + h.sha.Reset() + h.sha.Write(key) + buf := h.sha.Sum(t.hashKeyBuf[:0]) + returnHasherToPool(h) + return buf +} + +// getSecKeyCache returns the current secure key cache, creating a new one if +// ownership changed (i.e. the current secure trie is a copy of another owning +// the actual cache). +func (t *SecureTrie) getSecKeyCache() map[string][]byte { + if t != t.secKeyCacheOwner { + t.secKeyCacheOwner = t + t.secKeyCache = make(map[string][]byte) } - t.hash.Reset() - t.hash.Write(key) - t.hashKeyBuf = t.hash.Sum(t.hashKeyBuf[:0]) - return t.hashKeyBuf + return t.secKeyCache } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 0be5b3d15..3171b8c31 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -18,6 +18,8 @@ package trie import ( "bytes" + "runtime" + "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -31,6 +33,37 @@ func newEmptySecure() *SecureTrie { return trie } +// makeTestSecureTrie creates a large enough secure trie for testing. +func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) { + // Create an empty trie + db, _ := ethdb.NewMemDatabase() + trie, _ := NewSecure(common.Hash{}, db) + + // Fill it with some arbitrary data + content := make(map[string][]byte) + for i := byte(0); i < 255; i++ { + // Map the same data under multiple keys + key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i} + content[string(key)] = val + trie.Update(key, val) + + key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i} + content[string(key)] = val + trie.Update(key, val) + + // Add some other data to inflate th trie + for j := byte(3); j < 13; j++ { + key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i} + content[string(key)] = val + trie.Update(key, val) + } + } + trie.Commit() + + // Return the generated trie + return db, trie, content +} + func TestSecureDelete(t *testing.T) { trie := newEmptySecure() vals := []struct{ k, v string }{ @@ -72,3 +105,41 @@ func TestSecureGetKey(t *testing.T) { t.Errorf("GetKey returned %q, want %q", k, key) } } + +func TestSecureTrieConcurrency(t *testing.T) { + // Create an initial trie and copy if for concurrent access + _, trie, _ := makeTestSecureTrie() + + threads := runtime.NumCPU() + tries := make([]*SecureTrie, threads) + for i := 0; i < threads; i++ { + cpy := *trie + tries[i] = &cpy + } + // Start a batch of goroutines interactng with the trie + pend := new(sync.WaitGroup) + pend.Add(threads) + for i := 0; i < threads; i++ { + go func(index int) { + defer pend.Done() + + for j := byte(0); j < 255; j++ { + // Map the same data under multiple keys + key, val := common.LeftPadBytes([]byte{byte(index), 1, j}, 32), []byte{j} + tries[index].Update(key, val) + + key, val = common.LeftPadBytes([]byte{byte(index), 2, j}, 32), []byte{j} + tries[index].Update(key, val) + + // Add some other data to inflate the trie + for k := byte(3); k < 13; k++ { + key, val = common.LeftPadBytes([]byte{byte(index), k, j}, 32), []byte{k, j} + tries[index].Update(key, val) + } + } + tries[index].Commit() + }(i) + } + // Wait for all threads to finish + pend.Wait() +} diff --git a/trie/sync_test.go b/trie/sync_test.go index a81f7650e..a763dc564 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -51,9 +51,6 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { } trie.Commit() - // Remove any potentially cached data from the test trie creation - globalCache.Clear() - // Return the generated trie return db, trie, content } @@ -61,9 +58,6 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { // checkTrieContents cross references a reconstructed trie with an expected data // content map. func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) { - // Remove any potentially cached data from the trie synchronisation - globalCache.Clear() - // Check root availability and trie contents trie, err := New(common.BytesToHash(root), db) if err != nil { @@ -81,9 +75,6 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db Database, root common.Hash) error { - // Remove any potentially cached data from the test trie creation or previous checks - globalCache.Clear() - // Create and iterate a trie rooted in a subnode trie, err := New(root, db) if err != nil { diff --git a/trie/trie.go b/trie/trie.go index a530e7b2a..93e189e2e 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -20,22 +20,14 @@ package trie import ( "bytes" "fmt" - "hash" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" - "github.com/ethereum/go-ethereum/rlp" ) -const defaultCacheCapacity = 800 - var ( - // The global cache stores decoded trie nodes by hash as they get loaded. - globalCache = newARC(defaultCacheCapacity) - // This is the known root hash of an empty trie. emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") @@ -43,11 +35,6 @@ var ( emptyState = crypto.Keccak256Hash(nil) ) -// ClearGlobalCache clears the global trie cache -func ClearGlobalCache() { - globalCache.Clear() -} - // Database must be implemented by backing stores for the trie. type Database interface { DatabaseWriter @@ -72,7 +59,6 @@ type Trie struct { root node db Database originalRoot common.Hash - *hasher } // New creates a trie with an existing root node from db. @@ -118,32 +104,50 @@ func (t *Trie) Get(key []byte) []byte { // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryGet(key []byte) ([]byte, error) { key = compactHexDecode(key) - pos := 0 - tn := t.root - for pos < len(key) { - switch n := tn.(type) { - case shortNode: - if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) { - return nil, nil - } - tn = n.Val - pos += len(n.Key) - case fullNode: - tn = n.Children[key[pos]] - pos++ - case nil: - return nil, nil - case hashNode: - var err error - tn, err = t.resolveHash(n, key[:pos], key[pos:]) - if err != nil { - return nil, err - } - default: - panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) + value, newroot, didResolve, err := t.tryGet(t.root, key, 0) + if err == nil && didResolve { + t.root = newroot + } + return value, err +} + +func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode node, didResolve bool, err error) { + switch n := (origNode).(type) { + case nil: + return nil, nil, false, nil + case valueNode: + return n, n, false, nil + case shortNode: + if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) { + // key not found in trie + return nil, n, false, nil + } + value, newnode, didResolve, err = t.tryGet(n.Val, key, pos+len(n.Key)) + if err == nil && didResolve { + n.Val = newnode + return value, n, didResolve, err + } else { + return value, origNode, didResolve, err + } + case fullNode: + child := n.Children[key[pos]] + value, newnode, didResolve, err = t.tryGet(child, key, pos+1) + if err == nil && didResolve { + n.Children[key[pos]] = newnode + return value, n, didResolve, err + } else { + return value, origNode, didResolve, err + } + case hashNode: + child, err := t.resolveHash(n, key[:pos], key[pos:]) + if err != nil { + return nil, n, true, err } + value, newnode, _, err := t.tryGet(child, key, pos) + return value, newnode, true, err + default: + panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode)) } - return tn.(valueNode), nil } // Update associates key with value in the trie. Subsequent calls to @@ -410,9 +414,6 @@ func (t *Trie) resolve(n node, prefix, suffix []byte) (node, error) { } func (t *Trie) resolveHash(n hashNode, prefix, suffix []byte) (node, error) { - if v, ok := globalCache.Get(n); ok { - return v, nil - } enc, err := t.db.Get(n) if err != nil || enc == nil { return nil, &MissingNodeError{ @@ -424,9 +425,6 @@ func (t *Trie) resolveHash(n hashNode, prefix, suffix []byte) (node, error) { } } dec := mustDecodeNode(n, enc) - if dec != nil { - globalCache.Put(n, dec) - } return dec, nil } @@ -474,127 +472,7 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) { if t.root == nil { return hashNode(emptyRoot.Bytes()), nil, nil } - if t.hasher == nil { - t.hasher = newHasher() - } - return t.hasher.hash(t.root, db, true) -} - -type hasher struct { - tmp *bytes.Buffer - sha hash.Hash -} - -func newHasher() *hasher { - return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()} -} - -// hash collapses a node down into a hash node, also returning a copy of the -// original node initialzied with the computed hash to replace the original one. -func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) { - // If we're not storing the node, just hashing, use avaialble cached data - if hash, dirty := n.cache(); hash != nil && (db == nil || !dirty) { - return hash, n, nil - } - // Trie not processed yet or needs storage, walk the children - collapsed, cached, err := h.hashChildren(n, db) - if err != nil { - return hashNode{}, n, err - } - hashed, err := h.store(collapsed, db, force) - if err != nil { - return hashNode{}, n, err - } - // Cache the hash and RLP blob of the ndoe for later reuse - if hash, ok := hashed.(hashNode); ok && !force { - switch cached := cached.(type) { - case shortNode: - cached.hash = hash - if db != nil { - cached.dirty = false - } - return hashed, cached, nil - case fullNode: - cached.hash = hash - if db != nil { - cached.dirty = false - } - return hashed, cached, nil - } - } - return hashed, cached, nil -} - -// hashChildren replaces the children of a node with their hashes if the encoded -// size of the child is larger than a hash, returning the collapsed node as well -// as a replacement for the original node with the child hashes cached in. -func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, error) { - var err error - - switch n := original.(type) { - case shortNode: - // Hash the short node's child, caching the newly hashed subtree - cached := n - cached.Key = common.CopyBytes(cached.Key) - - n.Key = compactEncode(n.Key) - if _, ok := n.Val.(valueNode); !ok { - if n.Val, cached.Val, err = h.hash(n.Val, db, false); err != nil { - return n, original, err - } - } - if n.Val == nil { - n.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings. - } - return n, cached, nil - - case fullNode: - // Hash the full node's children, caching the newly hashed subtrees - cached := fullNode{dirty: n.dirty} - - for i := 0; i < 16; i++ { - if n.Children[i] != nil { - if n.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false); err != nil { - return n, original, err - } - } else { - n.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings. - } - } - cached.Children[16] = n.Children[16] - if n.Children[16] == nil { - n.Children[16] = valueNode(nil) - } - return n, cached, nil - - default: - // Value and hash nodes don't have children so they're left as were - return n, original, nil - } -} - -func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { - // Don't store hashes or empty nodes. - if _, isHash := n.(hashNode); n == nil || isHash { - return n, nil - } - // Generate the RLP encoding of the node - h.tmp.Reset() - if err := rlp.Encode(h.tmp, n); err != nil { - panic("encode error: " + err.Error()) - } - if h.tmp.Len() < 32 && !force { - return n, nil // Nodes smaller than 32 bytes are stored inside their parent - } - // Larger nodes are replaced by their hash and stored in the database. - hash, _ := n.cache() - if hash == nil { - h.sha.Reset() - h.sha.Write(h.tmp.Bytes()) - hash = hashNode(h.sha.Sum(nil)) - } - if db != nil { - return hash, db.Put(hash, h.tmp.Bytes()) - } - return hash, nil + h := newHasher() + defer returnHasherToPool(h) + return h.hash(t.root, db, true) } diff --git a/trie/trie_test.go b/trie/trie_test.go index 121ba24c1..5a3ea1be9 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -76,8 +76,6 @@ func TestMissingNode(t *testing.T) { updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, _ := trie.Commit() - ClearGlobalCache() - trie, _ = New(root, db) _, err := trie.TryGet([]byte("120000")) if err != nil { @@ -109,7 +107,6 @@ func TestMissingNode(t *testing.T) { } db.Delete(common.FromHex("e1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")) - ClearGlobalCache() trie, _ = New(root, db) _, err = trie.TryGet([]byte("120000")) @@ -362,44 +359,6 @@ func TestLargeValue(t *testing.T) { } -type kv struct { - k, v []byte - t bool -} - -func TestLargeData(t *testing.T) { - trie := newEmpty() - vals := make(map[string]*kv) - - for i := byte(0); i < 255; i++ { - value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} - value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false} - trie.Update(value.k, value.v) - trie.Update(value2.k, value2.v) - vals[string(value.k)] = value - vals[string(value2.k)] = value2 - } - - it := NewIterator(trie) - for it.Next() { - vals[string(it.Key)].t = true - } - - var untouched []*kv - for _, value := range vals { - if !value.t { - untouched = append(untouched, value) - } - } - - if len(untouched) > 0 { - t.Errorf("Missed %d nodes", len(untouched)) - for _, value := range untouched { - t.Error(value) - } - } -} - func BenchmarkGet(b *testing.B) { benchGet(b, false) } func BenchmarkGetDB(b *testing.B) { benchGet(b, true) } func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) } |