diff options
| -rw-r--r-- | .travis.yml | 8 | ||||
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | VERSION | 2 | ||||
| -rw-r--r-- | appveyor.yml | 4 | ||||
| -rw-r--r-- | build/ci.go | 19 | ||||
| -rw-r--r-- | cmd/geth/library.c | 24 | ||||
| -rw-r--r-- | cmd/geth/library.go | 46 | ||||
| -rw-r--r-- | cmd/geth/library_android.go | 56 | ||||
| -rw-r--r-- | cmd/swarm/hash.go (renamed from cmd/bzzhash/main.go) | 19 | ||||
| -rw-r--r-- | cmd/swarm/main.go (renamed from cmd/bzzd/main.go) | 105 | ||||
| -rw-r--r-- | cmd/swarm/upload.go (renamed from cmd/bzzup/main.go) | 108 | ||||
| -rw-r--r-- | cmd/utils/cmd.go | 31 | ||||
| -rw-r--r-- | core/tx_pool.go | 14 | ||||
| -rw-r--r-- | eth/api.go | 19 | ||||
| -rw-r--r-- | miner/unconfirmed.go | 118 | ||||
| -rw-r--r-- | miner/unconfirmed_test.go | 85 | ||||
| -rw-r--r-- | miner/worker.go | 88 | ||||
| -rw-r--r-- | params/version.go | 2 | ||||
| -rw-r--r-- | swarm/network/protocol.go | 2 |
19 files changed, 454 insertions, 300 deletions
diff --git a/.travis.yml b/.travis.yml index 87725251b..d13b77c17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,15 +13,15 @@ matrix: go: 1.6.2 - os: linux dist: trusty - go: 1.7 + go: 1.7.4 - os: osx - go: 1.7 + go: 1.7.4 # This builder does the Ubuntu PPA and Linux Azure uploads - os: linux dist: trusty sudo: required - go: 1.7 + go: 1.7.4 env: - ubuntu-ppa - azure-linux @@ -55,7 +55,7 @@ matrix: # This builder does the OSX Azure, Android Maven and Azure and iOS CocoaPods and Azure uploads - os: osx - go: 1.7 + go: 1.7.4 env: - azure-osx - mobile @@ -39,9 +39,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow insolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). | | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. | | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | -| `bzzd` | swarm daemon. This is the entrypoint for the swarm network. `bzzd --help` for command line options. See https://swarm-guide.readthedocs.io for swarm documentation. | -| `bzzup` | swarm command line file uploader. `bzzup --help` for command line options | -| `bzzhash` | command to calculate the swarm hash of a file or directory. `bzzhash --help` for command line options | +| `swarm` | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. | ## Running geth @@ -1 +1 @@ -1.5.5 +1.5.6 diff --git a/appveyor.yml b/appveyor.yml index dbdda9b6c..f5115f6f9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,8 +22,8 @@ environment: install: - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.3.windows-%GETH_ARCH%.zip - - 7z x go1.7.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-%GETH_ARCH%.zip + - 7z x go1.7.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/build/ci.go b/build/ci.go index 602eb8239..867fc3732 100644 --- a/build/ci.go +++ b/build/ci.go @@ -72,9 +72,7 @@ var ( executablePath("abigen"), executablePath("evm"), executablePath("geth"), - executablePath("bzzd"), - executablePath("bzzhash"), - executablePath("bzzup"), + executablePath("swarm"), executablePath("rlpdump"), } @@ -93,16 +91,8 @@ var ( Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", }, { - Name: "bzzd", - Description: "Ethereum Swarm daemon", - }, - { - Name: "bzzup", - Description: "Ethereum Swarm command line file/directory uploader", - }, - { - Name: "bzzhash", - Description: "Ethereum Swarm file/directory hash calculator", + Name: "swarm", + Description: "Ethereum Swarm daemon and tools", }, { Name: "abigen", @@ -112,7 +102,8 @@ var ( // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. - debDistros = []string{"trusty", "wily", "xenial", "yakkety"} + // Note: wily is unsupported because it was officially deprecated on lanchpad. + debDistros = []string{"trusty", "xenial", "yakkety"} ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) diff --git a/cmd/geth/library.c b/cmd/geth/library.c deleted file mode 100644 index f738621a8..000000000 --- a/cmd/geth/library.c +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. - -// Simple wrapper to translate the API exposed methods and types to inthernal -// Go versions of the same types. - -#include "_cgo_export.h" - -int run(const char* args) { - return doRun((char*)args); -} diff --git a/cmd/geth/library.go b/cmd/geth/library.go deleted file mode 100644 index b8ffaaed7..000000000 --- a/cmd/geth/library.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. - -// Contains a simple library definition to allow creating a Geth instance from -// straight C code. - -package main - -// #ifdef __cplusplus -// extern "C" { -// #endif -// -// extern int run(const char*); -// -// #ifdef __cplusplus -// } -// #endif -import "C" -import ( - "fmt" - "os" - "strings" -) - -//export doRun -func doRun(args *C.char) C.int { - // This is equivalent to geth.main, just modified to handle the function arg passing - if err := app.Run(strings.Split("geth "+C.GoString(args), " ")); err != nil { - fmt.Fprintln(os.Stderr, err) - return -1 - } - return 0 -} diff --git a/cmd/geth/library_android.go b/cmd/geth/library_android.go deleted file mode 100644 index fb021bfe0..000000000 --- a/cmd/geth/library_android.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. - -// Contains specialized code for running Geth on Android. - -package main - -// #include <android/log.h> -// #cgo LDFLAGS: -llog -import "C" -import ( - "bufio" - "os" -) - -func init() { - // Redirect the standard output and error to logcat - oldStdout, oldStderr := os.Stdout, os.Stderr - - outRead, outWrite, _ := os.Pipe() - errRead, errWrite, _ := os.Pipe() - - os.Stdout = outWrite - os.Stderr = errWrite - - go func() { - scanner := bufio.NewScanner(outRead) - for scanner.Scan() { - line := scanner.Text() - C.__android_log_write(C.ANDROID_LOG_INFO, C.CString("Stdout"), C.CString(line)) - oldStdout.WriteString(line + "\n") - } - }() - - go func() { - scanner := bufio.NewScanner(errRead) - for scanner.Scan() { - line := scanner.Text() - C.__android_log_write(C.ANDROID_LOG_INFO, C.CString("Stderr"), C.CString(line)) - oldStderr.WriteString(line + "\n") - } - }() -} diff --git a/cmd/bzzhash/main.go b/cmd/swarm/hash.go index 0ae99acc0..0a20bea82 100644 --- a/cmd/bzzhash/main.go +++ b/cmd/swarm/hash.go @@ -19,22 +19,21 @@ package main import ( "fmt" + "log" "os" - "runtime" "github.com/ethereum/go-ethereum/swarm/storage" + "gopkg.in/urfave/cli.v1" ) -func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - - if len(os.Args) < 2 { - fmt.Println("Usage: bzzhash <file name>") - os.Exit(0) +func hash(ctx *cli.Context) { + args := ctx.Args() + if len(args) < 1 { + log.Fatal("Usage: swarm hash <file name>") } - f, err := os.Open(os.Args[1]) + f, err := os.Open(args[0]) if err != nil { - fmt.Println("Error opening file " + os.Args[1]) + fmt.Println("Error opening file " + args[1]) os.Exit(1) } @@ -42,7 +41,7 @@ func main() { chunker := storage.NewTreeChunker(storage.NewChunkerParams()) key, err := chunker.Split(f, stat.Size(), nil, nil, nil) if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + log.Fatalf("%v\n", err) } else { fmt.Printf("%v\n", key) } diff --git a/cmd/bzzd/main.go b/cmd/swarm/main.go index 4bb2ca04a..04930760e 100644 --- a/cmd/bzzd/main.go +++ b/cmd/swarm/main.go @@ -43,11 +43,21 @@ import ( "gopkg.in/urfave/cli.v1" ) -const clientIdentifier = "bzzd" +const ( + clientIdentifier = "swarm" + versionString = "0.2" +) var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) - app = utils.NewApp(gitCommit, "Ethereum Swarm server daemon") + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + app = utils.NewApp(gitCommit, "Ethereum Swarm") + testbetBootNodes = []string{ + "enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429", + "enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430", + "enode://fe29b82319b734ce1ec68b84657d57145fee237387e63273989d354486731e59f78858e452ef800a020559da22dcca759536e6aa5517c53930d29ce0b1029286@13.74.157.139:30431", + "enode://1d7187e7bde45cf0bee489ce9852dd6d1a0d9aa67a33a6b8e6db8a4fbc6fcfa6f0f1a5419343671521b863b187d1c73bad3603bae66421d157ffef357669ddb8@13.74.157.139:30432", + "enode://0e4cba800f7b1ee73673afa6a4acead4018f0149d2e3216be3f133318fd165b324cd71b81fbe1e80deac8dbf56e57a49db7be67f8b9bc81bd2b7ee496434fb5d@13.74.157.139:30433", + } ) var ( @@ -65,7 +75,7 @@ var ( } SwarmNetworkIdFlag = cli.IntFlag{ Name: "bzznetworkid", - Usage: "Network identifier (integer, default 322=swarm testnet)", + Usage: "Network identifier (integer, default 3=swarm testnet)", Value: network.NetworkId, } SwarmConfigPathFlag = cli.StringFlag{ @@ -85,10 +95,25 @@ var ( Usage: "URL of the Ethereum API provider", Value: node.DefaultIPCEndpoint("geth"), } + SwarmApiFlag = cli.StringFlag{ + Name: "bzzapi", + Usage: "Swarm HTTP endpoint", + Value: "http://127.0.0.1:8500", + } + SwarmRecursiveUploadFlag = cli.BoolFlag{ + Name: "recursive", + Usage: "Upload directories recursively", + } + SwarmWantManifestFlag = cli.BoolTFlag{ + Name: "manifest", + Usage: "Automatic manifest upload", + } + SwarmUploadDefaultPath = cli.StringFlag{ + Name: "defaultpath", + Usage: "path to file served for empty url path (none)", + } ) -var defaultBootnodes = []string{} - func init() { // Override flag defaults so bzzd can run alongside geth. utils.ListenPortFlag.Value = 30399 @@ -96,8 +121,39 @@ func init() { utils.IPCApiFlag.Value = "admin, bzz, chequebook, debug, rpc, web3" // Set up the cli app. - app.Commands = nil app.Action = bzzd + app.HideVersion = true // we have a command to print the version + app.Copyright = "Copyright 2013-2016 The go-ethereum Authors" + app.Commands = []cli.Command{ + cli.Command{ + Action: version, + Name: "version", + Usage: "Print version numbers", + ArgsUsage: " ", + Description: ` +The output of this command is supposed to be machine-readable. +`, + }, + cli.Command{ + Action: upload, + Name: "up", + Usage: "upload a file or directory to swarm using the HTTP API", + ArgsUsage: " <file>", + Description: ` +"upload a file or directory to swarm using the HTTP API and prints the root hash", +`, + }, + cli.Command{ + Action: hash, + Name: "hash", + Usage: "print the swarm hash of a file or directory", + ArgsUsage: " <file>", + Description: ` +Prints the swarm hash of file or directory. +`, + }, + } + app.Flags = []cli.Flag{ utils.IdentityFlag, utils.DataDirFlag, @@ -123,6 +179,11 @@ func init() { SwarmAccountFlag, SwarmNetworkIdFlag, ChequebookAddrFlag, + // upload flags + SwarmApiFlag, + SwarmRecursiveUploadFlag, + SwarmWantManifestFlag, + SwarmUploadDefaultPath, } app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { @@ -142,17 +203,33 @@ func main() { } } +func version(ctx *cli.Context) error { + fmt.Println(strings.Title(clientIdentifier)) + fmt.Println("Version:", versionString) + if gitCommit != "" { + fmt.Println("Git Commit:", gitCommit) + } + fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name)) + fmt.Println("Go Version:", runtime.Version()) + fmt.Println("OS:", runtime.GOOS) + fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH")) + fmt.Printf("GOROOT=%s\n", runtime.GOROOT()) + return nil +} + func bzzd(ctx *cli.Context) error { stack := utils.MakeNode(ctx, clientIdentifier, gitCommit) registerBzzService(ctx, stack) utils.StartNode(stack) - + networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name) // Add bootnodes as initial peers. if ctx.GlobalIsSet(utils.BootnodesFlag.Name) { bootnodes := strings.Split(ctx.GlobalString(utils.BootnodesFlag.Name), ",") injectBootnodes(stack.Server(), bootnodes) } else { - injectBootnodes(stack.Server(), defaultBootnodes) + if networkId == 3 { + injectBootnodes(stack.Server(), testbetBootNodes) + } } stack.Wait() @@ -182,13 +259,11 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { boot := func(ctx *node.ServiceContext) (node.Service, error) { var client *ethclient.Client - if ethapi == "" { - err = fmt.Errorf("use ethapi flag to connect to a an eth client and talk to the blockchain") - } else { + if len(ethapi) > 0 { client, err = ethclient.Dial(ethapi) - } - if err != nil { - utils.Fatalf("Can't connect: %v", err) + if err != nil { + utils.Fatalf("Can't connect: %v", err) + } } return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled) } diff --git a/cmd/bzzup/main.go b/cmd/swarm/upload.go index 7d251aadb..d048bbc40 100644 --- a/cmd/bzzup/main.go +++ b/cmd/swarm/upload.go @@ -20,7 +20,6 @@ package main import ( "bytes" "encoding/json" - "flag" "fmt" "io" "io/ioutil" @@ -28,58 +27,83 @@ import ( "mime" "net/http" "os" + "os/user" + "path" "path/filepath" "strings" + + "gopkg.in/urfave/cli.v1" ) -func main() { +func upload(ctx *cli.Context) { + args := ctx.Args() var ( - bzzapiFlag = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint") - recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively") - manifestFlag = flag.Bool("manifest", true, "Skip automatic manifest upload") + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + recursive = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name) + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) ) - log.SetOutput(os.Stderr) - log.SetFlags(0) - flag.Parse() - if flag.NArg() != 1 { + if len(args) != 1 { log.Fatal("need filename as the first and only argument") } var ( - file = flag.Arg(0) - client = &client{api: *bzzapiFlag} + file = args[0] + client = &client{api: bzzapi} mroot manifest + entry manifestEntry ) - fi, err := os.Stat(file) + fi, err := os.Stat(expandPath(file)) if err != nil { log.Fatal(err) } if fi.IsDir() { - if !*recursiveFlag { + if !recursive { log.Fatal("argument is a directory and recursive upload is disabled") } - mroot, err = client.uploadDirectory(file) + mroot, err = client.uploadDirectory(file, defaultPath) } else { - mroot, err = client.uploadFile(file, fi) - if *manifestFlag { - // Wrap the raw file entry in a proper manifest so both hashes get printed. - mroot = manifest{Entries: []manifest{mroot}} - } + entry, err = client.uploadFile(file, fi) + mroot = manifest{[]manifestEntry{entry}} } if err != nil { log.Fatalln("upload failed:", err) } - if *manifestFlag { - hash, err := client.uploadManifest(mroot) - if err != nil { - log.Fatalln("manifest upload failed:", err) + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } + hash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + fmt.Println(hash) +} + +// Expands a file path +// 1. replace tilde with users home dir +// 2. expands embedded environment variables +// 3. cleans the path, e.g. /a/b/../c -> /a/c +// Note, it has limitations, e.g. ~someuser/tmp will not be expanded +func expandPath(p string) string { + if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { + if home := homeDir(); home != "" { + p = home + p[1:] } - mroot.Hash = hash } + return path.Clean(os.ExpandEnv(p)) +} - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) +func homeDir() string { + if home := os.Getenv("HOME"); home != "" { + return home + } + if usr, err := user.Current(); err == nil { + return usr.HomeDir + } + return "" } // client wraps interaction with the swarm HTTP gateway. @@ -88,24 +112,40 @@ type client struct { } // manifest is the JSON representation of a swarm manifest. +type manifestEntry struct { + Hash string `json:"hash,omitempty"` + ContentType string `json:"contentType,omitempty"` + Path string `json:"path,omitempty"` +} + +// manifest is the JSON representation of a swarm manifest. type manifest struct { - Hash string `json:"hash,omitempty"` - ContentType string `json:"contentType,omitempty"` - Path string `json:"path,omitempty"` - Entries []manifest `json:"entries,omitempty"` + Entries []manifestEntry `json:"entries,omitempty"` } -func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) { +func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) { hash, err := c.uploadFileContent(file, fi) - m := manifest{ + m := manifestEntry{ Hash: hash, ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())), } return m, err } -func (c *client) uploadDirectory(dir string) (manifest, error) { +func (c *client) uploadDirectory(dir string, defaultPath string) (manifest, error) { dirm := manifest{} + if len(defaultPath) > 0 { + fi, err := os.Stat(defaultPath) + if err != nil { + log.Fatal(err) + } + entry, err := c.uploadFile(defaultPath, fi) + if err != nil { + log.Fatal(err) + } + entry.Path = "" + dirm.Entries = append(dirm.Entries, entry) + } prefix := filepath.ToSlash(filepath.Clean(dir)) + "/" err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { if err != nil || fi.IsDir() { diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 584afc804..a56507e4d 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -18,12 +18,14 @@ package utils import ( + "compress/gzip" "fmt" "io" "os" "os/signal" "regexp" "runtime" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -133,7 +135,15 @@ func ImportChain(chain *core.BlockChain, fn string) error { return err } defer fh.Close() - stream := rlp.NewStream(fh, 0) + + var reader io.Reader = fh + if strings.HasSuffix(fn, ".gz") { + if reader, err = gzip.NewReader(reader); err != nil { + return err + } + } + + stream := rlp.NewStream(reader, 0) // Run actual the import. blocks := make(types.Blocks, importBatchSize) @@ -195,10 +205,18 @@ func ExportChain(blockchain *core.BlockChain, fn string) error { return err } defer fh.Close() - if err := blockchain.Export(fh); err != nil { + + var writer io.Writer = fh + if strings.HasSuffix(fn, ".gz") { + writer = gzip.NewWriter(writer) + defer writer.(*gzip.Writer).Close() + } + + if err := blockchain.Export(writer); err != nil { return err } glog.Infoln("Exported blockchain to ", fn) + return nil } @@ -210,7 +228,14 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return err } defer fh.Close() - if err := blockchain.ExportN(fh, first, last); err != nil { + + var writer io.Writer = fh + if strings.HasSuffix(fn, ".gz") { + writer = gzip.NewWriter(writer) + defer writer.(*gzip.Writer).Close() + } + + if err := blockchain.ExportN(writer, first, last); err != nil { return err } glog.Infoln("Exported blockchain to ", fn) diff --git a/core/tx_pool.go b/core/tx_pool.go index b805cf226..65e076df9 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -124,6 +124,8 @@ func NewTxPool(config *params.ChainConfig, eventMux *event.TypeMux, currentState quit: make(chan struct{}), } + pool.resetState() + pool.wg.Add(2) go pool.eventLoop() go pool.expirationLoop() @@ -196,12 +198,8 @@ func (pool *TxPool) Stop() { } func (pool *TxPool) State() *state.ManagedState { - pool.mu.Lock() - defer pool.mu.Unlock() - - if pool.pendingState == nil { - pool.resetState() - } + pool.mu.RLock() + defer pool.mu.RUnlock() return pool.pendingState } @@ -381,10 +379,6 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) { // // Note, this method assumes the pool lock is held! func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) { - // Init delayed since tx pool could have been started before any state sync - if pool.pendingState == nil { - pool.resetState() - } // Try to insert the transaction into the pending queue if pool.pending[addr] == nil { pool.pending[addr] = newTxList(true) diff --git a/eth/api.go b/eth/api.go index a86ed95cf..0a1d097e3 100644 --- a/eth/api.go +++ b/eth/api.go @@ -18,6 +18,7 @@ package eth import ( "bytes" + "compress/gzip" "errors" "fmt" "io" @@ -25,6 +26,7 @@ import ( "math/big" "os" "runtime" + "strings" "time" "github.com/ethereum/ethash" @@ -217,8 +219,14 @@ func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { } defer out.Close() + var writer io.Writer = out + if strings.HasSuffix(file, ".gz") { + writer = gzip.NewWriter(writer) + defer writer.(*gzip.Writer).Close() + } + // Export the blockchain - if err := api.eth.BlockChain().Export(out); err != nil { + if err := api.eth.BlockChain().Export(writer); err != nil { return false, err } return true, nil @@ -243,8 +251,15 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { } defer in.Close() + var reader io.Reader = in + if strings.HasSuffix(file, ".gz") { + if reader, err = gzip.NewReader(reader); err != nil { + return false, err + } + } + // Run actual the import in pre-configured batches - stream := rlp.NewStream(in, 0) + stream := rlp.NewStream(reader, 0) blocks, index := make([]*types.Block, 0, 2500), 0 for batch := 0; ; batch++ { diff --git a/miner/unconfirmed.go b/miner/unconfirmed.go new file mode 100644 index 000000000..86a30de35 --- /dev/null +++ b/miner/unconfirmed.go @@ -0,0 +1,118 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package miner + +import ( + "container/ring" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" +) + +// headerRetriever is used by the unconfirmed block set to verify whether a previously +// mined block is part of the canonical chain or not. +type headerRetriever interface { + // GetHeaderByNumber retrieves the canonical header associated with a block number. + GetHeaderByNumber(number uint64) *types.Header +} + +// unconfirmedBlock is a small collection of metadata about a locally mined block +// that is placed into a unconfirmed set for canonical chain inclusion tracking. +type unconfirmedBlock struct { + index uint64 + hash common.Hash +} + +// unconfirmedBlocks implements a data structure to maintain locally mined blocks +// have have not yet reached enough maturity to guarantee chain inclusion. It is +// used by the miner to provide logs to the user when a previously mined block +// has a high enough guarantee to not be reorged out of te canonical chain. +type unconfirmedBlocks struct { + chain headerRetriever // Blockchain to verify canonical status through + depth uint // Depth after which to discard previous blocks + blocks *ring.Ring // Block infos to allow canonical chain cross checks + lock sync.RWMutex // Protects the fields from concurrent access +} + +// newUnconfirmedBlocks returns new data structure to track currently unconfirmed blocks. +func newUnconfirmedBlocks(chain headerRetriever, depth uint) *unconfirmedBlocks { + return &unconfirmedBlocks{ + chain: chain, + depth: depth, + } +} + +// Insert adds a new block to the set of unconfirmed ones. +func (set *unconfirmedBlocks) Insert(index uint64, hash common.Hash) { + // If a new block was mined locally, shift out any old enough blocks + set.Shift(index) + + // Create the new item as its own ring + item := ring.New(1) + item.Value = &unconfirmedBlock{ + index: index, + hash: hash, + } + // Set as the initial ring or append to the end + set.lock.Lock() + defer set.lock.Unlock() + + if set.blocks == nil { + set.blocks = item + } else { + set.blocks.Move(-1).Link(item) + } + // Display a log for the user to notify of a new mined block unconfirmed + glog.V(logger.Info).Infof("🔨 mined potential block #%d [%x…], waiting for %d blocks to confirm", index, hash.Bytes()[:4], set.depth) +} + +// Shift drops all unconfirmed blocks from the set which exceed the unconfirmed sets depth +// allowance, checking them against the canonical chain for inclusion or staleness +// report. +func (set *unconfirmedBlocks) Shift(height uint64) { + set.lock.Lock() + defer set.lock.Unlock() + + for set.blocks != nil { + // Retrieve the next unconfirmed block and abort if too fresh + next := set.blocks.Value.(*unconfirmedBlock) + if next.index+uint64(set.depth) > height { + break + } + // Block seems to exceed depth allowance, check for canonical status + header := set.chain.GetHeaderByNumber(next.index) + switch { + case header == nil: + glog.V(logger.Warn).Infof("failed to retrieve header of mined block #%d [%x…]", next.index, next.hash.Bytes()[:4]) + case header.Hash() == next.hash: + glog.V(logger.Info).Infof("🔗 mined block #%d [%x…] reached canonical chain", next.index, next.hash.Bytes()[:4]) + default: + glog.V(logger.Info).Infof("⑂ mined block #%d [%x…] became a side fork", next.index, next.hash.Bytes()[:4]) + } + // Drop the block out of the ring + if set.blocks.Value == set.blocks.Next().Value { + set.blocks = nil + } else { + set.blocks = set.blocks.Move(-1) + set.blocks.Unlink(1) + set.blocks = set.blocks.Move(1) + } + } +} diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go new file mode 100644 index 000000000..456af1764 --- /dev/null +++ b/miner/unconfirmed_test.go @@ -0,0 +1,85 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package miner + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// noopHeaderRetriever is an implementation of headerRetriever that always +// returns nil for any requested headers. +type noopHeaderRetriever struct{} + +func (r *noopHeaderRetriever) GetHeaderByNumber(number uint64) *types.Header { + return nil +} + +// Tests that inserting blocks into the unconfirmed set accumulates them until +// the desired depth is reached, after which they begin to be dropped. +func TestUnconfirmedInsertBounds(t *testing.T) { + limit := uint(10) + + pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit) + for depth := uint64(0); depth < 2*uint64(limit); depth++ { + // Insert multiple blocks for the same level just to stress it + for i := 0; i < int(depth); i++ { + pool.Insert(depth, common.Hash([32]byte{byte(depth), byte(i)})) + } + // Validate that no blocks below the depth allowance are left in + pool.blocks.Do(func(block interface{}) { + if block := block.(*unconfirmedBlock); block.index+uint64(limit) <= depth { + t.Errorf("depth %d: block %x not dropped", depth, block.hash) + } + }) + } +} + +// Tests that shifting blocks out of the unconfirmed set works both for normal +// cases as well as for corner cases such as empty sets, empty shifts or full +// shifts. +func TestUnconfirmedShifts(t *testing.T) { + // Create a pool with a few blocks on various depths + limit, start := uint(10), uint64(25) + + pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit) + for depth := start; depth < start+uint64(limit); depth++ { + pool.Insert(depth, common.Hash([32]byte{byte(depth)})) + } + // Try to shift below the limit and ensure no blocks are dropped + pool.Shift(start + uint64(limit) - 1) + if n := pool.blocks.Len(); n != int(limit) { + t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit) + } + // Try to shift half the blocks out and verify remainder + pool.Shift(start + uint64(limit) - 1 + uint64(limit/2)) + if n := pool.blocks.Len(); n != int(limit)/2 { + t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit/2) + } + // Try to shift all the remaining blocks out and verify emptyness + pool.Shift(start + 2*uint64(limit)) + if n := pool.blocks.Len(); n != 0 { + t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0) + } + // Try to shift out from the empty set and make sure it doesn't break + pool.Shift(start + 3*uint64(limit)) + if n := pool.blocks.Len(); n != 0 { + t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0) + } +} diff --git a/miner/worker.go b/miner/worker.go index fdc6b7d8e..f29566c0a 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -55,26 +55,20 @@ type Agent interface { GetHashRate() int64 } -type uint64RingBuffer struct { - ints []uint64 //array of all integers in buffer - next int //where is the next insertion? assert 0 <= next < len(ints) -} - // Work is the workers current environment and holds // all of the current state information type Work struct { config *params.ChainConfig signer types.Signer - state *state.StateDB // apply state changes here - ancestors *set.Set // ancestor set (used for checking uncle parent validity) - family *set.Set // family set (used for checking uncle invalidity) - uncles *set.Set // uncle set - tcount int // tx count in cycle - ownedAccounts *set.Set - lowGasTxs types.Transactions - failedTxs types.Transactions - localMinedBlocks *uint64RingBuffer // the most recent block numbers that were mined locally (used to check block inclusion) + state *state.StateDB // apply state changes here + ancestors *set.Set // ancestor set (used for checking uncle parent validity) + family *set.Set // family set (used for checking uncle invalidity) + uncles *set.Set // uncle set + tcount int // tx count in cycle + ownedAccounts *set.Set + lowGasTxs types.Transactions + failedTxs types.Transactions Block *types.Block // the new block @@ -123,6 +117,8 @@ type worker struct { txQueueMu sync.Mutex txQueue map[common.Hash]*types.Transaction + unconfirmed *unconfirmedBlocks // set of locally mined blocks pending canonicalness confirmations + // atomic status counters mining int32 atWork int32 @@ -144,6 +140,7 @@ func newWorker(config *params.ChainConfig, coinbase common.Address, eth Backend, coinbase: coinbase, txQueue: make(map[common.Hash]*types.Transaction), agents: make(map[Agent]struct{}), + unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), 5), fullValidation: false, } worker.events = worker.mux.Subscribe(core.ChainHeadEvent{}, core.ChainSideEvent{}, core.TxPreEvent{}) @@ -269,18 +266,6 @@ func (self *worker) update() { } } -func newLocalMinedBlock(blockNumber uint64, prevMinedBlocks *uint64RingBuffer) (minedBlocks *uint64RingBuffer) { - if prevMinedBlocks == nil { - minedBlocks = &uint64RingBuffer{next: 0, ints: make([]uint64, miningLogAtDepth+1)} - } else { - minedBlocks = prevMinedBlocks - } - - minedBlocks.ints[minedBlocks.next] = blockNumber - minedBlocks.next = (minedBlocks.next + 1) % len(minedBlocks.ints) - return minedBlocks -} - func (self *worker) wait() { for { mustCommitNewWork := true @@ -355,17 +340,8 @@ func (self *worker) wait() { } }(block, work.state.Logs(), work.receipts) } - - // check staleness and display confirmation - var stale, confirm string - canonBlock := self.chain.GetBlockByNumber(block.NumberU64()) - if canonBlock != nil && canonBlock.Hash() != block.Hash() { - stale = "stale " - } else { - confirm = "Wait 5 blocks for confirmation" - work.localMinedBlocks = newLocalMinedBlock(block.Number().Uint64(), work.localMinedBlocks) - } - glog.V(logger.Info).Infof("🔨 Mined %sblock (#%v / %x). %s", stale, block.Number(), block.Hash().Bytes()[:4], confirm) + // Insert the block into the set of pending ones to wait for confirmations + self.unconfirmed.Insert(block.NumberU64(), block.Hash()) if mustCommitNewWork { self.commitNewWork() @@ -417,9 +393,6 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error // Keep track of transactions which return errors so they can be removed work.tcount = 0 work.ownedAccounts = accountAddressesSet(accounts) - if self.current != nil { - work.localMinedBlocks = self.current.localMinedBlocks - } self.current = work return nil } @@ -435,38 +408,6 @@ func (w *worker) setGasPrice(p *big.Int) { w.mux.Post(core.GasPriceChanged{Price: w.gasPrice}) } -func (self *worker) isBlockLocallyMined(current *Work, deepBlockNum uint64) bool { - //Did this instance mine a block at {deepBlockNum} ? - var isLocal = false - for idx, blockNum := range current.localMinedBlocks.ints { - if deepBlockNum == blockNum { - isLocal = true - current.localMinedBlocks.ints[idx] = 0 //prevent showing duplicate logs - break - } - } - //Short-circuit on false, because the previous and following tests must both be true - if !isLocal { - return false - } - - //Does the block at {deepBlockNum} send earnings to my coinbase? - var block = self.chain.GetBlockByNumber(deepBlockNum) - return block != nil && block.Coinbase() == self.coinbase -} - -func (self *worker) logLocalMinedBlocks(current, previous *Work) { - if previous != nil && current.localMinedBlocks != nil { - nextBlockNum := current.Block.NumberU64() - for checkBlockNum := previous.Block.NumberU64(); checkBlockNum < nextBlockNum; checkBlockNum++ { - inspectBlockNum := checkBlockNum - miningLogAtDepth - if self.isBlockLocallyMined(current, inspectBlockNum) { - glog.V(logger.Info).Infof("🔨 🔗 Mined %d blocks back: block #%v", miningLogAtDepth, inspectBlockNum) - } - } - } -} - func (self *worker) commitNewWork() { self.mu.Lock() defer self.mu.Unlock() @@ -513,7 +454,6 @@ func (self *worker) commitNewWork() { } } } - previous := self.current // Could potentially happen if starting to mine in an odd state. err := self.makeCurrent(parent, header) if err != nil { @@ -574,7 +514,7 @@ func (self *worker) commitNewWork() { // We only care about logging if we're actually mining. if atomic.LoadInt32(&self.mining) == 1 { glog.V(logger.Info).Infof("commit new work on block %v with %d txs & %d uncles. Took %v\n", work.Block.Number(), work.tcount, len(uncles), time.Since(tstart)) - self.logLocalMinedBlocks(work, previous) + self.unconfirmed.Shift(work.Block.NumberU64() - 1) } self.push(work) } diff --git a/params/version.go b/params/version.go index f8c0d3c9a..ba7a57381 100644 --- a/params/version.go +++ b/params/version.go @@ -21,7 +21,7 @@ import "fmt" const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 5 // Minor version component of the current release - VersionPatch = 5 // Patch version component of the current release + VersionPatch = 6 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index a3ffd338f..4fffaac6d 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -51,7 +51,7 @@ const ( Version = 0 ProtocolLength = uint64(8) ProtocolMaxMsgSize = 10 * 1024 * 1024 - NetworkId = 322 + NetworkId = 3 ) const ( |