diff options
66 files changed, 2323 insertions, 1086 deletions
diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index f2694a05c..36104b456 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,7 @@ { "ImportPath": "github.com/ethereum/go-ethereum", "GoVersion": "go1.5.2", + "GodepVersion": "v74", "Packages": [ "./..." ], @@ -14,11 +15,6 @@ "Rev": "165db2f241fd235aec29ba6d9b1ccd5f1c14637c" }, { - "ImportPath": "github.com/codegangsta/cli", - "Comment": "1.2.0-215-g0ab42fd", - "Rev": "0ab42fd482c27cf2c95e7794ad3bb2082c2ab2d7" - }, - { "ImportPath": "github.com/davecgh/go-spew/spew", "Rev": "5215b55f46b2b919f50a1df0eaa5886afe4e3b3d" }, @@ -156,6 +152,10 @@ "Rev": "5950cf11d77f8a61b432a25dd4d444b4ced01379" }, { + "ImportPath": "github.com/rs/xhandler", + "Rev": "d9d9599b6aaf6a058cb7b1f48291ded2cbd13390" + }, + { "ImportPath": "github.com/syndtr/goleveldb/leveldb", "Rev": "917f41c560270110ceb73c5b38be2a9127387071" }, @@ -319,6 +319,11 @@ { "ImportPath": "gopkg.in/karalabe/cookiejar.v2/collections/prque", "Rev": "8dcd6a7f4951f6ff3ee9cbb919a06d8925822e57" + }, + { + "ImportPath": "gopkg.in/urfave/cli.v1", + "Comment": "v1.17.0", + "Rev": "01857ac33766ce0c93856370626f9799281c14f4" } ] } diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete b/Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete deleted file mode 100644 index 21a232f1f..000000000 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -: ${PROG:=$(basename ${BASH_SOURCE})} - -_cli_bash_autocomplete() { - local cur opts base - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 - } - - complete -F _cli_bash_autocomplete $PROG diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete b/Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete deleted file mode 100644 index 5430a18f9..000000000 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete +++ /dev/null @@ -1,5 +0,0 @@ -autoload -U compinit && compinit -autoload -U bashcompinit && bashcompinit - -script_dir=$(dirname $0) -source ${script_dir}/bash_autocomplete diff --git a/Godeps/_workspace/src/github.com/rs/xhandler/.travis.yml b/Godeps/_workspace/src/github.com/rs/xhandler/.travis.yml deleted file mode 100644 index b65c7a9f1..000000000 --- a/Godeps/_workspace/src/github.com/rs/xhandler/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -go: -- 1.5 -- tip -matrix: - allow_failures: - - go: tip diff --git a/Godeps/_workspace/src/github.com/rs/xhandler/LICENSE b/Godeps/_workspace/src/github.com/rs/xhandler/LICENSE deleted file mode 100644 index 47c5e9d2d..000000000 --- a/Godeps/_workspace/src/github.com/rs/xhandler/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2015 Olivier Poitrey <rs@dailymotion.com> - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is furnished -to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/rs/xhandler/README.md b/Godeps/_workspace/src/github.com/rs/xhandler/README.md deleted file mode 100644 index 91c594bd2..000000000 --- a/Godeps/_workspace/src/github.com/rs/xhandler/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# XHandler - -[](https://godoc.org/github.com/rs/xhandler) [](https://raw.githubusercontent.com/rs/xhandler/master/LICENSE) [](https://travis-ci.org/rs/xhandler) [](http://gocover.io/github.com/rs/xhandler) - -XHandler is a bridge between [net/context](https://godoc.org/golang.org/x/net/context) and `http.Handler`. - -It lets you enforce `net/context` in your handlers without sacrificing compatibility with existing `http.Handlers` nor imposing a specific router. - -Thanks to `net/context` deadline management, `xhandler` is able to enforce a per request deadline and will cancel the context when the client closes the connection unexpectedly. - -You may create your own `net/context` aware handler pretty much the same way as you would do with http.Handler. - -Read more about xhandler on [Dailymotion engineering blog](http://engineering.dailymotion.com/our-way-to-go/). - -## Installing - - go get -u github.com/rs/xhandler - -## Usage - -```go -package main - -import ( - "log" - "net/http" - "time" - - "github.com/rs/cors" - "github.com/rs/xhandler" - "golang.org/x/net/context" -) - -type myMiddleware struct { - next xhandler.HandlerC -} - -func (h myMiddleware) ServeHTTPC(ctx context.Context, w http.ResponseWriter, r *http.Request) { - ctx = context.WithValue(ctx, "test", "World") - h.next.ServeHTTPC(ctx, w, r) -} - -func main() { - c := xhandler.Chain{} - - // Add close notifier handler so context is cancelled when the client closes - // the connection - c.UseC(xhandler.CloseHandler) - - // Add timeout handler - c.UseC(xhandler.TimeoutHandler(2 * time.Second)) - - // Middleware putting something in the context - c.UseC(func(next xhandler.HandlerC) xhandler.HandlerC { - return myMiddleware{next: next} - }) - - // Mix it with a non-context-aware middleware handler - c.Use(cors.Default().Handler) - - // Final handler (using handlerFuncC), reading from the context - xh := xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - value := ctx.Value("test").(string) - w.Write([]byte("Hello " + value)) - }) - - // Bridge context aware handlers with http.Handler using xhandler.Handle() - http.Handle("/test", c.Handler(xh)) - - if err := http.ListenAndServe(":8080", nil); err != nil { - log.Fatal(err) - } -} -``` - -### Using xmux - -Xhandler comes with an optional context aware [muxer](https://github.com/rs/xmux) forked from [httprouter](https://github.com/julienschmidt/httprouter): - -```go -package main - -import ( - "fmt" - "log" - "net/http" - "time" - - "github.com/rs/xhandler" - "github.com/rs/xmux" - "golang.org/x/net/context" -) - -func main() { - c := xhandler.Chain{} - - // Append a context-aware middleware handler - c.UseC(xhandler.CloseHandler) - - // Another context-aware middleware handler - c.UseC(xhandler.TimeoutHandler(2 * time.Second)) - - mux := xmux.New() - - // Use c.Handler to terminate the chain with your final handler - mux.GET("/welcome/:name", xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome %s!", xmux.Params(ctx).Get("name")) - })) - - if err := http.ListenAndServe(":8080", c.Handler(mux)); err != nil { - log.Fatal(err) - } -} -``` - -See [xmux](https://github.com/rs/xmux) for more examples. - -## Context Aware Middleware - -Here is a list of `net/context` aware middleware handlers implementing `xhandler.HandlerC` interface. - -Feel free to put up a PR linking your middleware if you have built one: - -| Middleware | Author | Description | -| ---------- | ------ | ----------- | -| [xmux](https://github.com/rs/xmux) | [Olivier Poitrey](https://github.com/rs) | HTTP request muxer | -| [xlog](https://github.com/rs/xlog) | [Olivier Poitrey](https://github.com/rs) | HTTP handler logger | -| [xstats](https://github.com/rs/xstats) | [Olivier Poitrey](https://github.com/rs) | A generic client for service instrumentation | -| [xaccess](https://github.com/rs/xaccess) | [Olivier Poitrey](https://github.com/rs) | HTTP handler access logger with [xlog](https://github.com/rs/xlog) and [xstats](https://github.com/rs/xstats) | -| [cors](https://github.com/rs/cors) | [Olivier Poitrey](https://github.com/rs) | [Cross Origin Resource Sharing](http://www.w3.org/TR/cors/) (CORS) support | - -## Licenses - -All source code is licensed under the [MIT License](https://raw.github.com/rs/xhandler/master/LICENSE). diff --git a/Godeps/_workspace/src/github.com/rs/xhandler/chain.go b/Godeps/_workspace/src/github.com/rs/xhandler/chain.go deleted file mode 100644 index 042274d17..000000000 --- a/Godeps/_workspace/src/github.com/rs/xhandler/chain.go +++ /dev/null @@ -1,93 +0,0 @@ -package xhandler - -import ( - "net/http" - - "golang.org/x/net/context" -) - -// Chain is an helper to chain middleware handlers together for an easier -// management. -type Chain []func(next HandlerC) HandlerC - -// UseC appends a context-aware handler to the middleware chain. -func (c *Chain) UseC(f func(next HandlerC) HandlerC) { - *c = append(*c, f) -} - -// Use appends a standard http.Handler to the middleware chain without -// lossing track of the context when inserted between two context aware handlers. -// -// Caveat: the f function will be called on each request so you are better to put -// any initialization sequence outside of this function. -func (c *Chain) Use(f func(next http.Handler) http.Handler) { - xf := func(next HandlerC) HandlerC { - return HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - n := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - next.ServeHTTPC(ctx, w, r) - }) - f(n).ServeHTTP(w, r) - }) - } - *c = append(*c, xf) -} - -// Handler wraps the provided final handler with all the middleware appended to -// the chain and return a new standard http.Handler instance. -// The context.Background() context is injected automatically. -func (c Chain) Handler(xh HandlerC) http.Handler { - ctx := context.Background() - return c.HandlerCtx(ctx, xh) -} - -// HandlerFC is an helper to provide a function (HandlerFuncC) to Handler(). -// -// HandlerFC is equivalent to: -// c.Handler(xhandler.HandlerFuncC(xhc)) -func (c Chain) HandlerFC(xhf HandlerFuncC) http.Handler { - ctx := context.Background() - return c.HandlerCtx(ctx, HandlerFuncC(xhf)) -} - -// HandlerH is an helper to provide a standard http handler (http.HandlerFunc) -// to Handler(). Your final handler won't have access the context though. -func (c Chain) HandlerH(h http.Handler) http.Handler { - ctx := context.Background() - return c.HandlerCtx(ctx, HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - h.ServeHTTP(w, r) - })) -} - -// HandlerF is an helper to provide a standard http handler function -// (http.HandlerFunc) to Handler(). Your final handler won't have access -// the context though. -func (c Chain) HandlerF(hf http.HandlerFunc) http.Handler { - ctx := context.Background() - return c.HandlerCtx(ctx, HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - hf(w, r) - })) -} - -// HandlerCtx wraps the provided final handler with all the middleware appended to -// the chain and return a new standard http.Handler instance. -func (c Chain) HandlerCtx(ctx context.Context, xh HandlerC) http.Handler { - return New(ctx, c.HandlerC(xh)) -} - -// HandlerC wraps the provided final handler with all the middleware appended to -// the chain and returns a HandlerC instance. -func (c Chain) HandlerC(xh HandlerC) HandlerC { - for i := len(c) - 1; i >= 0; i-- { - xh = c[i](xh) - } - return xh -} - -// HandlerCF wraps the provided final handler func with all the middleware appended to -// the chain and returns a HandlerC instance. -// -// HandlerCF is equivalent to: -// c.HandlerC(xhandler.HandlerFuncC(xhc)) -func (c Chain) HandlerCF(xhc HandlerFuncC) HandlerC { - return c.HandlerC(HandlerFuncC(xhc)) -} diff --git a/Godeps/_workspace/src/github.com/rs/xhandler/middleware.go b/Godeps/_workspace/src/github.com/rs/xhandler/middleware.go deleted file mode 100644 index 5de136419..000000000 --- a/Godeps/_workspace/src/github.com/rs/xhandler/middleware.go +++ /dev/null @@ -1,59 +0,0 @@ -package xhandler - -import ( - "net/http" - "time" - - "golang.org/x/net/context" -) - -// CloseHandler returns a Handler cancelling the context when the client -// connection close unexpectedly. -func CloseHandler(next HandlerC) HandlerC { - return HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - // Cancel the context if the client closes the connection - if wcn, ok := w.(http.CloseNotifier); ok { - var cancel context.CancelFunc - ctx, cancel = context.WithCancel(ctx) - defer cancel() - - notify := wcn.CloseNotify() - go func() { - select { - case <-notify: - cancel() - case <-ctx.Done(): - } - }() - } - - next.ServeHTTPC(ctx, w, r) - }) -} - -// TimeoutHandler returns a Handler which adds a timeout to the context. -// -// Child handlers have the responsability to obey the context deadline and to return -// an appropriate error (or not) response in case of timeout. -func TimeoutHandler(timeout time.Duration) func(next HandlerC) HandlerC { - return func(next HandlerC) HandlerC { - return HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - ctx, _ = context.WithTimeout(ctx, timeout) - next.ServeHTTPC(ctx, w, r) - }) - } -} - -// If is a special handler that will skip insert the condNext handler only if a condition -// applies at runtime. -func If(cond func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool, condNext func(next HandlerC) HandlerC) func(next HandlerC) HandlerC { - return func(next HandlerC) HandlerC { - return HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) { - if cond(ctx, w, r) { - condNext(next).ServeHTTPC(ctx, w, r) - } else { - next.ServeHTTPC(ctx, w, r) - } - }) - } -} diff --git a/Godeps/_workspace/src/github.com/rs/xhandler/xhandler.go b/Godeps/_workspace/src/github.com/rs/xhandler/xhandler.go deleted file mode 100644 index 718c25322..000000000 --- a/Godeps/_workspace/src/github.com/rs/xhandler/xhandler.go +++ /dev/null @@ -1,42 +0,0 @@ -// Package xhandler provides a bridge between http.Handler and net/context. -// -// xhandler enforces net/context in your handlers without sacrificing -// compatibility with existing http.Handlers nor imposing a specific router. -// -// Thanks to net/context deadline management, xhandler is able to enforce -// a per request deadline and will cancel the context in when the client close -// the connection unexpectedly. -// -// You may create net/context aware middlewares pretty much the same way as -// you would do with http.Handler. -package xhandler - -import ( - "net/http" - - "golang.org/x/net/context" -) - -// HandlerC is a net/context aware http.Handler -type HandlerC interface { - ServeHTTPC(context.Context, http.ResponseWriter, *http.Request) -} - -// HandlerFuncC type is an adapter to allow the use of ordinary functions -// as a xhandler.Handler. If f is a function with the appropriate signature, -// xhandler.HandlerFuncC(f) is a xhandler.Handler object that calls f. -type HandlerFuncC func(context.Context, http.ResponseWriter, *http.Request) - -// ServeHTTPC calls f(ctx, w, r). -func (f HandlerFuncC) ServeHTTPC(ctx context.Context, w http.ResponseWriter, r *http.Request) { - f(ctx, w, r) -} - -// New creates a conventional http.Handler injecting the provided root -// context to sub handlers. This handler is used as a bridge between conventional -// http.Handler and context aware handlers. -func New(ctx context.Context, h HandlerC) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h.ServeHTTPC(ctx, w, r) - }) -} diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/.travis.yml b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/.travis.yml index 87ba52f98..76f38a482 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/.travis.yml +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/.travis.yml @@ -2,18 +2,22 @@ language: go sudo: false go: -- 1.0.3 - 1.1.2 - 1.2.2 - 1.3.3 -- 1.4.2 -- 1.5.1 +- 1.4 +- 1.5.4 +- 1.6.2 - tip matrix: allow_failures: - go: tip +before_script: +- go get github.com/meatballhat/gfmxr/... + script: - go vet ./... - go test -v ./... +- gfmxr -c $(grep -c 'package main' README.md) -s README.md diff --git a/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/CHANGELOG.md b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/CHANGELOG.md new file mode 100644 index 000000000..f623e59b7 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/CHANGELOG.md @@ -0,0 +1,310 @@ +# Change Log + +**ATTN**: This project uses [semantic versioning](http://semver.org/). + +## [Unreleased] + +## [1.17.0] - 2016-05-09 +### Added +- Pluggable flag-level help text rendering via `cli.DefaultFlagStringFunc` +- `context.GlobalBoolT` was added as an analogue to `context.GlobalBool` +- Support for hiding commands by setting `Hidden: true` -- this will hide the + commands in help output + +### Changed +- `Float64Flag`, `IntFlag`, and `DurationFlag` default values are no longer + quoted in help text output. +- All flag types now include `(default: {value})` strings following usage when a + default value can be (reasonably) detected. +- `IntSliceFlag` and `StringSliceFlag` usage strings are now more consistent + with non-slice flag types +- Apps now exit with a code of 3 if an unknown subcommand is specified + (previously they printed "No help topic for...", but still exited 0. This + makes it easier to script around apps built using `cli` since they can trust + that a 0 exit code indicated a successful execution. +- cleanups based on [Go Report Card + feedback](https://goreportcard.com/report/github.com/codegangsta/cli) + +## [1.16.0] - 2016-05-02 +### Added +- `Hidden` field on all flag struct types to omit from generated help text + +### Changed +- `BashCompletionFlag` (`--enable-bash-completion`) is now omitted from +generated help text via the `Hidden` field + +### Fixed +- handling of error values in `HandleAction` and `HandleExitCoder` + +## [1.15.0] - 2016-04-30 +### Added +- This file! +- Support for placeholders in flag usage strings +- `App.Metadata` map for arbitrary data/state management +- `Set` and `GlobalSet` methods on `*cli.Context` for altering values after +parsing. +- Support for nested lookup of dot-delimited keys in structures loaded from +YAML. + +### Changed +- The `App.Action` and `Command.Action` now prefer a return signature of +`func(*cli.Context) error`, as defined by `cli.ActionFunc`. If a non-nil +`error` is returned, there may be two outcomes: + - If the error fulfills `cli.ExitCoder`, then `os.Exit` will be called + automatically + - Else the error is bubbled up and returned from `App.Run` +- Specifying an `Action` with the legacy return signature of +`func(*cli.Context)` will produce a deprecation message to stderr +- Specifying an `Action` that is not a `func` type will produce a non-zero exit +from `App.Run` +- Specifying an `Action` func that has an invalid (input) signature will +produce a non-zero exit from `App.Run` + +### Deprecated +- <a name="deprecated-cli-app-runandexitonerror"></a> +`cli.App.RunAndExitOnError`, which should now be done by returning an error +that fulfills `cli.ExitCoder` to `cli.App.Run`. +- <a name="deprecated-cli-app-action-signature"></a> the legacy signature for +`cli.App.Action` of `func(*cli.Context)`, which should now have a return +signature of `func(*cli.Context) error`, as defined by `cli.ActionFunc`. + +### Fixed +- Added missing `*cli.Context.GlobalFloat64` method + +## [1.14.0] - 2016-04-03 (backfilled 2016-04-25) +### Added +- Codebeat badge +- Support for categorization via `CategorizedHelp` and `Categories` on app. + +### Changed +- Use `filepath.Base` instead of `path.Base` in `Name` and `HelpName`. + +### Fixed +- Ensure version is not shown in help text when `HideVersion` set. + +## [1.13.0] - 2016-03-06 (backfilled 2016-04-25) +### Added +- YAML file input support. +- `NArg` method on context. + +## [1.12.0] - 2016-02-17 (backfilled 2016-04-25) +### Added +- Custom usage error handling. +- Custom text support in `USAGE` section of help output. +- Improved help messages for empty strings. +- AppVeyor CI configuration. + +### Changed +- Removed `panic` from default help printer func. +- De-duping and optimizations. + +### Fixed +- Correctly handle `Before`/`After` at command level when no subcommands. +- Case of literal `-` argument causing flag reordering. +- Environment variable hints on Windows. +- Docs updates. + +## [1.11.1] - 2015-12-21 (backfilled 2016-04-25) +### Changed +- Use `path.Base` in `Name` and `HelpName` +- Export `GetName` on flag types. + +### Fixed +- Flag parsing when skipping is enabled. +- Test output cleanup. +- Move completion check to account for empty input case. + +## [1.11.0] - 2015-11-15 (backfilled 2016-04-25) +### Added +- Destination scan support for flags. +- Testing against `tip` in Travis CI config. + +### Changed +- Go version in Travis CI config. + +### Fixed +- Removed redundant tests. +- Use correct example naming in tests. + +## [1.10.2] - 2015-10-29 (backfilled 2016-04-25) +### Fixed +- Remove unused var in bash completion. + +## [1.10.1] - 2015-10-21 (backfilled 2016-04-25) +### Added +- Coverage and reference logos in README. + +### Fixed +- Use specified values in help and version parsing. +- Only display app version and help message once. + +## [1.10.0] - 2015-10-06 (backfilled 2016-04-25) +### Added +- More tests for existing functionality. +- `ArgsUsage` at app and command level for help text flexibility. + +### Fixed +- Honor `HideHelp` and `HideVersion` in `App.Run`. +- Remove juvenile word from README. + +## [1.9.0] - 2015-09-08 (backfilled 2016-04-25) +### Added +- `FullName` on command with accompanying help output update. +- Set default `$PROG` in bash completion. + +### Changed +- Docs formatting. + +### Fixed +- Removed self-referential imports in tests. + +## [1.8.0] - 2015-06-30 (backfilled 2016-04-25) +### Added +- Support for `Copyright` at app level. +- `Parent` func at context level to walk up context lineage. + +### Fixed +- Global flag processing at top level. + +## [1.7.1] - 2015-06-11 (backfilled 2016-04-25) +### Added +- Aggregate errors from `Before`/`After` funcs. +- Doc comments on flag structs. +- Include non-global flags when checking version and help. +- Travis CI config updates. + +### Fixed +- Ensure slice type flags have non-nil values. +- Collect global flags from the full command hierarchy. +- Docs prose. + +## [1.7.0] - 2015-05-03 (backfilled 2016-04-25) +### Changed +- `HelpPrinter` signature includes output writer. + +### Fixed +- Specify go 1.1+ in docs. +- Set `Writer` when running command as app. + +## [1.6.0] - 2015-03-23 (backfilled 2016-04-25) +### Added +- Multiple author support. +- `NumFlags` at context level. +- `Aliases` at command level. + +### Deprecated +- `ShortName` at command level. + +### Fixed +- Subcommand help output. +- Backward compatible support for deprecated `Author` and `Email` fields. +- Docs regarding `Names`/`Aliases`. + +## [1.5.0] - 2015-02-20 (backfilled 2016-04-25) +### Added +- `After` hook func support at app and command level. + +### Fixed +- Use parsed context when running command as subcommand. +- Docs prose. + +## [1.4.1] - 2015-01-09 (backfilled 2016-04-25) +### Added +- Support for hiding `-h / --help` flags, but not `help` subcommand. +- Stop flag parsing after `--`. + +### Fixed +- Help text for generic flags to specify single value. +- Use double quotes in output for defaults. +- Use `ParseInt` instead of `ParseUint` for int environment var values. +- Use `0` as base when parsing int environment var values. + +## [1.4.0] - 2014-12-12 (backfilled 2016-04-25) +### Added +- Support for environment variable lookup "cascade". +- Support for `Stdout` on app for output redirection. + +### Fixed +- Print command help instead of app help in `ShowCommandHelp`. + +## [1.3.1] - 2014-11-13 (backfilled 2016-04-25) +### Added +- Docs and example code updates. + +### Changed +- Default `-v / --version` flag made optional. + +## [1.3.0] - 2014-08-10 (backfilled 2016-04-25) +### Added +- `FlagNames` at context level. +- Exposed `VersionPrinter` var for more control over version output. +- Zsh completion hook. +- `AUTHOR` section in default app help template. +- Contribution guidelines. +- `DurationFlag` type. + +## [1.2.0] - 2014-08-02 +### Added +- Support for environment variable defaults on flags plus tests. + +## [1.1.0] - 2014-07-15 +### Added +- Bash completion. +- Optional hiding of built-in help command. +- Optional skipping of flag parsing at command level. +- `Author`, `Email`, and `Compiled` metadata on app. +- `Before` hook func support at app and command level. +- `CommandNotFound` func support at app level. +- Command reference available on context. +- `GenericFlag` type. +- `Float64Flag` type. +- `BoolTFlag` type. +- `IsSet` flag helper on context. +- More flag lookup funcs at context level. +- More tests & docs. + +### Changed +- Help template updates to account for presence/absence of flags. +- Separated subcommand help template. +- Exposed `HelpPrinter` var for more control over help output. + +## [1.0.0] - 2013-11-01 +### Added +- `help` flag in default app flag set and each command flag set. +- Custom handling of argument parsing errors. +- Command lookup by name at app level. +- `StringSliceFlag` type and supporting `StringSlice` type. +- `IntSliceFlag` type and supporting `IntSlice` type. +- Slice type flag lookups by name at context level. +- Export of app and command help functions. +- More tests & docs. + +## 0.1.0 - 2013-07-22 +### Added +- Initial implementation. + +[Unreleased]: https://github.com/codegangsta/cli/compare/v1.17.0...HEAD +[1.17.0]: https://github.com/codegangsta/cli/compare/v1.16.0...v1.17.0 +[1.16.0]: https://github.com/codegangsta/cli/compare/v1.15.0...v1.16.0 +[1.15.0]: https://github.com/codegangsta/cli/compare/v1.14.0...v1.15.0 +[1.14.0]: https://github.com/codegangsta/cli/compare/v1.13.0...v1.14.0 +[1.13.0]: https://github.com/codegangsta/cli/compare/v1.12.0...v1.13.0 +[1.12.0]: https://github.com/codegangsta/cli/compare/v1.11.1...v1.12.0 +[1.11.1]: https://github.com/codegangsta/cli/compare/v1.11.0...v1.11.1 +[1.11.0]: https://github.com/codegangsta/cli/compare/v1.10.2...v1.11.0 +[1.10.2]: https://github.com/codegangsta/cli/compare/v1.10.1...v1.10.2 +[1.10.1]: https://github.com/codegangsta/cli/compare/v1.10.0...v1.10.1 +[1.10.0]: https://github.com/codegangsta/cli/compare/v1.9.0...v1.10.0 +[1.9.0]: https://github.com/codegangsta/cli/compare/v1.8.0...v1.9.0 +[1.8.0]: https://github.com/codegangsta/cli/compare/v1.7.1...v1.8.0 +[1.7.1]: https://github.com/codegangsta/cli/compare/v1.7.0...v1.7.1 +[1.7.0]: https://github.com/codegangsta/cli/compare/v1.6.0...v1.7.0 +[1.6.0]: https://github.com/codegangsta/cli/compare/v1.5.0...v1.6.0 +[1.5.0]: https://github.com/codegangsta/cli/compare/v1.4.1...v1.5.0 +[1.4.1]: https://github.com/codegangsta/cli/compare/v1.4.0...v1.4.1 +[1.4.0]: https://github.com/codegangsta/cli/compare/v1.3.1...v1.4.0 +[1.3.1]: https://github.com/codegangsta/cli/compare/v1.3.0...v1.3.1 +[1.3.0]: https://github.com/codegangsta/cli/compare/v1.2.0...v1.3.0 +[1.2.0]: https://github.com/codegangsta/cli/compare/v1.1.0...v1.2.0 +[1.1.0]: https://github.com/codegangsta/cli/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/codegangsta/cli/compare/v0.1.0...v1.0.0 diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/LICENSE b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/LICENSE index 5515ccfb7..5515ccfb7 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/LICENSE +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/LICENSE diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/README.md b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/README.md index ae0a4ca3a..c1709cef8 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/README.md +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/README.md @@ -1,22 +1,24 @@ [](http://gocover.io/github.com/codegangsta/cli) [](https://travis-ci.org/codegangsta/cli) [](https://godoc.org/github.com/codegangsta/cli) +[](https://codebeat.co/projects/github-com-codegangsta-cli) +[](https://goreportcard.com/report/codegangsta/cli) -# cli.go +# cli -`cli.go` is simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way. +cli is a simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way. ## Overview Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app. -**This is where `cli.go` comes into play.** `cli.go` makes command line programming fun, organized, and expressive! +**This is where cli comes into play.** cli makes command line programming fun, organized, and expressive! ## Installation Make sure you have a working Go environment (go 1.1+ is *required*). [See the install instructions](http://golang.org/doc/install.html). -To install `cli.go`, simply run: +To install cli, simply run: ``` $ go get github.com/codegangsta/cli ``` @@ -28,7 +30,7 @@ export PATH=$PATH:$GOPATH/bin ## Getting Started -One of the philosophies behind `cli.go` is that an API should be playful and full of discovery. So a `cli.go` app can be as little as one line of code in `main()`. +One of the philosophies behind cli is that an API should be playful and full of discovery. So a cli app can be as little as one line of code in `main()`. ``` go package main @@ -45,11 +47,16 @@ func main() { This app will run and show help text, but is not very useful. Let's give an action to execute and some help documentation: +<!-- { + "output": "boom! I say!" +} --> ``` go package main import ( + "fmt" "os" + "github.com/codegangsta/cli" ) @@ -57,10 +64,11 @@ func main() { app := cli.NewApp() app.Name = "boom" app.Usage = "make an explosive entrance" - app.Action = func(c *cli.Context) { - println("boom! I say!") + app.Action = func(c *cli.Context) error { + fmt.Println("boom! I say!") + return nil } - + app.Run(os.Args) } ``` @@ -73,11 +81,16 @@ Being a programmer can be a lonely job. Thankfully by the power of automation th Start by creating a directory named `greet`, and within it, add a file, `greet.go` with the following code in it: +<!-- { + "output": "Hello friend!" +} --> ``` go package main import ( + "fmt" "os" + "github.com/codegangsta/cli" ) @@ -85,8 +98,9 @@ func main() { app := cli.NewApp() app.Name = "greet" app.Usage = "fight the loneliness!" - app.Action = func(c *cli.Context) { - println("Hello friend!") + app.Action = func(c *cli.Context) error { + fmt.Println("Hello friend!") + return nil } app.Run(os.Args) @@ -106,7 +120,7 @@ $ greet Hello friend! ``` -`cli.go` also generates neat help text: +cli also generates neat help text: ``` $ greet help @@ -132,8 +146,9 @@ You can lookup arguments by calling the `Args` function on `cli.Context`. ``` go ... -app.Action = func(c *cli.Context) { - println("Hello", c.Args()[0]) +app.Action = func(c *cli.Context) error { + fmt.Println("Hello", c.Args()[0]) + return nil } ... ``` @@ -151,16 +166,17 @@ app.Flags = []cli.Flag { Usage: "language for the greeting", }, } -app.Action = func(c *cli.Context) { +app.Action = func(c *cli.Context) error { name := "someone" - if len(c.Args()) > 0 { + if c.NArg() > 0 { name = c.Args()[0] } if c.String("lang") == "spanish" { - println("Hola", name) + fmt.Println("Hola", name) } else { - println("Hello", name) + fmt.Println("Hello", name) } + return nil } ... ``` @@ -178,22 +194,45 @@ app.Flags = []cli.Flag { Destination: &language, }, } -app.Action = func(c *cli.Context) { +app.Action = func(c *cli.Context) error { name := "someone" - if len(c.Args()) > 0 { + if c.NArg() > 0 { name = c.Args()[0] } if language == "spanish" { - println("Hola", name) + fmt.Println("Hola", name) } else { - println("Hello", name) + fmt.Println("Hello", name) } + return nil } ... ``` See full list of flags at http://godoc.org/github.com/codegangsta/cli +#### Placeholder Values + +Sometimes it's useful to specify a flag's value within the usage string itself. Such placeholders are +indicated with back quotes. + +For example this: + +```go +cli.StringFlag{ + Name: "config, c", + Usage: "Load configuration from `FILE`", +} +``` + +Will result in help output like: + +``` +--config FILE, -c FILE Load configuration from FILE +``` + +Note that only the first placeholder is used. Subsequent back-quoted words will be left as-is. + #### Alternate Names You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g. @@ -238,6 +277,49 @@ app.Flags = []cli.Flag { } ``` +#### Values from alternate input sources (YAML and others) + +There is a separate package altsrc that adds support for getting flag values from other input sources like YAML. + +In order to get values for a flag from an alternate input source the following code would be added to wrap an existing cli.Flag like below: + +``` go + altsrc.NewIntFlag(cli.IntFlag{Name: "test"}) +``` + +Initialization must also occur for these flags. Below is an example initializing getting data from a yaml file below. + +``` go + command.Before = altsrc.InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) +``` + +The code above will use the "load" string as a flag name to get the file name of a yaml file from the cli.Context. +It will then use that file name to initialize the yaml input source for any flags that are defined on that command. +As a note the "load" flag used would also have to be defined on the command flags in order for this code snipped to work. + +Currently only YAML files are supported but developers can add support for other input sources by implementing the +altsrc.InputSourceContext for their given sources. + +Here is a more complete sample of a command using YAML support: + +``` go + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) error { + // Action to run + return nil + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test"}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + err := command.Run(c) +``` + ### Subcommands Subcommands can be defined for a more git-like command line app. @@ -249,16 +331,18 @@ app.Commands = []cli.Command{ Name: "add", Aliases: []string{"a"}, Usage: "add a task to the list", - Action: func(c *cli.Context) { - println("added task: ", c.Args().First()) + Action: func(c *cli.Context) error { + fmt.Println("added task: ", c.Args().First()) + return nil }, }, { Name: "complete", Aliases: []string{"c"}, Usage: "complete a task on the list", - Action: func(c *cli.Context) { - println("completed task: ", c.Args().First()) + Action: func(c *cli.Context) error { + fmt.Println("completed task: ", c.Args().First()) + return nil }, }, { @@ -269,15 +353,17 @@ app.Commands = []cli.Command{ { Name: "add", Usage: "add a new template", - Action: func(c *cli.Context) { - println("new task template: ", c.Args().First()) + Action: func(c *cli.Context) error { + fmt.Println("new task template: ", c.Args().First()) + return nil }, }, { Name: "remove", Usage: "remove an existing template", - Action: func(c *cli.Context) { - println("removed task template: ", c.Args().First()) + Action: func(c *cli.Context) error { + fmt.Println("removed task template: ", c.Args().First()) + return nil }, }, }, @@ -286,6 +372,80 @@ app.Commands = []cli.Command{ ... ``` +### Subcommands categories + +For additional organization in apps that have many subcommands, you can +associate a category for each command to group them together in the help +output. + +E.g. + +```go +... + app.Commands = []cli.Command{ + { + Name: "noop", + }, + { + Name: "add", + Category: "template", + }, + { + Name: "remove", + Category: "template", + }, + } +... +``` + +Will include: + +``` +... +COMMANDS: + noop + + Template actions: + add + remove +... +``` + +### Exit code + +Calling `App.Run` will not automatically call `os.Exit`, which means that by +default the exit code will "fall through" to being `0`. An explicit exit code +may be set by returning a non-nil error that fulfills `cli.ExitCoder`, *or* a +`cli.MultiError` that includes an error that fulfills `cli.ExitCoder`, e.g.: + +``` go +package main + +import ( + "os" + + "github.com/codegangsta/cli" +) + +func main() { + app := cli.NewApp() + app.Flags = []cli.Flag{ + cli.BoolTFlag{ + Name: "ginger-crouton", + Usage: "is it in the soup?", + }, + } + app.Action = func(ctx *cli.Context) error { + if !ctx.Bool("ginger-crouton") { + return cli.NewExitError("it is not in the soup", 86) + } + return nil + } + + app.Run(os.Args) +} +``` + ### Bash Completion You can enable completion commands by setting the `EnableBashCompletion` @@ -303,12 +463,13 @@ app.Commands = []cli.Command{ Name: "complete", Aliases: []string{"c"}, Usage: "complete a task on the list", - Action: func(c *cli.Context) { - println("completed task: ", c.Args().First()) + Action: func(c *cli.Context) error { + fmt.Println("completed task: ", c.Args().First()) + return nil }, BashComplete: func(c *cli.Context) { // This will complete if no args are passed - if len(c.Args()) > 0 { + if c.NArg() > 0 { return } for _, t := range tasks { @@ -343,6 +504,72 @@ Alternatively, you can just document that users should source the generic `autocomplete/bash_autocomplete` in their bash configuration with `$PROG` set to the name of their program (as above). +### Generated Help Text Customization + +All of the help text generation may be customized, and at multiple levels. The +templates are exposed as variables `AppHelpTemplate`, `CommandHelpTemplate`, and +`SubcommandHelpTemplate` which may be reassigned or augmented, and full override +is possible by assigning a compatible func to the `cli.HelpPrinter` variable, +e.g.: + +<!-- { + "output": "Ha HA. I pwnd the help!!1" +} --> +``` go +package main + +import ( + "fmt" + "io" + "os" + + "github.com/codegangsta/cli" +) + +func main() { + // EXAMPLE: Append to an existing template + cli.AppHelpTemplate = fmt.Sprintf(`%s + +WEBSITE: http://awesometown.example.com + +SUPPORT: support@awesometown.example.com + +`, cli.AppHelpTemplate) + + // EXAMPLE: Override a template + cli.AppHelpTemplate = `NAME: + {{.Name}} - {{.Usage}} +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}{{if .Commands}} command +[command options]{{end}} {{if +.ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}} + {{if len .Authors}} +AUTHOR(S): + {{range .Authors}}{{ . }}{{end}} + {{end}}{{if .Commands}} +COMMANDS: +{{range .Commands}}{{if not .HideHelp}} {{join .Names ", "}}{{ "\t" +}}{{.Usage}}{{ "\n" }}{{end}}{{end}}{{end}}{{if .VisibleFlags}} +GLOBAL OPTIONS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}}{{if .Copyright }} +COPYRIGHT: + {{.Copyright}} + {{end}}{{if .Version}} +VERSION: + {{.Version}} + {{end}} +` + + // EXAMPLE: Replace the `HelpPrinter` func + cli.HelpPrinter = func(w io.Writer, templ string, data interface{}) { + fmt.Println("Ha HA. I pwnd the help!!1") + } + + cli.NewApp().Run(os.Args) +} +``` + ## Contribution Guidelines Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch. diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/app.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/app.go index 1ea3fd0b1..7c9b95804 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/app.go +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/app.go @@ -5,10 +5,27 @@ import ( "io" "io/ioutil" "os" - "path" + "path/filepath" + "reflect" + "sort" "time" ) +var ( + changeLogURL = "https://github.com/codegangsta/cli/blob/master/CHANGELOG.md" + appActionDeprecationURL = fmt.Sprintf("%s#deprecated-cli-app-action-signature", changeLogURL) + runAndExitOnErrorDeprecationURL = fmt.Sprintf("%s#deprecated-cli-app-runandexitonerror", changeLogURL) + + contactSysadmin = "This is an error in the application. Please contact the distributor of this application if this is not you." + + errNonFuncAction = NewExitError("ERROR invalid Action type. "+ + fmt.Sprintf("Must be a func of type `cli.ActionFunc`. %s", contactSysadmin)+ + fmt.Sprintf("See %s", appActionDeprecationURL), 2) + errInvalidActionSignature = NewExitError("ERROR invalid Action signature. "+ + fmt.Sprintf("Must be `cli.ActionFunc`. %s", contactSysadmin)+ + fmt.Sprintf("See %s", appActionDeprecationURL), 2) +) + // App is the main structure of a cli application. It is recommended that // an app be created with the cli.NewApp() function type App struct { @@ -32,24 +49,27 @@ type App struct { EnableBashCompletion bool // Boolean to hide built-in help command HideHelp bool - // Boolean to hide built-in version flag + // Boolean to hide built-in version flag and the VERSION section of help HideVersion bool + // Populate on app startup, only gettable through method Categories() + categories CommandCategories // An action to execute when the bash-completion flag is set - BashComplete func(context *Context) + BashComplete BashCompleteFunc // An action to execute before any subcommands are run, but after the context is ready // If a non-nil error is returned, no subcommands are run - Before func(context *Context) error + Before BeforeFunc // An action to execute after any subcommands are run, but after the subcommand has finished // It is run even if Action() panics - After func(context *Context) error + After AfterFunc // The action to execute when no subcommands are specified - Action func(context *Context) + Action interface{} + // TODO: replace `Action: interface{}` with `Action: ActionFunc` once some kind + // of deprecation period has passed, maybe? + // Execute this function if the proper command cannot be found - CommandNotFound func(context *Context, command string) - // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. - // This function is able to replace the original error messages. - // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. - OnUsageError func(context *Context, err error, isSubcommand bool) error + CommandNotFound CommandNotFoundFunc + // Execute this function if an usage error occurs + OnUsageError OnUsageErrorFunc // Compilation date Compiled time.Time // List of all authors who contributed @@ -62,6 +82,12 @@ type App struct { Email string // Writer writer to write output to Writer io.Writer + // ErrWriter writes error output + ErrWriter io.Writer + // Other custom info + Metadata map[string]interface{} + + didSetup bool } // Tries to find out when this binary was compiled. @@ -74,11 +100,12 @@ func compileTime() time.Time { return info.ModTime() } -// Creates a new cli Application with some reasonable defaults for Name, Usage, Version and Action. +// NewApp creates a new cli Application with some reasonable defaults for Name, +// Usage, Version and Action. func NewApp() *App { return &App{ - Name: path.Base(os.Args[0]), - HelpName: path.Base(os.Args[0]), + Name: filepath.Base(os.Args[0]), + HelpName: filepath.Base(os.Args[0]), Usage: "A new cli application", UsageText: "", Version: "0.0.0", @@ -89,8 +116,16 @@ func NewApp() *App { } } -// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination -func (a *App) Run(arguments []string) (err error) { +// Setup runs initialization code to ensure all data structures are ready for +// `Run` or inspection prior to `Run`. It is internally called by `Run`, but +// will return early if setup has already happened. +func (a *App) Setup() { + if a.didSetup { + return + } + + a.didSetup = true + if a.Author != "" || a.Email != "" { a.Authors = append(a.Authors, Author{Name: a.Author, Email: a.Email}) } @@ -104,6 +139,12 @@ func (a *App) Run(arguments []string) (err error) { } a.Commands = newCmds + a.categories = CommandCategories{} + for _, command := range a.Commands { + a.categories = a.categories.AddCommand(command.Category, command) + } + sort.Sort(a.categories) + // append help to commands if a.Command(helpCommand.Name) == nil && !a.HideHelp { a.Commands = append(a.Commands, helpCommand) @@ -120,6 +161,12 @@ func (a *App) Run(arguments []string) (err error) { if !a.HideVersion { a.appendFlag(VersionFlag) } +} + +// Run is the entry point to the cli app. Parses the arguments slice and routes +// to the proper flag/args combination +func (a *App) Run(arguments []string) (err error) { + a.Setup() // parse flags set := flagSet(a.Name, a.Flags) @@ -140,12 +187,12 @@ func (a *App) Run(arguments []string) (err error) { if err != nil { if a.OnUsageError != nil { err := a.OnUsageError(context, err, false) - return err - } else { - fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") - ShowAppHelp(context) + HandleExitCoder(err) return err } + fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") + ShowAppHelp(context) + return err } if !a.HideHelp && checkHelp(context) { @@ -171,10 +218,12 @@ func (a *App) Run(arguments []string) (err error) { } if a.Before != nil { - err = a.Before(context) - if err != nil { - fmt.Fprintf(a.Writer, "%v\n\n", err) + beforeErr := a.Before(context) + if beforeErr != nil { + fmt.Fprintf(a.Writer, "%v\n\n", beforeErr) ShowAppHelp(context) + HandleExitCoder(beforeErr) + err = beforeErr return err } } @@ -189,19 +238,25 @@ func (a *App) Run(arguments []string) (err error) { } // Run default Action - a.Action(context) - return nil + err = HandleAction(a.Action, context) + + HandleExitCoder(err) + return err } -// Another entry point to the cli app, takes care of passing arguments and error handling +// DEPRECATED: Another entry point to the cli app, takes care of passing arguments and error handling func (a *App) RunAndExitOnError() { + fmt.Fprintf(a.errWriter(), + "DEPRECATED cli.App.RunAndExitOnError. %s See %s\n", + contactSysadmin, runAndExitOnErrorDeprecationURL) if err := a.Run(os.Args); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + fmt.Fprintln(a.errWriter(), err) + OsExiter(1) } } -// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags +// RunAsSubcommand invokes the subcommand given the context, parses ctx.Args() to +// generate command-specific flags func (a *App) RunAsSubcommand(ctx *Context) (err error) { // append help to commands if len(a.Commands) > 0 { @@ -252,12 +307,12 @@ func (a *App) RunAsSubcommand(ctx *Context) (err error) { if err != nil { if a.OnUsageError != nil { err = a.OnUsageError(context, err, true) - return err - } else { - fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") - ShowSubcommandHelp(context) + HandleExitCoder(err) return err } + fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") + ShowSubcommandHelp(context) + return err } if len(a.Commands) > 0 { @@ -274,6 +329,7 @@ func (a *App) RunAsSubcommand(ctx *Context) (err error) { defer func() { afterErr := a.After(context) if afterErr != nil { + HandleExitCoder(err) if err != nil { err = NewMultiError(err, afterErr) } else { @@ -284,8 +340,10 @@ func (a *App) RunAsSubcommand(ctx *Context) (err error) { } if a.Before != nil { - err := a.Before(context) - if err != nil { + beforeErr := a.Before(context) + if beforeErr != nil { + HandleExitCoder(beforeErr) + err = beforeErr return err } } @@ -300,12 +358,13 @@ func (a *App) RunAsSubcommand(ctx *Context) (err error) { } // Run default Action - a.Action(context) + err = HandleAction(a.Action, context) - return nil + HandleExitCoder(err) + return err } -// Returns the named command on App. Returns nil if the command does not exist +// Command returns the named command on App. Returns nil if the command does not exist func (a *App) Command(name string) *Command { for _, c := range a.Commands { if c.HasName(name) { @@ -316,6 +375,46 @@ func (a *App) Command(name string) *Command { return nil } +// Categories returns a slice containing all the categories with the commands they contain +func (a *App) Categories() CommandCategories { + return a.categories +} + +// VisibleCategories returns a slice of categories and commands that are +// Hidden=false +func (a *App) VisibleCategories() []*CommandCategory { + ret := []*CommandCategory{} + for _, category := range a.categories { + if visible := func() *CommandCategory { + for _, command := range category.Commands { + if !command.Hidden { + return category + } + } + return nil + }(); visible != nil { + ret = append(ret, visible) + } + } + return ret +} + +// VisibleCommands returns a slice of the Commands with Hidden=false +func (a *App) VisibleCommands() []Command { + ret := []Command{} + for _, command := range a.Commands { + if !command.Hidden { + ret = append(ret, command) + } + } + return ret +} + +// VisibleFlags returns a slice of the Flags with Hidden=false +func (a *App) VisibleFlags() []Flag { + return visibleFlags(a.Flags) +} + func (a *App) hasFlag(flag Flag) bool { for _, f := range a.Flags { if flag == f { @@ -326,6 +425,16 @@ func (a *App) hasFlag(flag Flag) bool { return false } +func (a *App) errWriter() io.Writer { + + // When the app ErrWriter is nil use the package level one. + if a.ErrWriter == nil { + return ErrWriter + } + + return a.ErrWriter +} + func (a *App) appendFlag(flag Flag) { if !a.hasFlag(flag) { a.Flags = append(a.Flags, flag) @@ -347,3 +456,43 @@ func (a Author) String() string { return fmt.Sprintf("%v %v", a.Name, e) } + +// HandleAction uses ✧✧✧reflection✧✧✧ to figure out if the given Action is an +// ActionFunc, a func with the legacy signature for Action, or some other +// invalid thing. If it's an ActionFunc or a func with the legacy signature for +// Action, the func is run! +func HandleAction(action interface{}, context *Context) (err error) { + defer func() { + if r := recover(); r != nil { + switch r.(type) { + case error: + err = r.(error) + default: + err = NewExitError(fmt.Sprintf("ERROR unknown Action error: %v. See %s", r, appActionDeprecationURL), 2) + } + } + }() + + if reflect.TypeOf(action).Kind() != reflect.Func { + return errNonFuncAction + } + + vals := reflect.ValueOf(action).Call([]reflect.Value{reflect.ValueOf(context)}) + + if len(vals) == 0 { + fmt.Fprintf(ErrWriter, + "DEPRECATED Action signature. Must be `cli.ActionFunc`. %s See %s\n", + contactSysadmin, appActionDeprecationURL) + return nil + } + + if len(vals) > 1 { + return errInvalidActionSignature + } + + if retErr, ok := vals[0].Interface().(error); vals[0].IsValid() && ok { + return retErr + } + + return err +} diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/appveyor.yml b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/appveyor.yml index 3ca7afabd..3ca7afabd 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/appveyor.yml +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/appveyor.yml diff --git a/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/category.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/category.go new file mode 100644 index 000000000..1a6055023 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/category.go @@ -0,0 +1,44 @@ +package cli + +// CommandCategories is a slice of *CommandCategory. +type CommandCategories []*CommandCategory + +// CommandCategory is a category containing commands. +type CommandCategory struct { + Name string + Commands Commands +} + +func (c CommandCategories) Less(i, j int) bool { + return c[i].Name < c[j].Name +} + +func (c CommandCategories) Len() int { + return len(c) +} + +func (c CommandCategories) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} + +// AddCommand adds a command to a category. +func (c CommandCategories) AddCommand(category string, command Command) CommandCategories { + for _, commandCategory := range c { + if commandCategory.Name == category { + commandCategory.Commands = append(commandCategory.Commands, command) + return c + } + } + return append(c, &CommandCategory{Name: category, Commands: []Command{command}}) +} + +// VisibleCommands returns a slice of the Commands with Hidden=false +func (c *CommandCategory) VisibleCommands() []Command { + ret := []Command{} + for _, command := range c.Commands { + if !command.Hidden { + ret = append(ret, command) + } + } + return ret +} diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/cli.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/cli.go index 31dc9124d..f0440c563 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/cli.go +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/cli.go @@ -10,31 +10,10 @@ // app := cli.NewApp() // app.Name = "greet" // app.Usage = "say a greeting" -// app.Action = func(c *cli.Context) { +// app.Action = func(c *cli.Context) error { // println("Greetings") // } // // app.Run(os.Args) // } package cli - -import ( - "strings" -) - -type MultiError struct { - Errors []error -} - -func NewMultiError(err ...error) MultiError { - return MultiError{Errors: err} -} - -func (m MultiError) Error() string { - errs := make([]string, len(m.Errors)) - for i, err := range m.Errors { - errs[i] = err.Error() - } - - return strings.Join(errs, "\n") -} diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/command.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/command.go index bbf42ae40..8950ccae4 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/command.go +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/command.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "io/ioutil" + "sort" "strings" ) @@ -22,35 +23,40 @@ type Command struct { Description string // A short description of the arguments of this command ArgsUsage string + // The category the command is part of + Category string // The function to call when checking for bash command completions - BashComplete func(context *Context) + BashComplete BashCompleteFunc // An action to execute before any sub-subcommands are run, but after the context is ready // If a non-nil error is returned, no sub-subcommands are run - Before func(context *Context) error + Before BeforeFunc // An action to execute after any subcommands are run, but after the subcommand has finished // It is run even if Action() panics - After func(context *Context) error + After AfterFunc // The function to call when this command is invoked - Action func(context *Context) - // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. - // This function is able to replace the original error messages. - // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. - OnUsageError func(context *Context, err error) error + Action interface{} + // TODO: replace `Action: interface{}` with `Action: ActionFunc` once some kind + // of deprecation period has passed, maybe? + + // Execute this function if a usage error occurs. + OnUsageError OnUsageErrorFunc // List of child commands - Subcommands []Command + Subcommands Commands // List of flags to parse Flags []Flag // Treat all flags as normal arguments if true SkipFlagParsing bool // Boolean to hide built-in help command HideHelp bool + // Boolean to hide this command from help or completion + Hidden bool // Full name of command for help, defaults to full command name, including parent commands. HelpName string commandNamePath []string } -// Returns the full name of the command. +// FullName returns the full name of the command. // For subcommands this ensures that parent commands are part of the command path func (c Command) FullName() string { if c.commandNamePath == nil { @@ -59,7 +65,10 @@ func (c Command) FullName() string { return strings.Join(c.commandNamePath, " ") } -// Invokes the command given the context, parses ctx.Args() to generate command-specific flags +// Commands is a slice of Command +type Commands []Command + +// Run invokes the command given the context, parses ctx.Args() to generate command-specific flags func (c Command) Run(ctx *Context) (err error) { if len(c.Subcommands) > 0 { return c.startApp(ctx) @@ -120,14 +129,14 @@ func (c Command) Run(ctx *Context) (err error) { if err != nil { if c.OnUsageError != nil { - err := c.OnUsageError(ctx, err) - return err - } else { - fmt.Fprintln(ctx.App.Writer, "Incorrect Usage.") - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) + err := c.OnUsageError(ctx, err, false) + HandleExitCoder(err) return err } + fmt.Fprintln(ctx.App.Writer, "Incorrect Usage.") + fmt.Fprintln(ctx.App.Writer) + ShowCommandHelp(ctx, c.Name) + return err } nerr := normalizeFlags(c.Flags, set) @@ -137,6 +146,7 @@ func (c Command) Run(ctx *Context) (err error) { ShowCommandHelp(ctx, c.Name) return nerr } + context := NewContext(ctx.App, set, ctx) if checkCommandCompletions(context, c.Name) { @@ -151,6 +161,7 @@ func (c Command) Run(ctx *Context) (err error) { defer func() { afterErr := c.After(context) if afterErr != nil { + HandleExitCoder(err) if err != nil { err = NewMultiError(err, afterErr) } else { @@ -161,20 +172,26 @@ func (c Command) Run(ctx *Context) (err error) { } if c.Before != nil { - err := c.Before(context) + err = c.Before(context) if err != nil { fmt.Fprintln(ctx.App.Writer, err) fmt.Fprintln(ctx.App.Writer) ShowCommandHelp(ctx, c.Name) + HandleExitCoder(err) return err } } context.Command = c - c.Action(context) - return nil + err = HandleAction(c.Action, context) + + if err != nil { + HandleExitCoder(err) + } + return err } +// Names returns the names including short names and aliases. func (c Command) Names() []string { names := []string{c.Name} @@ -185,7 +202,7 @@ func (c Command) Names() []string { return append(names, c.Aliases...) } -// Returns true if Command.Name or Command.ShortName matches given name +// HasName returns true if Command.Name or Command.ShortName matches given name func (c Command) HasName(name string) bool { for _, n := range c.Names() { if n == name { @@ -197,7 +214,7 @@ func (c Command) HasName(name string) bool { func (c Command) startApp(ctx *Context) error { app := NewApp() - + app.Metadata = ctx.App.Metadata // set the name and usage app.Name = fmt.Sprintf("%s %s", ctx.App.Name, c.Name) if c.HelpName == "" { @@ -227,6 +244,13 @@ func (c Command) startApp(ctx *Context) error { app.Email = ctx.App.Email app.Writer = ctx.App.Writer + app.categories = CommandCategories{} + for _, command := range c.Subcommands { + app.categories = app.categories.AddCommand(command.Category, command) + } + + sort.Sort(app.categories) + // bash completion app.EnableBashCompletion = ctx.App.EnableBashCompletion if c.BashComplete != nil { @@ -248,3 +272,8 @@ func (c Command) startApp(ctx *Context) error { return app.RunAsSubcommand(ctx) } + +// VisibleFlags returns a slice of the Flags with Hidden=false +func (c Command) VisibleFlags() []Flag { + return visibleFlags(c.Flags) +} diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/context.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/context.go index 0513d34f6..c34246369 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/context.go +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/context.go @@ -21,57 +21,62 @@ type Context struct { parentContext *Context } -// Creates a new context. For use in when invoking an App or Command action. +// NewContext creates a new context. For use in when invoking an App or Command action. func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context { return &Context{App: app, flagSet: set, parentContext: parentCtx} } -// Looks up the value of a local int flag, returns 0 if no int flag exists +// Int looks up the value of a local int flag, returns 0 if no int flag exists func (c *Context) Int(name string) int { return lookupInt(name, c.flagSet) } -// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists +// Duration looks up the value of a local time.Duration flag, returns 0 if no +// time.Duration flag exists func (c *Context) Duration(name string) time.Duration { return lookupDuration(name, c.flagSet) } -// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists +// Float64 looks up the value of a local float64 flag, returns 0 if no float64 +// flag exists func (c *Context) Float64(name string) float64 { return lookupFloat64(name, c.flagSet) } -// Looks up the value of a local bool flag, returns false if no bool flag exists +// Bool looks up the value of a local bool flag, returns false if no bool flag exists func (c *Context) Bool(name string) bool { return lookupBool(name, c.flagSet) } -// Looks up the value of a local boolT flag, returns false if no bool flag exists +// BoolT looks up the value of a local boolT flag, returns false if no bool flag exists func (c *Context) BoolT(name string) bool { return lookupBoolT(name, c.flagSet) } -// Looks up the value of a local string flag, returns "" if no string flag exists +// String looks up the value of a local string flag, returns "" if no string flag exists func (c *Context) String(name string) string { return lookupString(name, c.flagSet) } -// Looks up the value of a local string slice flag, returns nil if no string slice flag exists +// StringSlice looks up the value of a local string slice flag, returns nil if no +// string slice flag exists func (c *Context) StringSlice(name string) []string { return lookupStringSlice(name, c.flagSet) } -// Looks up the value of a local int slice flag, returns nil if no int slice flag exists +// IntSlice looks up the value of a local int slice flag, returns nil if no int +// slice flag exists func (c *Context) IntSlice(name string) []int { return lookupIntSlice(name, c.flagSet) } -// Looks up the value of a local generic flag, returns nil if no generic flag exists +// Generic looks up the value of a local generic flag, returns nil if no generic +// flag exists func (c *Context) Generic(name string) interface{} { return lookupGeneric(name, c.flagSet) } -// Looks up the value of a global int flag, returns 0 if no int flag exists +// GlobalInt looks up the value of a global int flag, returns 0 if no int flag exists func (c *Context) GlobalInt(name string) int { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupInt(name, fs) @@ -79,7 +84,17 @@ func (c *Context) GlobalInt(name string) int { return 0 } -// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists +// GlobalFloat64 looks up the value of a global float64 flag, returns float64(0) +// if no float64 flag exists +func (c *Context) GlobalFloat64(name string) float64 { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupFloat64(name, fs) + } + return float64(0) +} + +// GlobalDuration looks up the value of a global time.Duration flag, returns 0 +// if no time.Duration flag exists func (c *Context) GlobalDuration(name string) time.Duration { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupDuration(name, fs) @@ -87,7 +102,8 @@ func (c *Context) GlobalDuration(name string) time.Duration { return 0 } -// Looks up the value of a global bool flag, returns false if no bool flag exists +// GlobalBool looks up the value of a global bool flag, returns false if no bool +// flag exists func (c *Context) GlobalBool(name string) bool { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupBool(name, fs) @@ -95,7 +111,17 @@ func (c *Context) GlobalBool(name string) bool { return false } -// Looks up the value of a global string flag, returns "" if no string flag exists +// GlobalBoolT looks up the value of a global bool flag, returns true if no bool +// flag exists +func (c *Context) GlobalBoolT(name string) bool { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupBoolT(name, fs) + } + return false +} + +// GlobalString looks up the value of a global string flag, returns "" if no +// string flag exists func (c *Context) GlobalString(name string) string { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupString(name, fs) @@ -103,7 +129,8 @@ func (c *Context) GlobalString(name string) string { return "" } -// Looks up the value of a global string slice flag, returns nil if no string slice flag exists +// GlobalStringSlice looks up the value of a global string slice flag, returns +// nil if no string slice flag exists func (c *Context) GlobalStringSlice(name string) []string { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupStringSlice(name, fs) @@ -111,7 +138,8 @@ func (c *Context) GlobalStringSlice(name string) []string { return nil } -// Looks up the value of a global int slice flag, returns nil if no int slice flag exists +// GlobalIntSlice looks up the value of a global int slice flag, returns nil if +// no int slice flag exists func (c *Context) GlobalIntSlice(name string) []int { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupIntSlice(name, fs) @@ -119,7 +147,8 @@ func (c *Context) GlobalIntSlice(name string) []int { return nil } -// Looks up the value of a global generic flag, returns nil if no generic flag exists +// GlobalGeneric looks up the value of a global generic flag, returns nil if no +// generic flag exists func (c *Context) GlobalGeneric(name string) interface{} { if fs := lookupGlobalFlagSet(name, c); fs != nil { return lookupGeneric(name, fs) @@ -127,12 +156,22 @@ func (c *Context) GlobalGeneric(name string) interface{} { return nil } -// Returns the number of flags set +// NumFlags returns the number of flags set func (c *Context) NumFlags() int { return c.flagSet.NFlag() } -// Determines if the flag was actually set +// Set sets a context flag to a value. +func (c *Context) Set(name, value string) error { + return c.flagSet.Set(name, value) +} + +// GlobalSet sets a context flag to a value on the global flagset +func (c *Context) GlobalSet(name, value string) error { + return globalContext(c).flagSet.Set(name, value) +} + +// IsSet determines if the flag was actually set func (c *Context) IsSet(name string) bool { if c.setFlags == nil { c.setFlags = make(map[string]bool) @@ -143,7 +182,7 @@ func (c *Context) IsSet(name string) bool { return c.setFlags[name] == true } -// Determines if the global flag was actually set +// GlobalIsSet determines if the global flag was actually set func (c *Context) GlobalIsSet(name string) bool { if c.globalSetFlags == nil { c.globalSetFlags = make(map[string]bool) @@ -160,7 +199,7 @@ func (c *Context) GlobalIsSet(name string) bool { return c.globalSetFlags[name] } -// Returns a slice of flag names used in this context. +// FlagNames returns a slice of flag names used in this context. func (c *Context) FlagNames() (names []string) { for _, flag := range c.Command.Flags { name := strings.Split(flag.GetName(), ",")[0] @@ -172,7 +211,7 @@ func (c *Context) FlagNames() (names []string) { return } -// Returns a slice of global flag names used by the app. +// GlobalFlagNames returns a slice of global flag names used by the app. func (c *Context) GlobalFlagNames() (names []string) { for _, flag := range c.App.Flags { name := strings.Split(flag.GetName(), ",")[0] @@ -184,20 +223,26 @@ func (c *Context) GlobalFlagNames() (names []string) { return } -// Returns the parent context, if any +// Parent returns the parent context, if any func (c *Context) Parent() *Context { return c.parentContext } +// Args contains apps console arguments type Args []string -// Returns the command line arguments associated with the context. +// Args returns the command line arguments associated with the context. func (c *Context) Args() Args { args := Args(c.flagSet.Args()) return args } -// Returns the nth argument, or else a blank string +// NArg returns the number of the command line arguments. +func (c *Context) NArg() int { + return len(c.Args()) +} + +// Get returns the nth argument, or else a blank string func (a Args) Get(n int) string { if len(a) > n { return a[n] @@ -205,12 +250,12 @@ func (a Args) Get(n int) string { return "" } -// Returns the first argument, or else a blank string +// First returns the first argument, or else a blank string func (a Args) First() string { return a.Get(0) } -// Return the rest of the arguments (not the first one) +// Tail returns the rest of the arguments (not the first one) // or else an empty string slice func (a Args) Tail() []string { if len(a) >= 2 { @@ -219,12 +264,12 @@ func (a Args) Tail() []string { return []string{} } -// Checks if there are any arguments present +// Present checks if there are any arguments present func (a Args) Present() bool { return len(a) != 0 } -// Swaps arguments at the given indexes +// Swap swaps arguments at the given indexes func (a Args) Swap(from, to int) error { if from >= len(a) || to >= len(a) { return errors.New("index out of range") @@ -233,6 +278,19 @@ func (a Args) Swap(from, to int) error { return nil } +func globalContext(ctx *Context) *Context { + if ctx == nil { + return nil + } + + for { + if ctx.parentContext == nil { + return ctx + } + ctx = ctx.parentContext + } +} + func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet { if ctx.parentContext != nil { ctx = ctx.parentContext diff --git a/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/errors.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/errors.go new file mode 100644 index 000000000..ea551be16 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/errors.go @@ -0,0 +1,92 @@ +package cli + +import ( + "fmt" + "io" + "os" + "strings" +) + +// OsExiter is the function used when the app exits. If not set defaults to os.Exit. +var OsExiter = os.Exit + +// ErrWriter is used to write errors to the user. This can be anything +// implementing the io.Writer interface and defaults to os.Stderr. +var ErrWriter io.Writer = os.Stderr + +// MultiError is an error that wraps multiple errors. +type MultiError struct { + Errors []error +} + +// NewMultiError creates a new MultiError. Pass in one or more errors. +func NewMultiError(err ...error) MultiError { + return MultiError{Errors: err} +} + +// Error implents the error interface. +func (m MultiError) Error() string { + errs := make([]string, len(m.Errors)) + for i, err := range m.Errors { + errs[i] = err.Error() + } + + return strings.Join(errs, "\n") +} + +// ExitCoder is the interface checked by `App` and `Command` for a custom exit +// code +type ExitCoder interface { + error + ExitCode() int +} + +// ExitError fulfills both the builtin `error` interface and `ExitCoder` +type ExitError struct { + exitCode int + message string +} + +// NewExitError makes a new *ExitError +func NewExitError(message string, exitCode int) *ExitError { + return &ExitError{ + exitCode: exitCode, + message: message, + } +} + +// Error returns the string message, fulfilling the interface required by +// `error` +func (ee *ExitError) Error() string { + return ee.message +} + +// ExitCode returns the exit code, fulfilling the interface required by +// `ExitCoder` +func (ee *ExitError) ExitCode() int { + return ee.exitCode +} + +// HandleExitCoder checks if the error fulfills the ExitCoder interface, and if +// so prints the error to stderr (if it is non-empty) and calls OsExiter with the +// given exit code. If the given error is a MultiError, then this func is +// called on all members of the Errors slice. +func HandleExitCoder(err error) { + if err == nil { + return + } + + if exitErr, ok := err.(ExitCoder); ok { + if err.Error() != "" { + fmt.Fprintln(ErrWriter, err) + } + OsExiter(exitErr.ExitCode()) + return + } + + if multiErr, ok := err.(MultiError); ok { + for _, merr := range multiErr.Errors { + HandleExitCoder(merr) + } + } +} diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/flag.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/flag.go index e951c2df7..1e8112e7e 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/flag.go +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/flag.go @@ -4,24 +4,28 @@ import ( "flag" "fmt" "os" + "reflect" "runtime" "strconv" "strings" "time" ) -// This flag enables bash-completion for all commands and subcommands +const defaultPlaceholder = "value" + +// BashCompletionFlag enables bash-completion for all commands and subcommands var BashCompletionFlag = BoolFlag{ - Name: "generate-bash-completion", + Name: "generate-bash-completion", + Hidden: true, } -// This flag prints the version for the application +// VersionFlag prints the version for the application var VersionFlag = BoolFlag{ Name: "version, v", Usage: "print the version", } -// This flag prints the help for all commands and subcommands +// HelpFlag prints the help for all commands and subcommands // Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand // unless HideHelp is set to true) var HelpFlag = BoolFlag{ @@ -29,6 +33,10 @@ var HelpFlag = BoolFlag{ Usage: "show help", } +// FlagStringer converts a flag definition to a string. This is used by help +// to display a flag. +var FlagStringer FlagStringFunc = stringifyFlag + // Flag is a common interface related to parsing flags in cli. // For more advanced flag parsing techniques, it is recommended that // this interface be implemented. @@ -68,24 +76,14 @@ type GenericFlag struct { Value Generic Usage string EnvVar string + Hidden bool } // String returns the string representation of the generic flag to display the // help text to the user (uses the String() method of the generic flag to show // the value) func (f GenericFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) -} - -func (f GenericFlag) FormatValueHelp() string { - if f.Value == nil { - return "" - } - s := f.Value.String() - if len(s) == 0 { - return "" - } - return fmt.Sprintf("\"%s\"", s) + return FlagStringer(f) } // Apply takes the flagset and calls Set on the generic flag with the value @@ -107,6 +105,7 @@ func (f GenericFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of a flag. func (f GenericFlag) GetName() string { return f.Name } @@ -130,20 +129,19 @@ func (f *StringSlice) Value() []string { return *f } -// StringSlice is a string flag that can be specified multiple times on the +// StringSliceFlag is a string flag that can be specified multiple times on the // command-line type StringSliceFlag struct { Name string Value *StringSlice Usage string EnvVar string + Hidden bool } // String returns the usage func (f StringSliceFlag) String() string { - firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") - pref := prefixFor(firstName) - return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -171,11 +169,12 @@ func (f StringSliceFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of a flag. func (f StringSliceFlag) GetName() string { return f.Name } -// StringSlice is an opaque type for []int to satisfy flag.Value +// IntSlice is an opaque type for []int to satisfy flag.Value type IntSlice []int // Set parses the value into an integer and appends it to the list of values @@ -183,9 +182,8 @@ func (f *IntSlice) Set(value string) error { tmp, err := strconv.Atoi(value) if err != nil { return err - } else { - *f = append(*f, tmp) } + *f = append(*f, tmp) return nil } @@ -206,13 +204,12 @@ type IntSliceFlag struct { Value *IntSlice Usage string EnvVar string + Hidden bool } // String returns the usage func (f IntSliceFlag) String() string { - firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") - pref := prefixFor(firstName) - return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -226,7 +223,7 @@ func (f IntSliceFlag) Apply(set *flag.FlagSet) { s = strings.TrimSpace(s) err := newVal.Set(s) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(ErrWriter, err.Error()) } } f.Value = newVal @@ -243,6 +240,7 @@ func (f IntSliceFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f IntSliceFlag) GetName() string { return f.Name } @@ -253,11 +251,12 @@ type BoolFlag struct { Usage string EnvVar string Destination *bool + Hidden bool } // String returns a readable representation of this value (for usage defaults) func (f BoolFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -285,6 +284,7 @@ func (f BoolFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f BoolFlag) GetName() string { return f.Name } @@ -296,11 +296,12 @@ type BoolTFlag struct { Usage string EnvVar string Destination *bool + Hidden bool } // String returns a readable representation of this value (for usage defaults) func (f BoolTFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -328,6 +329,7 @@ func (f BoolTFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f BoolTFlag) GetName() string { return f.Name } @@ -339,19 +341,12 @@ type StringFlag struct { Usage string EnvVar string Destination *string + Hidden bool } // String returns the usage func (f StringFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) -} - -func (f StringFlag) FormatValueHelp() string { - s := f.Value - if len(s) == 0 { - return "" - } - return fmt.Sprintf("\"%s\"", s) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -375,6 +370,7 @@ func (f StringFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f StringFlag) GetName() string { return f.Name } @@ -387,11 +383,12 @@ type IntFlag struct { Usage string EnvVar string Destination *int + Hidden bool } // String returns the usage func (f IntFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -418,6 +415,7 @@ func (f IntFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f IntFlag) GetName() string { return f.Name } @@ -430,11 +428,12 @@ type DurationFlag struct { Usage string EnvVar string Destination *time.Duration + Hidden bool } // String returns a readable representation of this value (for usage defaults) func (f DurationFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -461,6 +460,7 @@ func (f DurationFlag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f DurationFlag) GetName() string { return f.Name } @@ -473,11 +473,12 @@ type Float64Flag struct { Usage string EnvVar string Destination *float64 + Hidden bool } // String returns the usage func (f Float64Flag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) + return FlagStringer(f) } // Apply populates the flag given the flag set and environment @@ -503,10 +504,21 @@ func (f Float64Flag) Apply(set *flag.FlagSet) { }) } +// GetName returns the name of the flag. func (f Float64Flag) GetName() string { return f.Name } +func visibleFlags(fl []Flag) []Flag { + visible := []Flag{} + for _, flag := range fl { + if !reflect.ValueOf(flag).FieldByName("Hidden").Bool() { + visible = append(visible, flag) + } + } + return visible +} + func prefixFor(name string) (prefix string) { if len(name) == 1 { prefix = "-" @@ -517,16 +529,37 @@ func prefixFor(name string) (prefix string) { return } -func prefixedNames(fullName string) (prefixed string) { +// Returns the placeholder, if any, and the unquoted usage string. +func unquoteUsage(usage string) (string, string) { + for i := 0; i < len(usage); i++ { + if usage[i] == '`' { + for j := i + 1; j < len(usage); j++ { + if usage[j] == '`' { + name := usage[i+1 : j] + usage = usage[:i] + name + usage[j+1:] + return name, usage + } + } + break + } + } + return "", usage +} + +func prefixedNames(fullName, placeholder string) string { + var prefixed string parts := strings.Split(fullName, ",") for i, name := range parts { name = strings.Trim(name, " ") prefixed += prefixFor(name) + name + if placeholder != "" { + prefixed += " " + placeholder + } if i < len(parts)-1 { prefixed += ", " } } - return + return prefixed } func withEnvHint(envVar, str string) string { @@ -544,3 +577,83 @@ func withEnvHint(envVar, str string) string { } return str + envText } + +func stringifyFlag(f Flag) string { + fv := reflect.ValueOf(f) + + switch f.(type) { + case IntSliceFlag: + return withEnvHint(fv.FieldByName("EnvVar").String(), + stringifyIntSliceFlag(f.(IntSliceFlag))) + case StringSliceFlag: + return withEnvHint(fv.FieldByName("EnvVar").String(), + stringifyStringSliceFlag(f.(StringSliceFlag))) + } + + placeholder, usage := unquoteUsage(fv.FieldByName("Usage").String()) + + needsPlaceholder := false + defaultValueString := "" + val := fv.FieldByName("Value") + + if val.IsValid() { + needsPlaceholder = true + defaultValueString = fmt.Sprintf(" (default: %v)", val.Interface()) + + if val.Kind() == reflect.String && val.String() != "" { + defaultValueString = fmt.Sprintf(" (default: %q)", val.String()) + } + } + + if defaultValueString == " (default: )" { + defaultValueString = "" + } + + if needsPlaceholder && placeholder == "" { + placeholder = defaultPlaceholder + } + + usageWithDefault := strings.TrimSpace(fmt.Sprintf("%s%s", usage, defaultValueString)) + + return withEnvHint(fv.FieldByName("EnvVar").String(), + fmt.Sprintf("%s\t%s", prefixedNames(fv.FieldByName("Name").String(), placeholder), usageWithDefault)) +} + +func stringifyIntSliceFlag(f IntSliceFlag) string { + defaultVals := []string{} + if f.Value != nil && len(f.Value.Value()) > 0 { + for _, i := range f.Value.Value() { + defaultVals = append(defaultVals, fmt.Sprintf("%d", i)) + } + } + + return stringifySliceFlag(f.Usage, f.Name, defaultVals) +} + +func stringifyStringSliceFlag(f StringSliceFlag) string { + defaultVals := []string{} + if f.Value != nil && len(f.Value.Value()) > 0 { + for _, s := range f.Value.Value() { + if len(s) > 0 { + defaultVals = append(defaultVals, fmt.Sprintf("%q", s)) + } + } + } + + return stringifySliceFlag(f.Usage, f.Name, defaultVals) +} + +func stringifySliceFlag(usage, name string, defaultVals []string) string { + placeholder, usage := unquoteUsage(usage) + if placeholder == "" { + placeholder = defaultPlaceholder + } + + defaultVal := "" + if len(defaultVals) > 0 { + defaultVal = fmt.Sprintf(" (default: %s)", strings.Join(defaultVals, ", ")) + } + + usageWithDefault := strings.TrimSpace(fmt.Sprintf("%s%s", usage, defaultVal)) + return fmt.Sprintf("%s\t%s", prefixedNames(name, placeholder), usageWithDefault) +} diff --git a/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/funcs.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/funcs.go new file mode 100644 index 000000000..cba5e6cb0 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/funcs.go @@ -0,0 +1,28 @@ +package cli + +// BashCompleteFunc is an action to execute when the bash-completion flag is set +type BashCompleteFunc func(*Context) + +// BeforeFunc is an action to execute before any subcommands are run, but after +// the context is ready if a non-nil error is returned, no subcommands are run +type BeforeFunc func(*Context) error + +// AfterFunc is an action to execute after any subcommands are run, but after the +// subcommand has finished it is run even if Action() panics +type AfterFunc func(*Context) error + +// ActionFunc is the action to execute when no subcommands are specified +type ActionFunc func(*Context) error + +// CommandNotFoundFunc is executed if the proper command cannot be found +type CommandNotFoundFunc func(*Context, string) + +// OnUsageErrorFunc is executed if an usage error occurs. This is useful for displaying +// customized usage error messages. This function is able to replace the +// original error messages. If this function is not set, the "Incorrect usage" +// is displayed and the execution is interrupted. +type OnUsageErrorFunc func(context *Context, err error, isSubcommand bool) error + +// FlagStringFunc is used by the help generation to display a flag, which is +// expected to be a single line. +type FlagStringFunc func(Flag) string diff --git a/Godeps/_workspace/src/github.com/codegangsta/cli/help.go b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/help.go index 15916f86a..801d2b167 100644 --- a/Godeps/_workspace/src/github.com/codegangsta/cli/help.go +++ b/Godeps/_workspace/src/gopkg.in/urfave/cli.v1/help.go @@ -3,68 +3,74 @@ package cli import ( "fmt" "io" + "os" "strings" "text/tabwriter" "text/template" ) -// The text template for the Default help topic. +// AppHelpTemplate is the text template for the Default help topic. // cli.go uses text/template to render templates. You can // render custom help text by setting this variable. var AppHelpTemplate = `NAME: {{.Name}} - {{.Usage}} USAGE: - {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .Flags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}} - {{if .Version}} + {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}} + {{if .Version}}{{if not .HideVersion}} VERSION: {{.Version}} - {{end}}{{if len .Authors}} + {{end}}{{end}}{{if len .Authors}} AUTHOR(S): - {{range .Authors}}{{ . }}{{end}} - {{end}}{{if .Commands}} -COMMANDS: - {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} + {{range .Authors}}{{.}}{{end}} + {{end}}{{if .VisibleCommands}} +COMMANDS:{{range .VisibleCategories}}{{if .Name}} + {{.Name}}:{{end}}{{range .VisibleCommands}} + {{.Name}}{{with .ShortName}}, {{.}}{{end}}{{"\t"}}{{.Usage}}{{end}} +{{end}}{{end}}{{if .VisibleFlags}} GLOBAL OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}}{{if .Copyright }} + {{range .VisibleFlags}}{{.}} + {{end}}{{end}}{{if .Copyright}} COPYRIGHT: {{.Copyright}} {{end}} ` -// The text template for the command help topic. +// CommandHelpTemplate is the text template for the command help topic. // cli.go uses text/template to render templates. You can // render custom help text by setting this variable. var CommandHelpTemplate = `NAME: {{.HelpName}} - {{.Usage}} USAGE: - {{.HelpName}}{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{if .Description}} + {{.HelpName}}{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{if .Category}} + +CATEGORY: + {{.Category}}{{end}}{{if .Description}} DESCRIPTION: - {{.Description}}{{end}}{{if .Flags}} + {{.Description}}{{end}}{{if .VisibleFlags}} OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{ end }} + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} ` -// The text template for the subcommand help topic. +// SubcommandHelpTemplate is the text template for the subcommand help topic. // cli.go uses text/template to render templates. You can // render custom help text by setting this variable. var SubcommandHelpTemplate = `NAME: {{.HelpName}} - {{.Usage}} USAGE: - {{.HelpName}} command{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}} + {{.HelpName}} command{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}} -COMMANDS: - {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{if .Flags}} +COMMANDS:{{range .VisibleCategories}}{{if .Name}} + {{.Name}}:{{end}}{{range .VisibleCommands}} + {{.Name}}{{with .ShortName}}, {{.}}{{end}}{{"\t"}}{{.Usage}}{{end}} +{{end}}{{if .VisibleFlags}} OPTIONS: - {{range .Flags}}{{.}} + {{range .VisibleFlags}}{{.}} {{end}}{{end}} ` @@ -73,13 +79,14 @@ var helpCommand = Command{ Aliases: []string{"h"}, Usage: "Shows a list of commands or help for one command", ArgsUsage: "[command]", - Action: func(c *Context) { + Action: func(c *Context) error { args := c.Args() if args.Present() { - ShowCommandHelp(c, args.First()) - } else { - ShowAppHelp(c) + return ShowCommandHelp(c, args.First()) } + + ShowAppHelp(c) + return nil }, } @@ -88,65 +95,73 @@ var helpSubcommand = Command{ Aliases: []string{"h"}, Usage: "Shows a list of commands or help for one command", ArgsUsage: "[command]", - Action: func(c *Context) { + Action: func(c *Context) error { args := c.Args() if args.Present() { - ShowCommandHelp(c, args.First()) - } else { - ShowSubcommandHelp(c) + return ShowCommandHelp(c, args.First()) } + + return ShowSubcommandHelp(c) }, } // Prints help for the App or Command type helpPrinter func(w io.Writer, templ string, data interface{}) +// HelpPrinter is a function that writes the help output. If not set a default +// is used. The function signature is: +// func(w io.Writer, templ string, data interface{}) var HelpPrinter helpPrinter = printHelp -// Prints version for the App +// VersionPrinter prints the version for the App var VersionPrinter = printVersion +// ShowAppHelp is an action that displays the help. func ShowAppHelp(c *Context) { HelpPrinter(c.App.Writer, AppHelpTemplate, c.App) } -// Prints the list of subcommands as the default app completion method +// DefaultAppComplete prints the list of subcommands as the default app completion method func DefaultAppComplete(c *Context) { for _, command := range c.App.Commands { + if command.Hidden { + continue + } for _, name := range command.Names() { fmt.Fprintln(c.App.Writer, name) } } } -// Prints help for the given command -func ShowCommandHelp(ctx *Context, command string) { +// ShowCommandHelp prints help for the given command +func ShowCommandHelp(ctx *Context, command string) error { // show the subcommand help for a command with subcommands if command == "" { HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App) - return + return nil } for _, c := range ctx.App.Commands { if c.HasName(command) { HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c) - return + return nil } } - if ctx.App.CommandNotFound != nil { - ctx.App.CommandNotFound(ctx, command) - } else { - fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command) + if ctx.App.CommandNotFound == nil { + return NewExitError(fmt.Sprintf("No help topic for '%v'", command), 3) } + + ctx.App.CommandNotFound(ctx, command) + return nil } -// Prints help for the given subcommand -func ShowSubcommandHelp(c *Context) { - ShowCommandHelp(c, c.Command.Name) +// ShowSubcommandHelp prints help for the given subcommand +func ShowSubcommandHelp(c *Context) error { + return ShowCommandHelp(c, c.Command.Name) } -// Prints the version number of the App +// ShowVersion prints the version number of the App func ShowVersion(c *Context) { VersionPrinter(c) } @@ -155,7 +170,7 @@ func printVersion(c *Context) { fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version) } -// Prints the lists of commands within a given context +// ShowCompletions prints the lists of commands within a given context func ShowCompletions(c *Context) { a := c.App if a != nil && a.BashComplete != nil { @@ -163,7 +178,7 @@ func ShowCompletions(c *Context) { } } -// Prints the custom completions for a given command +// ShowCommandCompletions prints the custom completions for a given command func ShowCommandCompletions(ctx *Context, command string) { c := ctx.App.Command(command) if c != nil && c.BashComplete != nil { @@ -181,7 +196,10 @@ func printHelp(out io.Writer, templ string, data interface{}) { err := t.Execute(w, data) if err != nil { // If the writer is closed, t.Execute will fail, and there's nothing - // we can do to recover. We could send this to os.Stderr if we need. + // we can do to recover. + if os.Getenv("CLI_TEMPLATE_ERROR_DEBUG") != "" { + fmt.Fprintf(ErrWriter, "CLI TEMPLATE ERROR: %#v\n", err) + } return } w.Flush() diff --git a/accounts/abi/method.go b/accounts/abi/method.go index f3d1a44b5..d56f3bc3d 100644 --- a/accounts/abi/method.go +++ b/accounts/abi/method.go @@ -62,7 +62,7 @@ func (m Method) pack(method Method, args ...interface{}) ([]byte, error) { // calculate the offset offset := len(method.Inputs)*32 + len(variableInput) // set the offset - ret = append(ret, packNum(reflect.ValueOf(offset), UintTy)...) + ret = append(ret, packNum(reflect.ValueOf(offset))...) // Append the packed output to the variable input. The variable input // will be appended at the end of the input. variableInput = append(variableInput, packed...) diff --git a/accounts/abi/numbers.go b/accounts/abi/numbers.go index 5a31cf2b5..06c4422f9 100644 --- a/accounts/abi/numbers.go +++ b/accounts/abi/numbers.go @@ -61,54 +61,20 @@ func U256(n *big.Int) []byte { return common.LeftPadBytes(common.U256(n).Bytes(), 32) } -func S256(n *big.Int) []byte { - sint := common.S256(n) - ret := common.LeftPadBytes(sint.Bytes(), 32) - if sint.Cmp(common.Big0) < 0 { - for i, b := range ret { - if b == 0 { - ret[i] = 1 - continue - } - break - } - } - - return ret -} - // S256 will ensure signed 256bit on big nums func U2U256(n uint64) []byte { return U256(big.NewInt(int64(n))) } -func S2S256(n int64) []byte { - return S256(big.NewInt(n)) -} - // packNum packs the given number (using the reflect value) and will cast it to appropriate number representation -func packNum(value reflect.Value, to byte) []byte { +func packNum(value reflect.Value) []byte { switch kind := value.Kind(); kind { case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - if to == UintTy { - return U2U256(value.Uint()) - } else { - return S2S256(int64(value.Uint())) - } + return U2U256(value.Uint()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if to == UintTy { - return U2U256(uint64(value.Int())) - } else { - return S2S256(value.Int()) - } + return U2U256(uint64(value.Int())) case reflect.Ptr: - // This only takes care of packing and casting. No type checking is done here. It should be done prior to using this function. - if to == UintTy { - return U256(value.Interface().(*big.Int)) - } else { - return S256(value.Interface().(*big.Int)) - } - + return U256(value.Interface().(*big.Int)) } return nil diff --git a/accounts/abi/numbers_test.go b/accounts/abi/numbers_test.go index d66a43258..f409aa60f 100644 --- a/accounts/abi/numbers_test.go +++ b/accounts/abi/numbers_test.go @@ -26,48 +26,28 @@ import ( func TestNumberTypes(t *testing.T) { ubytes := make([]byte, 32) ubytes[31] = 1 - sbytesmin := []byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} unsigned := U256(big.NewInt(1)) if !bytes.Equal(unsigned, ubytes) { t.Errorf("expected %x got %x", ubytes, unsigned) } - - signed := S256(big.NewInt(1)) - if !bytes.Equal(signed, ubytes) { - t.Errorf("expected %x got %x", ubytes, unsigned) - } - - signed = S256(big.NewInt(-1)) - if !bytes.Equal(signed, sbytesmin) { - t.Errorf("expected %x got %x", ubytes, unsigned) - } } func TestPackNumber(t *testing.T) { ubytes := make([]byte, 32) ubytes[31] = 1 - sbytesmin := []byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} maxunsigned := []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255} - packed := packNum(reflect.ValueOf(1), IntTy) - if !bytes.Equal(packed, ubytes) { - t.Errorf("expected %x got %x", ubytes, packed) - } - packed = packNum(reflect.ValueOf(-1), IntTy) - if !bytes.Equal(packed, sbytesmin) { - t.Errorf("expected %x got %x", ubytes, packed) - } - packed = packNum(reflect.ValueOf(1), UintTy) + packed := packNum(reflect.ValueOf(1)) if !bytes.Equal(packed, ubytes) { t.Errorf("expected %x got %x", ubytes, packed) } - packed = packNum(reflect.ValueOf(-1), UintTy) + packed = packNum(reflect.ValueOf(-1)) if !bytes.Equal(packed, maxunsigned) { t.Errorf("expected %x got %x", maxunsigned, packed) } - packed = packNum(reflect.ValueOf("string"), UintTy) + packed = packNum(reflect.ValueOf("string")) if packed != nil { t.Errorf("expected 'string' to pack to nil. got %x instead", packed) } diff --git a/accounts/abi/packing.go b/accounts/abi/packing.go index c765dfdf3..0c37edf17 100644 --- a/accounts/abi/packing.go +++ b/accounts/abi/packing.go @@ -25,7 +25,7 @@ import ( // packBytesSlice packs the given bytes as [L, V] as the canonical representation // bytes slice func packBytesSlice(bytes []byte, l int) []byte { - len := packNum(reflect.ValueOf(l), UintTy) + len := packNum(reflect.ValueOf(l)) return append(len, common.RightPadBytes(bytes, (l+31)/32*32)...) } @@ -34,7 +34,7 @@ func packBytesSlice(bytes []byte, l int) []byte { func packElement(t Type, reflectValue reflect.Value) []byte { switch t.T { case IntTy, UintTy: - return packNum(reflectValue, t.T) + return packNum(reflectValue) case StringTy: return packBytesSlice([]byte(reflectValue.String()), reflectValue.Len()) case AddressTy: diff --git a/cmd/ethtest/main.go b/cmd/ethtest/main.go index d8f969636..e0ad0a7ea 100644 --- a/cmd/ethtest/main.go +++ b/cmd/ethtest/main.go @@ -25,10 +25,10 @@ import ( "path/filepath" "strings" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/tests" + "gopkg.in/urfave/cli.v1" ) var ( @@ -183,7 +183,7 @@ func runSuite(test, file string) { } } -func setupApp(c *cli.Context) { +func setupApp(c *cli.Context) error { flagTest := c.GlobalString(TestFlag.Name) flagFile := c.GlobalString(FileFlag.Name) continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name) @@ -196,8 +196,8 @@ func setupApp(c *cli.Context) { if err := runTestWithReader(flagTest, os.Stdin); err != nil { glog.Fatalln(err) } - } + return nil } func main() { diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 7d9b3a6c3..e7b266d4e 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -24,7 +24,6 @@ import ( "runtime" "time" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -33,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger/glog" + "gopkg.in/urfave/cli.v1" ) var ( @@ -104,7 +104,7 @@ func init() { app.Action = run } -func run(ctx *cli.Context) { +func run(ctx *cli.Context) error { glog.SetToStderr(true) glog.SetV(ctx.GlobalInt(VerbosityFlag.Name)) @@ -154,6 +154,7 @@ num gc: %d fmt.Printf(" error: %v", e) } fmt.Println() + return nil } func main() { diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0f9d95c2c..1415240eb 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -20,13 +20,13 @@ import ( "fmt" "io/ioutil" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "gopkg.in/urfave/cli.v1" ) var ( @@ -70,7 +70,7 @@ either new or import). Without it you are not able to unlock your account. Note that exporting your key in unencrypted format is NOT supported. -Keys are stored under <DATADIR>/keys. +Keys are stored under <DATADIR>/keystore. It is safe to transfer the entire directory or the individual keys therein between ethereum nodes by simply copying. Make sure you backup your keys regularly. @@ -167,11 +167,12 @@ nodes. } ) -func accountList(ctx *cli.Context) { +func accountList(ctx *cli.Context) error { accman := utils.MakeAccountManager(ctx) for i, acct := range accman.Accounts() { fmt.Printf("Account #%d: {%x} %s\n", i, acct.Address, acct.File) } + return nil } // tries unlocking the specified account a few times. @@ -259,7 +260,7 @@ func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrErro } // accountCreate creates a new account into the keystore defined by the CLI flags. -func accountCreate(ctx *cli.Context) { +func accountCreate(ctx *cli.Context) error { accman := utils.MakeAccountManager(ctx) password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) @@ -268,11 +269,12 @@ func accountCreate(ctx *cli.Context) { utils.Fatalf("Failed to create account: %v", err) } fmt.Printf("Address: {%x}\n", account.Address) + return nil } // accountUpdate transitions an account from a previous format to the current // one, also providing the possibility to change the pass-phrase. -func accountUpdate(ctx *cli.Context) { +func accountUpdate(ctx *cli.Context) error { if len(ctx.Args()) == 0 { utils.Fatalf("No accounts specified to update") } @@ -283,9 +285,10 @@ func accountUpdate(ctx *cli.Context) { if err := accman.Update(account, oldPassword, newPassword); err != nil { utils.Fatalf("Could not update the account: %v", err) } + return nil } -func importWallet(ctx *cli.Context) { +func importWallet(ctx *cli.Context) error { keyfile := ctx.Args().First() if len(keyfile) == 0 { utils.Fatalf("keyfile must be given as argument") @@ -303,9 +306,10 @@ func importWallet(ctx *cli.Context) { utils.Fatalf("%v", err) } fmt.Printf("Address: {%x}\n", acct.Address) + return nil } -func accountImport(ctx *cli.Context) { +func accountImport(ctx *cli.Context) error { keyfile := ctx.Args().First() if len(keyfile) == 0 { utils.Fatalf("keyfile must be given as argument") @@ -321,4 +325,5 @@ func accountImport(ctx *cli.Context) { utils.Fatalf("Could not create the account: %v", err) } fmt.Printf("Address: {%x}\n", acct.Address) + return nil } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 4f47de5d7..321551ce0 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -23,7 +23,6 @@ import ( "strconv" "time" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" @@ -32,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger/glog" + "gopkg.in/urfave/cli.v1" ) var ( @@ -72,7 +72,7 @@ Use "ethereum dump 0" to dump the genesis block. } ) -func importChain(ctx *cli.Context) { +func importChain(ctx *cli.Context) error { if len(ctx.Args()) != 1 { utils.Fatalf("This command requires an argument.") } @@ -84,9 +84,10 @@ func importChain(ctx *cli.Context) { utils.Fatalf("Import error: %v", err) } fmt.Printf("Import done in %v", time.Since(start)) + return nil } -func exportChain(ctx *cli.Context) { +func exportChain(ctx *cli.Context) error { if len(ctx.Args()) < 1 { utils.Fatalf("This command requires an argument.") } @@ -114,9 +115,10 @@ func exportChain(ctx *cli.Context) { utils.Fatalf("Export error: %v\n", err) } fmt.Printf("Export done in %v", time.Since(start)) + return nil } -func removeDB(ctx *cli.Context) { +func removeDB(ctx *cli.Context) error { confirm, err := console.Stdin.PromptConfirm("Remove local database?") if err != nil { utils.Fatalf("%v", err) @@ -132,9 +134,10 @@ func removeDB(ctx *cli.Context) { } else { fmt.Println("Operation aborted") } + return nil } -func upgradeDB(ctx *cli.Context) { +func upgradeDB(ctx *cli.Context) error { glog.Infoln("Upgrading blockchain database") chain, chainDb := utils.MakeChain(ctx) @@ -163,14 +166,15 @@ func upgradeDB(ctx *cli.Context) { os.Remove(exportFile) glog.Infoln("Import finished") } + return nil } -func dump(ctx *cli.Context) { +func dump(ctx *cli.Context) error { chain, chainDb := utils.MakeChain(ctx) for _, arg := range ctx.Args() { var block *types.Block if hashish(arg) { - block = chain.GetBlock(common.HexToHash(arg)) + block = chain.GetBlockByHash(common.HexToHash(arg)) } else { num, _ := strconv.Atoi(arg) block = chain.GetBlockByNumber(uint64(num)) @@ -182,12 +186,12 @@ func dump(ctx *cli.Context) { state, err := state.New(block.Root(), chainDb) if err != nil { utils.Fatalf("could not create new state: %v", err) - return } fmt.Printf("%s\n", state.Dump()) } } chainDb.Close() + return nil } // hashish returns true for strings that look like hashes. diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 8bfe27fef..257050a62 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -20,9 +20,9 @@ import ( "os" "os/signal" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" + "gopkg.in/urfave/cli.v1" ) var ( @@ -60,7 +60,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso // localConsole starts a new geth node, attaching a JavaScript console to it at the // same time. -func localConsole(ctx *cli.Context) { +func localConsole(ctx *cli.Context) error { // Create and start the node based on the CLI flags node := utils.MakeSystemNode(clientIdentifier, verString, relConfig, makeDefaultExtra(), ctx) startNode(ctx, node) @@ -86,16 +86,18 @@ func localConsole(ctx *cli.Context) { // If only a short execution was requested, evaluate and return if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { console.Evaluate(script) - return + return nil } // Otherwise print the welcome screen and enter interactive mode console.Welcome() console.Interactive() + + return nil } // remoteConsole will connect to a remote geth instance, attaching a JavaScript // console to it. -func remoteConsole(ctx *cli.Context) { +func remoteConsole(ctx *cli.Context) error { // Attach to a remotely running geth instance and start the JavaScript console client, err := utils.NewRemoteRPCClient(ctx) if err != nil { @@ -116,17 +118,19 @@ func remoteConsole(ctx *cli.Context) { // If only a short execution was requested, evaluate and return if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { console.Evaluate(script) - return + return nil } // Otherwise print the welcome screen and enter interactive mode console.Welcome() console.Interactive() + + return nil } // ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript // console to it, and each of the files specified as arguments and tears the // everything down. -func ephemeralConsole(ctx *cli.Context) { +func ephemeralConsole(ctx *cli.Context) error { // Create and start the node based on the CLI flags node := utils.MakeSystemNode(clientIdentifier, verString, relConfig, makeDefaultExtra(), ctx) startNode(ctx, node) @@ -164,4 +168,6 @@ func ephemeralConsole(ctx *cli.Context) { os.Exit(0) }() console.Stop(true) + + return nil } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index e59fe1415..e0e549e12 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -17,7 +17,8 @@ package main import ( - "math/rand" + "crypto/rand" + "math/big" "os" "path/filepath" "runtime" @@ -73,7 +74,7 @@ func TestIPCAttachWelcome(t *testing.T) { coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182" var ipc string if runtime.GOOS == "windows" { - ipc = `\\.\pipe\geth` + strconv.Itoa(rand.Int()) + ipc = `\\.\pipe\geth` + strconv.Itoa(trulyRandInt(100000, 999999)) } else { ws := tmpdir(t) defer os.RemoveAll(ws) @@ -94,7 +95,7 @@ func TestIPCAttachWelcome(t *testing.T) { func TestHTTPAttachWelcome(t *testing.T) { coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182" - port := strconv.Itoa(rand.Intn(65535-1024) + 1024) // Yeah, sometimes this will fail, sorry :P + port := strconv.Itoa(trulyRandInt(1024, 65536)) // Yeah, sometimes this will fail, sorry :P geth := runGeth(t, "--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--etherbase", coinbase, "--rpc", "--rpcport", port) @@ -108,7 +109,7 @@ func TestHTTPAttachWelcome(t *testing.T) { func TestWSAttachWelcome(t *testing.T) { coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182" - port := strconv.Itoa(rand.Intn(65535-1024) + 1024) // Yeah, sometimes this will fail, sorry :P + port := strconv.Itoa(trulyRandInt(1024, 65536)) // Yeah, sometimes this will fail, sorry :P geth := runGeth(t, "--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", @@ -160,3 +161,10 @@ at block: 0 ({{niltime}}){{if ipc}} `) attach.expectExit() } + +// trulyRandInt generates a crypto random integer used by the console tests to +// not clash network ports with other tests running cocurrently. +func trulyRandInt(lo, hi int) int { + num, _ := rand.Int(rand.Reader, big.NewInt(int64(hi-lo))) + return int(num.Int64()) + lo +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 5ff1a7368..c372430f1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -28,7 +28,6 @@ import ( "strings" "time" - "github.com/codegangsta/cli" "github.com/ethereum/ethash" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" @@ -44,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/release" "github.com/ethereum/go-ethereum/rlp" + "gopkg.in/urfave/cli.v1" ) const ( @@ -271,15 +271,17 @@ func makeDefaultExtra() []byte { // geth is the main entry point into the system if no special subcommand is ran. // It creates a default node based on the command line arguments and runs it in // blocking mode, waiting for it to be shut down. -func geth(ctx *cli.Context) { +func geth(ctx *cli.Context) error { node := utils.MakeSystemNode(clientIdentifier, verString, relConfig, makeDefaultExtra(), ctx) startNode(ctx, node) node.Wait() + + return nil } // initGenesis will initialise the given JSON format genesis file and writes it as // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. -func initGenesis(ctx *cli.Context) { +func initGenesis(ctx *cli.Context) error { genesisPath := ctx.Args().First() if len(genesisPath) == 0 { utils.Fatalf("must supply path to genesis JSON file") @@ -300,6 +302,7 @@ func initGenesis(ctx *cli.Context) { utils.Fatalf("failed to write genesis block: %v", err) } glog.V(logger.Info).Infof("successfully wrote genesis block and/or chain rule set: %x", block.Hash()) + return nil } // startNode boots up the system node and all registered protocols, after which @@ -331,7 +334,7 @@ func startNode(ctx *cli.Context, stack *node.Node) { } } -func makedag(ctx *cli.Context) { +func makedag(ctx *cli.Context) error { args := ctx.Args() wrongArgs := func() { utils.Fatalf(`Usage: geth makedag <block number> <outputdir>`) @@ -358,13 +361,15 @@ func makedag(ctx *cli.Context) { default: wrongArgs() } + return nil } -func gpuinfo(ctx *cli.Context) { +func gpuinfo(ctx *cli.Context) error { eth.PrintOpenCLDevices() + return nil } -func gpubench(ctx *cli.Context) { +func gpubench(ctx *cli.Context) error { args := ctx.Args() wrongArgs := func() { utils.Fatalf(`Usage: geth gpubench <gpu number>`) @@ -381,9 +386,10 @@ func gpubench(ctx *cli.Context) { default: wrongArgs() } + return nil } -func version(c *cli.Context) { +func version(c *cli.Context) error { fmt.Println(clientIdentifier) fmt.Println("Version:", verString) fmt.Println("Protocol Versions:", eth.ProtocolVersions) @@ -392,4 +398,6 @@ func version(c *cli.Context) { fmt.Println("OS:", runtime.GOOS) fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH")) fmt.Printf("GOROOT=%s\n", runtime.GOROOT()) + + return nil } diff --git a/cmd/geth/monitorcmd.go b/cmd/geth/monitorcmd.go index 5d839b5a3..11fdca89c 100644 --- a/cmd/geth/monitorcmd.go +++ b/cmd/geth/monitorcmd.go @@ -26,11 +26,11 @@ import ( "sort" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/gizak/termui" + "gopkg.in/urfave/cli.v1" ) var ( @@ -67,7 +67,7 @@ to display multiple metrics simultaneously. ) // monitor starts a terminal UI based monitoring tool for the requested metrics. -func monitor(ctx *cli.Context) { +func monitor(ctx *cli.Context) error { var ( client rpc.Client err error @@ -154,6 +154,7 @@ func monitor(ctx *cli.Context) { } }() termui.Loop() + return nil } // retrieveMetrics contacts the attached geth node and retrieves the entire set diff --git a/cmd/geth/run_test.go b/cmd/geth/run_test.go index f6bc3f869..e26b4509a 100644 --- a/cmd/geth/run_test.go +++ b/cmd/geth/run_test.go @@ -58,7 +58,10 @@ type testgeth struct { func init() { // Run the app if we're the child process for runGeth. if os.Getenv("GETH_TEST_CHILD") != "" { - app.RunAndExitOnError() + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } os.Exit(0) } } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 01a71c1f6..e7ef9e2c7 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -21,9 +21,9 @@ package main import ( "io" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/debug" + "gopkg.in/urfave/cli.v1" ) // AppHelpTemplate is the test template for the default, global app help topic. diff --git a/cmd/utils/client.go b/cmd/utils/client.go index ec72a1a4b..cc9647580 100644 --- a/cmd/utils/client.go +++ b/cmd/utils/client.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" + "gopkg.in/urfave/cli.v1" ) // NewRemoteRPCClient returns a RPC client which connects to a running geth instance. diff --git a/cmd/utils/customflags.go b/cmd/utils/customflags.go index 4450065c1..5cbccfe98 100644 --- a/cmd/utils/customflags.go +++ b/cmd/utils/customflags.go @@ -24,7 +24,7 @@ import ( "path" "strings" - "github.com/codegangsta/cli" + "gopkg.in/urfave/cli.v1" ) // Custom type which is registered in the flags library which cli uses for diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c476e1c77..14898b987 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -30,7 +30,6 @@ import ( "strings" "time" - "github.com/codegangsta/cli" "github.com/ethereum/ethash" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -51,6 +50,7 @@ import ( "github.com/ethereum/go-ethereum/release" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/whisper" + "gopkg.in/urfave/cli.v1" ) func init() { @@ -806,7 +806,7 @@ func MustMakeChainConfig(ctx *cli.Context) *core.ChainConfig { // MustMakeChainConfigFromDb reads the chain configuration from the given database. func MustMakeChainConfigFromDb(ctx *cli.Context, db ethdb.Database) *core.ChainConfig { - genesis := core.GetBlock(db, core.GetCanonicalHash(db, 0)) + genesis := core.GetBlock(db, core.GetCanonicalHash(db, 0), 0) if genesis != nil { // Existing genesis block, use stored config if available. diff --git a/console/console.go b/console/console.go index baa9cf545..00d1fea1d 100644 --- a/console/console.go +++ b/console/console.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/internal/jsre" "github.com/ethereum/go-ethereum/internal/web3ext" "github.com/ethereum/go-ethereum/rpc" + "github.com/mattn/go-colorable" "github.com/peterh/liner" "github.com/robertkrimen/otto" ) @@ -80,7 +81,7 @@ func New(config Config) (*Console, error) { config.Prompt = DefaultPrompt } if config.Printer == nil { - config.Printer = os.Stdout + config.Printer = colorable.NewColorableStdout() } // Initialize the console and return console := &Console{ @@ -330,11 +331,11 @@ func (c *Console) Interactive() { // Append the line to the input and check for multi-line interpretation input += line + "\n" - indents = strings.Count(input, "{") + strings.Count(input, "(") - strings.Count(input, "}") - strings.Count(input, ")") + indents = countIndents(input) if indents <= 0 { prompt = c.prompt } else { - prompt = strings.Repeat("..", indents*2) + " " + prompt = strings.Repeat(".", indents*3) + " " } // If all the needed lines are present, save the command and run if indents <= 0 { @@ -353,6 +354,49 @@ func (c *Console) Interactive() { } } +// countIndents returns the number of identations for the given input. +// In case of invalid input such as var a = } the result can be negative. +func countIndents(input string) int { + var ( + indents = 0 + inString = false + strOpenChar = ' ' // keep track of the string open char to allow var str = "I'm ...."; + charEscaped = false // keep track if the previous char was the '\' char, allow var str = "abc\"def"; + ) + + for _, c := range input { + switch c { + case '\\': + // indicate next char as escaped when in string and previous char isn't escaping this backslash + if !charEscaped && inString { + charEscaped = true + } + case '\'', '"': + if inString && !charEscaped && strOpenChar == c { // end string + inString = false + } else if !inString && !charEscaped { // begin string + inString = true + strOpenChar = c + } + charEscaped = false + case '{', '(': + if !inString { // ignore brackets when in string, allow var str = "a{"; without indenting + indents++ + } + charEscaped = false + case '}', ')': + if !inString { + indents-- + } + charEscaped = false + default: + charEscaped = false + } + } + + return indents +} + // Execute runs the JavaScript file specified as the argument. func (c *Console) Execute(path string) error { return c.jsre.Exec(path) diff --git a/console/console_test.go b/console/console_test.go index 911087824..7738d0c44 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -294,3 +294,49 @@ func TestPrettyError(t *testing.T) { t.Fatalf("pretty error mismatch: have %s, want %s", output, want) } } + +// Tests that tests if the number of indents for JS input is calculated correct. +func TestIndenting(t *testing.T) { + testCases := []struct { + input string + expectedIndentCount int + }{ + {`var a = 1;`, 0}, + {`"some string"`, 0}, + {`"some string with (parentesis`, 0}, + {`"some string with newline + ("`, 0}, + {`function v(a,b) {}`, 0}, + {`function f(a,b) { var str = "asd("; };`, 0}, + {`function f(a) {`, 1}, + {`function f(a, function(b) {`, 2}, + {`function f(a, function(b) { + var str = "a)}"; + });`, 0}, + {`function f(a,b) { + var str = "a{b(" + a, ", " + b; + }`, 0}, + {`var str = "\"{"`, 0}, + {`var str = "'("`, 0}, + {`var str = "\\{"`, 0}, + {`var str = "\\\\{"`, 0}, + {`var str = 'a"{`, 0}, + {`var obj = {`, 1}, + {`var obj = { {a:1`, 2}, + {`var obj = { {a:1}`, 1}, + {`var obj = { {a:1}, b:2}`, 0}, + {`var obj = {}`, 0}, + {`var obj = { + a: 1, b: 2 + }`, 0}, + {`var test = }`, -1}, + {`var str = "a\""; var obj = {`, 1}, + } + + for i, tt := range testCases { + counted := countIndents(tt.input) + if counted != tt.expectedIndentCount { + t.Errorf("test %d: invalid indenting: have %d, want %d", i, counted, tt.expectedIndentCount) + } + } +} diff --git a/core/bench_test.go b/core/bench_test.go index ac5b57bc8..c6029499a 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -176,3 +176,122 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { b.Fatalf("insert error (block %d): %v\n", i, err) } } + +func BenchmarkChainRead_header_10k(b *testing.B) { + benchReadChain(b, false, 10000) +} +func BenchmarkChainRead_full_10k(b *testing.B) { + benchReadChain(b, true, 10000) +} +func BenchmarkChainRead_header_100k(b *testing.B) { + benchReadChain(b, false, 100000) +} +func BenchmarkChainRead_full_100k(b *testing.B) { + benchReadChain(b, true, 100000) +} +func BenchmarkChainRead_header_500k(b *testing.B) { + benchReadChain(b, false, 500000) +} +func BenchmarkChainRead_full_500k(b *testing.B) { + benchReadChain(b, true, 500000) +} +func BenchmarkChainWrite_header_10k(b *testing.B) { + benchWriteChain(b, false, 10000) +} +func BenchmarkChainWrite_full_10k(b *testing.B) { + benchWriteChain(b, true, 10000) +} +func BenchmarkChainWrite_header_100k(b *testing.B) { + benchWriteChain(b, false, 100000) +} +func BenchmarkChainWrite_full_100k(b *testing.B) { + benchWriteChain(b, true, 100000) +} +func BenchmarkChainWrite_header_500k(b *testing.B) { + benchWriteChain(b, false, 500000) +} +func BenchmarkChainWrite_full_500k(b *testing.B) { + benchWriteChain(b, true, 500000) +} + +// makeChainForBench writes a given number of headers or empty blocks/receipts +// into a database. +func makeChainForBench(db ethdb.Database, full bool, count uint64) { + var hash common.Hash + for n := uint64(0); n < count; n++ { + header := &types.Header{ + Coinbase: common.Address{}, + Number: big.NewInt(int64(n)), + ParentHash: hash, + Difficulty: big.NewInt(1), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + } + hash = header.Hash() + WriteHeader(db, header) + WriteCanonicalHash(db, hash, n) + WriteTd(db, hash, n, big.NewInt(int64(n+1))) + if full || n == 0 { + block := types.NewBlockWithHeader(header) + WriteBody(db, hash, n, block.Body()) + WriteBlockReceipts(db, hash, n, nil) + } + } +} + +func benchWriteChain(b *testing.B, full bool, count uint64) { + for i := 0; i < b.N; i++ { + dir, err := ioutil.TempDir("", "eth-chain-bench") + if err != nil { + b.Fatalf("cannot create temporary directory: %v", err) + } + db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", dir, err) + } + makeChainForBench(db, full, count) + db.Close() + os.RemoveAll(dir) + } +} + +func benchReadChain(b *testing.B, full bool, count uint64) { + dir, err := ioutil.TempDir("", "eth-chain-bench") + if err != nil { + b.Fatalf("cannot create temporary directory: %v", err) + } + defer os.RemoveAll(dir) + + db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", dir, err) + } + makeChainForBench(db, full, count) + db.Close() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", dir, err) + } + chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux)) + if err != nil { + b.Fatalf("error creating chain: %v", err) + } + + for n := uint64(0); n < count; n++ { + header := chain.GetHeaderByNumber(n) + if full { + hash := header.Hash() + GetBody(db, hash, n) + GetBlockReceipts(db, hash, n) + } + } + + db.Close() + } +} diff --git a/core/block_validator.go b/core/block_validator.go index 801d2572b..f777b9f23 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -72,7 +72,7 @@ func (v *BlockValidator) ValidateBlock(block *types.Block) error { return &KnownBlockError{block.Number(), block.Hash()} } } - parent := v.bc.GetBlock(block.ParentHash()) + parent := v.bc.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { return ParentError(block.ParentHash()) } diff --git a/core/blockchain.go b/core/blockchain.go index bd84adfe9..95bada5ee 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -54,9 +54,7 @@ var ( ) const ( - headerCacheLimit = 512 bodyCacheLimit = 256 - tdCacheLimit = 1024 blockCacheLimit = 256 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 @@ -151,7 +149,7 @@ func NewBlockChain(chainDb ethdb.Database, config *ChainConfig, pow pow.PoW, mux } // Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain for hash, _ := range BadHashes { - if header := bc.GetHeader(hash); header != nil { + if header := bc.GetHeaderByHash(hash); header != nil { glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%x…]", header.Number, header.ParentHash[:4]) bc.SetHead(header.Number.Uint64() - 1) glog.V(logger.Error).Infoln("Chain rewind was successful, resuming normal operation") @@ -175,7 +173,7 @@ func (self *BlockChain) loadLastState() error { // Corrupt or empty database, init from scratch self.Reset() } else { - if block := self.GetBlock(head); block != nil { + if block := self.GetBlockByHash(head); block != nil { // Block found, set as the current head self.currentBlock = block } else { @@ -186,7 +184,7 @@ func (self *BlockChain) loadLastState() error { // Restore the last known head header currentHeader := self.currentBlock.Header() if head := GetHeadHeaderHash(self.chainDb); head != (common.Hash{}) { - if header := self.GetHeader(head); header != nil { + if header := self.GetHeaderByHash(head); header != nil { currentHeader = header } } @@ -194,16 +192,16 @@ func (self *BlockChain) loadLastState() error { // Restore the last known head fast block self.currentFastBlock = self.currentBlock if head := GetHeadFastBlockHash(self.chainDb); head != (common.Hash{}) { - if block := self.GetBlock(head); block != nil { + if block := self.GetBlockByHash(head); block != nil { self.currentFastBlock = block } } // Issue a status log and return - headerTd := self.GetTd(self.hc.CurrentHeader().Hash()) - blockTd := self.GetTd(self.currentBlock.Hash()) - fastTd := self.GetTd(self.currentFastBlock.Hash()) + headerTd := self.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) + blockTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) + fastTd := self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()) - glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", self.hc.CurrentHeader().Number, self.hc.CurrentHeader().Hash().Bytes()[:4], headerTd) + glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", currentHeader.Number, currentHeader.Hash().Bytes()[:4], headerTd) glog.V(logger.Info).Infof("Last block: #%d [%x…] TD=%v", self.currentBlock.Number(), self.currentBlock.Hash().Bytes()[:4], blockTd) glog.V(logger.Info).Infof("Fast block: #%d [%x…] TD=%v", self.currentFastBlock.Number(), self.currentFastBlock.Hash().Bytes()[:4], fastTd) @@ -218,8 +216,8 @@ func (bc *BlockChain) SetHead(head uint64) { bc.mu.Lock() defer bc.mu.Unlock() - delFn := func(hash common.Hash) { - DeleteBody(bc.chainDb, hash) + delFn := func(hash common.Hash, num uint64) { + DeleteBody(bc.chainDb, hash, num) } bc.hc.SetHead(head, delFn) @@ -230,11 +228,12 @@ func (bc *BlockChain) SetHead(head uint64) { bc.futureBlocks.Purge() // Update all computed fields to the new head - if bc.currentBlock != nil && bc.hc.CurrentHeader().Number.Uint64() < bc.currentBlock.NumberU64() { - bc.currentBlock = bc.GetBlock(bc.hc.CurrentHeader().Hash()) + currentHeader := bc.hc.CurrentHeader() + if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() { + bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } - if bc.currentFastBlock != nil && bc.hc.CurrentHeader().Number.Uint64() < bc.currentFastBlock.NumberU64() { - bc.currentFastBlock = bc.GetBlock(bc.hc.CurrentHeader().Hash()) + if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() { + bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } if bc.currentBlock == nil { @@ -257,7 +256,7 @@ func (bc *BlockChain) SetHead(head uint64) { // irrelevant what the chain contents were prior. func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error { // Make sure that both the block as well at its state trie exists - block := self.GetBlock(hash) + block := self.GetBlockByHash(hash) if block == nil { return fmt.Errorf("non existent block [%x…]", hash[:4]) } @@ -313,7 +312,7 @@ func (self *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesis self.mu.RLock() defer self.mu.RUnlock() - return self.GetTd(self.currentBlock.Hash()), self.currentBlock.Hash(), self.genesisBlock.Hash() + return self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()), self.currentBlock.Hash(), self.genesisBlock.Hash() } // SetProcessor sets the processor required for making state modifications. @@ -367,7 +366,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) { defer bc.mu.Unlock() // Prepare the genesis block and reinitialise the chain - if err := bc.hc.WriteTd(genesis.Hash(), genesis.Difficulty()); err != nil { + if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { glog.Fatalf("failed to write genesis block TD: %v", err) } if err := WriteBlock(bc.chainDb, genesis); err != nil { @@ -457,7 +456,7 @@ func (self *BlockChain) GetBody(hash common.Hash) *types.Body { body := cached.(*types.Body) return body } - body := GetBody(self.chainDb, hash) + body := GetBody(self.chainDb, hash, self.hc.GetBlockNumber(hash)) if body == nil { return nil } @@ -473,7 +472,7 @@ func (self *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { if cached, ok := self.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(self.chainDb, hash) + body := GetBodyRLP(self.chainDb, hash, self.hc.GetBlockNumber(hash)) if len(body) == 0 { return nil } @@ -485,14 +484,14 @@ func (self *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { // HasBlock checks if a block is fully present in the database or not, caching // it if present. func (bc *BlockChain) HasBlock(hash common.Hash) bool { - return bc.GetBlock(hash) != nil + return bc.GetBlockByHash(hash) != nil } // HasBlockAndState checks if a block and associated state trie is fully present // in the database or not, caching it if present. func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { // Check first that the block itself is known - block := bc.GetBlock(hash) + block := bc.GetBlockByHash(hash) if block == nil { return false } @@ -501,13 +500,14 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { return err == nil } -// GetBlock retrieves a block from the database by hash, caching it if found. -func (self *BlockChain) GetBlock(hash common.Hash) *types.Block { +// GetBlock retrieves a block from the database by hash and number, +// caching it if found. +func (self *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { // Short circuit if the block's already in the cache, retrieve otherwise if block, ok := self.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(self.chainDb, hash) + block := GetBlock(self.chainDb, hash, number) if block == nil { return nil } @@ -516,6 +516,11 @@ func (self *BlockChain) GetBlock(hash common.Hash) *types.Block { return block } +// GetBlockByHash retrieves a block from the database by hash, caching it if found. +func (self *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + return self.GetBlock(hash, self.hc.GetBlockNumber(hash)) +} + // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block { @@ -523,19 +528,21 @@ func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block { if hash == (common.Hash{}) { return nil } - return self.GetBlock(hash) + return self.GetBlock(hash, number) } // [deprecated by eth/62] // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { + number := self.hc.GetBlockNumber(hash) for i := 0; i < n; i++ { - block := self.GetBlock(hash) + block := self.GetBlock(hash, number) if block == nil { break } blocks = append(blocks, block) hash = block.ParentHash() + number-- } return } @@ -546,7 +553,7 @@ func (self *BlockChain) GetUnclesInChain(block *types.Block, length int) []*type uncles := []*types.Header{} for i := 0; block != nil && i < length; i++ { uncles = append(uncles, block.Uncles()...) - block = self.GetBlock(block.ParentHash()) + block = self.GetBlock(block.ParentHash(), block.NumberU64()-1) } return uncles } @@ -596,15 +603,16 @@ func (self *BlockChain) Rollback(chain []common.Hash) { for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] - if self.hc.CurrentHeader().Hash() == hash { - self.hc.SetCurrentHeader(self.GetHeader(self.hc.CurrentHeader().ParentHash)) + currentHeader := self.hc.CurrentHeader() + if currentHeader.Hash() == hash { + self.hc.SetCurrentHeader(self.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) } if self.currentFastBlock.Hash() == hash { - self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash()) + self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash(), self.currentFastBlock.NumberU64()-1) WriteHeadFastBlockHash(self.chainDb, self.currentFastBlock.Hash()) } if self.currentBlock.Hash() == hash { - self.currentBlock = self.GetBlock(self.currentBlock.ParentHash()) + self.currentBlock = self.GetBlock(self.currentBlock.ParentHash(), self.currentBlock.NumberU64()-1) WriteHeadBlockHash(self.chainDb, self.currentBlock.Hash()) } } @@ -678,13 +686,13 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain } } // Write all the data out into the database - if err := WriteBody(self.chainDb, block.Hash(), block.Body()); err != nil { + if err := WriteBody(self.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { errs[index] = fmt.Errorf("failed to write block body: %v", err) atomic.AddInt32(&failed, 1) glog.Fatal(errs[index]) return } - if err := WriteBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil { + if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { errs[index] = fmt.Errorf("failed to write block receipts: %v", err) atomic.AddInt32(&failed, 1) glog.Fatal(errs[index]) @@ -737,7 +745,7 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain // Update the head fast sync block if better self.mu.Lock() head := blockChain[len(errs)-1] - if self.GetTd(self.currentFastBlock.Hash()).Cmp(self.GetTd(head.Hash())) < 0 { + if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(self.GetTd(head.Hash(), head.NumberU64())) < 0 { if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil { glog.Fatalf("failed to update head fast block hash: %v", err) } @@ -759,12 +767,12 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err defer self.wg.Done() // Calculate the total difficulty of the block - ptd := self.GetTd(block.ParentHash()) + ptd := self.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { return NonStatTy, ParentError(block.ParentHash()) } - localTd := self.GetTd(self.currentBlock.Hash()) + localTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) // Make sure no inconsistent state is leaked during insertion @@ -788,7 +796,7 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err status = SideStatTy } // Irrelevant of the canonical status, write the block itself to the database - if err := self.hc.WriteTd(block.Hash(), externTd); err != nil { + if err := self.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { glog.Fatalf("failed to write block total difficulty: %v", err) } if err := WriteBlock(self.chainDb, block); err != nil { @@ -887,7 +895,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { // Create a new statedb using the parent block and report an // error if it fails. if statedb == nil { - statedb, err = state.New(self.GetBlock(block.ParentHash()).Root(), self.chainDb) + statedb, err = state.New(self.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), self.chainDb) } else { err = statedb.Reset(chain[i-1].Root()) } @@ -902,7 +910,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { return i, err } // Validate the state using the default validator - err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash()), statedb, receipts, usedGas) + err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash(), block.NumberU64()-1), statedb, receipts, usedGas) if err != nil { reportBlock(block, err) return i, err @@ -916,7 +924,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { // coalesce logs for later processing coalescedLogs = append(coalescedLogs, logs...) - if err := WriteBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil { + if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { return i, err } @@ -986,7 +994,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // These logs are later announced as deleted. collectLogs = func(h common.Hash) { // Coalesce logs - receipts := GetBlockReceipts(self.chainDb, h) + receipts := GetBlockReceipts(self.chainDb, h, self.hc.GetBlockNumber(h)) for _, receipt := range receipts { deletedLogs = append(deletedLogs, receipt.Logs...) @@ -998,7 +1006,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // first reduce whoever is higher bound if oldBlock.NumberU64() > newBlock.NumberU64() { // reduce old chain - for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = self.GetBlock(oldBlock.ParentHash()) { + for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = self.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { oldChain = append(oldChain, oldBlock) deletedTxs = append(deletedTxs, oldBlock.Transactions()...) @@ -1006,7 +1014,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } } else { // reduce new chain and append new chain blocks for inserting later on - for ; newBlock != nil && newBlock.NumberU64() != oldBlock.NumberU64(); newBlock = self.GetBlock(newBlock.ParentHash()) { + for ; newBlock != nil && newBlock.NumberU64() != oldBlock.NumberU64(); newBlock = self.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) { newChain = append(newChain, newBlock) } } @@ -1029,7 +1037,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { deletedTxs = append(deletedTxs, oldBlock.Transactions()...) collectLogs(oldBlock.Hash()) - oldBlock, newBlock = self.GetBlock(oldBlock.ParentHash()), self.GetBlock(newBlock.ParentHash()) + oldBlock, newBlock = self.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1), self.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) if oldBlock == nil { return fmt.Errorf("Invalid old chain") } @@ -1052,7 +1060,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := WriteTransactions(self.chainDb, block); err != nil { return err } - receipts := GetBlockReceipts(self.chainDb, block.Hash()) + receipts := GetBlockReceipts(self.chainDb, block.Hash(), block.NumberU64()) // write receipts if err := WriteReceipts(self.chainDb, receipts); err != nil { return err @@ -1187,15 +1195,27 @@ func (self *BlockChain) CurrentHeader() *types.Header { } // GetTd retrieves a block's total difficulty in the canonical chain from the +// database by hash and number, caching it if found. +func (self *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int { + return self.hc.GetTd(hash, number) +} + +// GetTdByHash retrieves a block's total difficulty in the canonical chain from the // database by hash, caching it if found. -func (self *BlockChain) GetTd(hash common.Hash) *big.Int { - return self.hc.GetTd(hash) +func (self *BlockChain) GetTdByHash(hash common.Hash) *big.Int { + return self.hc.GetTdByHash(hash) +} + +// GetHeader retrieves a block header from the database by hash and number, +// caching it if found. +func (self *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + return self.hc.GetHeader(hash, number) } -// GetHeader retrieves a block header from the database by hash, caching it if +// GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. -func (self *BlockChain) GetHeader(hash common.Hash) *types.Header { - return self.hc.GetHeader(hash) +func (self *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { + return self.hc.GetHeaderByHash(hash) } // HasHeader checks if a block header is present in the database or not, caching diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 876dd2ba1..a26fe4a1b 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -102,17 +102,17 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara var tdPre, tdPost *big.Int if full { - tdPre = blockchain.GetTd(blockchain.CurrentBlock().Hash()) + tdPre = blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()) if err := testBlockChainImport(blockChainB, blockchain); err != nil { t.Fatalf("failed to import forked block chain: %v", err) } - tdPost = blockchain.GetTd(blockChainB[len(blockChainB)-1].Hash()) + tdPost = blockchain.GetTdByHash(blockChainB[len(blockChainB)-1].Hash()) } else { - tdPre = blockchain.GetTd(blockchain.CurrentHeader().Hash()) + tdPre = blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()) if err := testHeaderChainImport(headerChainB, blockchain); err != nil { t.Fatalf("failed to import forked header chain: %v", err) } - tdPost = blockchain.GetTd(headerChainB[len(headerChainB)-1].Hash()) + tdPost = blockchain.GetTdByHash(headerChainB[len(headerChainB)-1].Hash()) } // Compare the total difficulties of the chains comparator(tdPre, tdPost) @@ -137,7 +137,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlock(block.ParentHash()).Root(), blockchain.chainDb) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.chainDb) if err != nil { return err } @@ -146,13 +146,13 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { reportBlock(block, err) return err } - err = blockchain.Validator().ValidateState(block, blockchain.GetBlock(block.ParentHash()), statedb, receipts, usedGas) + err = blockchain.Validator().ValidateState(block, blockchain.GetBlockByHash(block.ParentHash()), statedb, receipts, usedGas) if err != nil { reportBlock(block, err) return err } blockchain.mu.Lock() - WriteTd(blockchain.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), blockchain.GetTd(block.ParentHash()))) + WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) WriteBlock(blockchain.chainDb, block) statedb.Commit() blockchain.mu.Unlock() @@ -165,12 +165,12 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error { for _, header := range chain { // Try and validate the header - if err := blockchain.Validator().ValidateHeader(header, blockchain.GetHeader(header.ParentHash), false); err != nil { + if err := blockchain.Validator().ValidateHeader(header, blockchain.GetHeaderByHash(header.ParentHash), false); err != nil { return err } // Manually insert the header into the database, but don't reorganise (allows subsequent testing) blockchain.mu.Lock() - WriteTd(blockchain.chainDb, header.Hash(), new(big.Int).Add(header.Difficulty, blockchain.GetTd(header.ParentHash))) + WriteTd(blockchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) WriteHeader(blockchain.chainDb, header) blockchain.mu.Unlock() } @@ -543,11 +543,11 @@ func testReorg(t *testing.T, first, second []int, td int64, full bool) { // Make sure the chain total difficulty is the correct one want := new(big.Int).Add(genesis.Difficulty(), big.NewInt(td)) if full { - if have := bc.GetTd(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { + if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } else { - if have := bc.GetTd(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { + if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } @@ -758,20 +758,20 @@ func TestFastVsFullChains(t *testing.T) { for i := 0; i < len(blocks); i++ { num, hash := blocks[i].NumberU64(), blocks[i].Hash() - if ftd, atd := fast.GetTd(hash), archive.GetTd(hash); ftd.Cmp(atd) != 0 { + if ftd, atd := fast.GetTdByHash(hash), archive.GetTdByHash(hash); ftd.Cmp(atd) != 0 { t.Errorf("block #%d [%x]: td mismatch: have %v, want %v", num, hash, ftd, atd) } - if fheader, aheader := fast.GetHeader(hash), archive.GetHeader(hash); fheader.Hash() != aheader.Hash() { + if fheader, aheader := fast.GetHeaderByHash(hash), archive.GetHeaderByHash(hash); fheader.Hash() != aheader.Hash() { t.Errorf("block #%d [%x]: header mismatch: have %v, want %v", num, hash, fheader, aheader) } - if fblock, ablock := fast.GetBlock(hash), archive.GetBlock(hash); fblock.Hash() != ablock.Hash() { + if fblock, ablock := fast.GetBlockByHash(hash), archive.GetBlockByHash(hash); fblock.Hash() != ablock.Hash() { t.Errorf("block #%d [%x]: block mismatch: have %v, want %v", num, hash, fblock, ablock) } else if types.DeriveSha(fblock.Transactions()) != types.DeriveSha(ablock.Transactions()) { t.Errorf("block #%d [%x]: transactions mismatch: have %v, want %v", num, hash, fblock.Transactions(), ablock.Transactions()) } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles()) } - if freceipts, areceipts := GetBlockReceipts(fastDb, hash), GetBlockReceipts(archiveDb, hash); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { + if freceipts, areceipts := GetBlockReceipts(fastDb, hash, GetBlockNumber(fastDb, hash)), GetBlockReceipts(archiveDb, hash, GetBlockNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts) } } diff --git a/core/database_util.go b/core/database_util.go index 3ba80062c..1529d7369 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -36,34 +36,72 @@ var ( headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - blockPrefix = []byte("block-") - blockNumPrefix = []byte("block-num-") + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - headerSuffix = []byte("-header") - bodySuffix = []byte("-body") - tdSuffix = []byte("-td") - - txMetaSuffix = []byte{0x01} - receiptsPrefix = []byte("receipts-") - blockReceiptsPrefix = []byte("receipts-block-") + txMetaSuffix = []byte{0x01} + receiptsPrefix = []byte("receipts-") mipmapPre = []byte("mipmap-log-bloom-") MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000} - blockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] - configPrefix = []byte("ethereum-config-") // config prefix for the db + + // used by old (non-sequential keys) db, now only used for conversion + oldBlockPrefix = []byte("block-") + oldHeaderSuffix = []byte("-header") + oldTdSuffix = []byte("-td") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + oldBodySuffix = []byte("-body") + oldBlockNumPrefix = []byte("block-num-") + oldBlockReceiptsPrefix = []byte("receipts-block-") + oldBlockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] ) +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + // GetCanonicalHash retrieves a hash assigned to a canonical block number. func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { - data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) if len(data) == 0 { - return common.Hash{} + data, _ = db.Get(append(oldBlockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + if len(data) == 0 { + return common.Hash{} + } } return common.BytesToHash(data) } +// missingNumber is returned by GetBlockNumber if no header with the +// given block hash has been stored in the database +const missingNumber = uint64(0xffffffffffffffff) + +// GetBlockNumber returns the block number assigned to a block hash +// if the corresponding header is present in the database +func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { + data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) + if len(data) != 8 { + data, _ := db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldHeaderSuffix...)) + if len(data) == 0 { + return missingNumber + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + glog.Fatalf("failed to decode block header: %v", err) + } + return header.Number.Uint64() + } + return binary.BigEndian.Uint64(data) +} + // GetHeadHeaderHash retrieves the hash of the current canonical head block's // header. The difference between this and GetHeadBlockHash is that whereas the // last block hash is only updated upon a full block import, the last header @@ -100,15 +138,18 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. -func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), headerSuffix...)) +func GetHeaderRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if len(data) == 0 { + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldHeaderSuffix...)) + } return data } // GetHeader retrieves the block header corresponding to the hash, nil if none // found. -func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { - data := GetHeaderRLP(db, hash) +func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header { + data := GetHeaderRLP(db, hash, number) if len(data) == 0 { return nil } @@ -121,15 +162,18 @@ func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { } // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...)) +func GetBodyRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if len(data) == 0 { + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldBodySuffix...)) + } return data } // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash) *types.Body { - data := GetBodyRLP(db, hash) +func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { + data := GetBodyRLP(db, hash, number) if len(data) == 0 { return nil } @@ -143,10 +187,13 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body { // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. -func GetTd(db ethdb.Database, hash common.Hash) *big.Int { - data, _ := db.Get(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { + data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), tdSuffix...)) if len(data) == 0 { - return nil + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldTdSuffix...)) + if len(data) == 0 { + return nil + } } td := new(big.Int) if err := rlp.Decode(bytes.NewReader(data), td); err != nil { @@ -158,13 +205,13 @@ func GetTd(db ethdb.Database, hash common.Hash) *big.Int { // GetBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. -func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { +func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { // Retrieve the block header and body contents - header := GetHeader(db, hash) + header := GetHeader(db, hash, number) if header == nil { return nil } - body := GetBody(db, hash) + body := GetBody(db, hash, number) if body == nil { return nil } @@ -174,10 +221,13 @@ func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { - data, _ := db.Get(append(blockReceiptsPrefix, hash[:]...)) +func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types.Receipts { + data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { - return nil + data, _ = db.Get(append(oldBlockReceiptsPrefix, hash.Bytes()...)) + if len(data) == 0 { + return nil + } } storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { @@ -235,10 +285,9 @@ func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { // WriteCanonicalHash stores the canonical hash for the given block number. func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { - key := append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...) + key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { glog.Fatalf("failed to store number to hash mapping into database: %v", err) - return err } return nil } @@ -247,7 +296,6 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last header's hash into database: %v", err) - return err } return nil } @@ -256,7 +304,6 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last block's hash into database: %v", err) - return err } return nil } @@ -265,7 +312,6 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last fast block's hash into database: %v", err) - return err } return nil } @@ -276,40 +322,44 @@ func WriteHeader(db ethdb.Database, header *types.Header) error { if err != nil { return err } - key := append(append(blockPrefix, header.Hash().Bytes()...), headerSuffix...) + hash := header.Hash().Bytes() + num := header.Number.Uint64() + encNum := encodeBlockNumber(num) + key := append(blockHashPrefix, hash...) + if err := db.Put(key, encNum); err != nil { + glog.Fatalf("failed to store hash to number mapping into database: %v", err) + } + key = append(append(headerPrefix, encNum...), hash...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store header into database: %v", err) - return err } - glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, header.Hash().Bytes()[:4]) + glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, hash[:4]) return nil } // WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error { +func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err } - key := append(append(blockPrefix, hash.Bytes()...), bodySuffix...) + key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store block body into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4]) return nil } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { +func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err } - key := append(append(blockPrefix, hash.Bytes()...), tdSuffix...) + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store block total difficulty into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block total difficulty [%x…]: %v", hash.Bytes()[:4], td) return nil @@ -318,7 +368,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { // WriteBlock serializes a block into the database, header and body separately. func WriteBlock(db ethdb.Database, block *types.Block) error { // Store the body first to retain database consistency - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err } // Store the header too, signaling full block ownership @@ -331,7 +381,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -342,9 +392,9 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Rece return err } // Store the flattened receipt slice - if err := db.Put(append(blockReceiptsPrefix, hash.Bytes()...), bytes); err != nil { + key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, bytes); err != nil { glog.Fatalf("failed to store block receipts into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block receipts [%x…]", hash.Bytes()[:4]) return nil @@ -388,7 +438,6 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error { // Write the scheduled data into the database if err := batch.Write(); err != nil { glog.Fatalf("failed to store transactions into database: %v", err) - return err } return nil } @@ -411,42 +460,42 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error { // Write the scheduled data into the database if err := batch.Write(); err != nil { glog.Fatalf("failed to store receipts into database: %v", err) - return err } return nil } // DeleteCanonicalHash removes the number to hash canonical mapping. func DeleteCanonicalHash(db ethdb.Database, number uint64) { - db.Delete(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), headerSuffix...)) +func DeleteHeader(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(blockHashPrefix, hash.Bytes()...)) + db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), bodySuffix...)) +func DeleteBody(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +func DeleteTd(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...)) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db ethdb.Database, hash common.Hash) { - DeleteBlockReceipts(db, hash) - DeleteHeader(db, hash) - DeleteBody(db, hash) - DeleteTd(db, hash) +func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { + DeleteBlockReceipts(db, hash, number) + DeleteHeader(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) } // DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db ethdb.Database, hash common.Hash) { - db.Delete(append(blockReceiptsPrefix, hash.Bytes()...)) +func DeleteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTransaction removes all transaction data associated with a hash. @@ -466,7 +515,7 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) { // access the old combined block representation. It will be dropped after the // network transitions to eth/63. func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block { - data, _ := db.Get(append(blockHashPrefix, hash[:]...)) + data, _ := db.Get(append(oldBlockHashPrefix, hash[:]...)) if len(data) == 0 { return nil } diff --git a/core/database_util_test.go b/core/database_util_test.go index 9ef787624..6c19f78c8 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -89,20 +89,20 @@ func TestHeaderStorage(t *testing.T) { db, _ := ethdb.NewMemDatabase() // Create a test header to move around the database and make sure it's really new - header := &types.Header{Extra: []byte("test header")} - if entry := GetHeader(db, header.Hash()); entry != nil { + header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} + if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } // Write and verify the header in the database if err := WriteHeader(db, header); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if entry := GetHeader(db, header.Hash()); entry == nil { + if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != header.Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, header) } - if entry := GetHeaderRLP(db, header.Hash()); entry == nil { + if entry := GetHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { hasher := sha3.NewKeccak256() @@ -113,8 +113,8 @@ func TestHeaderStorage(t *testing.T) { } } // Delete the header and verify the execution - DeleteHeader(db, header.Hash()) - if entry := GetHeader(db, header.Hash()); entry != nil { + DeleteHeader(db, header.Hash(), header.Number.Uint64()) + if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } } @@ -130,19 +130,19 @@ func TestBodyStorage(t *testing.T) { rlp.Encode(hasher, body) hash := common.BytesToHash(hasher.Sum(nil)) - if entry := GetBody(db, hash); entry != nil { + if entry := GetBody(db, hash, 0); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the body in the database - if err := WriteBody(db, hash, body); err != nil { + if err := WriteBody(db, hash, 0, body); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBody(db, hash); entry == nil { + if entry := GetBody(db, hash, 0); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) } - if entry := GetBodyRLP(db, hash); entry == nil { + if entry := GetBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { hasher := sha3.NewKeccak256() @@ -153,8 +153,8 @@ func TestBodyStorage(t *testing.T) { } } // Delete the body and verify the execution - DeleteBody(db, hash) - if entry := GetBody(db, hash); entry != nil { + DeleteBody(db, hash, 0) + if entry := GetBody(db, hash, 0); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } @@ -170,43 +170,43 @@ func TestBlockStorage(t *testing.T) { TxHash: types.EmptyRootHash, ReceiptHash: types.EmptyRootHash, }) - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - if entry := GetHeader(db, block.Hash()); entry != nil { + if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } - if entry := GetBody(db, block.Hash()); entry != nil { + if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the block in the database if err := WriteBlock(db, block); err != nil { t.Fatalf("Failed to write block into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry == nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) } - if entry := GetHeader(db, block.Hash()); entry == nil { + if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != block.Header().Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if entry := GetBody(db, block.Hash()); entry == nil { + if entry := GetBody(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body()) } // Delete the block and verify the execution - DeleteBlock(db, block.Hash()) - if entry := GetBlock(db, block.Hash()); entry != nil { + DeleteBlock(db, block.Hash(), block.NumberU64()) + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted block returned: %v", entry) } - if entry := GetHeader(db, block.Hash()); entry != nil { + if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } - if entry := GetBody(db, block.Hash()); entry != nil { + if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } @@ -224,28 +224,28 @@ func TestPartialBlockStorage(t *testing.T) { if err := WriteHeader(db, block.Header()); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - DeleteHeader(db, block.Hash()) + DeleteHeader(db, block.Hash(), block.NumberU64()) // Store a body and check that it's not recognized as a block - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - DeleteBody(db, block.Hash()) + DeleteBody(db, block.Hash(), block.NumberU64()) // Store a header and a body separately and check reassembly if err := WriteHeader(db, block.Header()); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry == nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) @@ -258,21 +258,21 @@ func TestTdStorage(t *testing.T) { // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) - if entry := GetTd(db, hash); entry != nil { + if entry := GetTd(db, hash, 0); entry != nil { t.Fatalf("Non existent TD returned: %v", entry) } // Write and verify the TD in the database - if err := WriteTd(db, hash, td); err != nil { + if err := WriteTd(db, hash, 0, td); err != nil { t.Fatalf("Failed to write TD into database: %v", err) } - if entry := GetTd(db, hash); entry == nil { + if entry := GetTd(db, hash, 0); entry == nil { t.Fatalf("Stored TD not found") } else if entry.Cmp(td) != 0 { t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td) } // Delete the TD and verify the execution - DeleteTd(db, hash) - if entry := GetTd(db, hash); entry != nil { + DeleteTd(db, hash, 0) + if entry := GetTd(db, hash, 0); entry != nil { t.Fatalf("Deleted TD returned: %v", entry) } } @@ -473,14 +473,14 @@ func TestBlockReceiptStorage(t *testing.T) { // Check that no receipt entries are in a pristine database hash := common.BytesToHash([]byte{0x03, 0x14}) - if rs := GetBlockReceipts(db, hash); len(rs) != 0 { + if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } // Insert the receipt slice into the database and check presence - if err := WriteBlockReceipts(db, hash, receipts); err != nil { + if err := WriteBlockReceipts(db, hash, 0, receipts); err != nil { t.Fatalf("failed to write block receipts: %v", err) } - if rs := GetBlockReceipts(db, hash); len(rs) == 0 { + if rs := GetBlockReceipts(db, hash, 0); len(rs) == 0 { t.Fatalf("no receipts returned") } else { for i := 0; i < len(receipts); i++ { @@ -493,8 +493,8 @@ func TestBlockReceiptStorage(t *testing.T) { } } // Delete the receipt slice and check purge - DeleteBlockReceipts(db, hash) - if rs := GetBlockReceipts(db, hash); len(rs) != 0 { + DeleteBlockReceipts(db, hash, 0) + if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } @@ -597,7 +597,7 @@ func TestMipmapChain(t *testing.T) { if err := WriteHeadBlockHash(db, block.Hash()); err != nil { t.Fatalf("failed to insert block number: %v", err) } - if err := WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { t.Fatal("error writing block receipts:", err) } } diff --git a/core/genesis.go b/core/genesis.go index 40d799621..637b63320 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -88,7 +88,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, Root: root, }, nil, nil, nil) - if block := GetBlock(chainDb, block.Hash()); block != nil { + if block := GetBlock(chainDb, block.Hash(), block.NumberU64()); block != nil { glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number") err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) if err != nil { @@ -100,13 +100,13 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, if err := stateBatch.Write(); err != nil { return nil, fmt.Errorf("cannot write state: %v", err) } - if err := WriteTd(chainDb, block.Hash(), difficulty); err != nil { + if err := WriteTd(chainDb, block.Hash(), block.NumberU64(), difficulty); err != nil { return nil, err } if err := WriteBlock(chainDb, block); err != nil { return nil, err } - if err := WriteBlockReceipts(chainDb, block.Hash(), nil); err != nil { + if err := WriteBlockReceipts(chainDb, block.Hash(), block.NumberU64(), nil); err != nil { return nil, err } if err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()); err != nil { diff --git a/core/headerchain.go b/core/headerchain.go index 5e0fbfb08..f856333a0 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -35,6 +35,12 @@ import ( "github.com/hashicorp/golang-lru" ) +const ( + headerCacheLimit = 512 + tdCacheLimit = 1024 + numberCacheLimit = 2048 +) + // HeaderChain implements the basic block header chain logic that is shared by // core.BlockChain and light.LightChain. It is not usable in itself, only as // a part of either structure. @@ -51,6 +57,7 @@ type HeaderChain struct { headerCache *lru.Cache // Cache for the most recent block headers tdCache *lru.Cache // Cache for the most recent block total difficulties + numberCache *lru.Cache // Cache for the most recent block numbers procInterrupt func() bool @@ -68,6 +75,7 @@ type getHeaderValidatorFn func() HeaderValidator func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator getHeaderValidatorFn, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) + numberCache, _ := lru.New(numberCacheLimit) // Seed a fast but crypto originating random generator seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) @@ -80,6 +88,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator ge chainDb: chainDb, headerCache: headerCache, tdCache: tdCache, + numberCache: numberCache, procInterrupt: procInterrupt, rand: mrand.New(mrand.NewSource(seed.Int64())), getValidator: getValidator, @@ -97,7 +106,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator ge hc.currentHeader = hc.genesisHeader if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) { - if chead := hc.GetHeader(head); chead != nil { + if chead := hc.GetHeaderByHash(head); chead != nil { hc.currentHeader = chead } } @@ -106,6 +115,19 @@ func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator ge return hc, nil } +// GetBlockNumber retrieves the block number belonging to the given hash +// from the cache or database +func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 { + if cached, ok := hc.numberCache.Get(hash); ok { + return cached.(uint64) + } + number := GetBlockNumber(hc.chainDb, hash) + if number != missingNumber { + hc.numberCache.Add(hash, number) + } + return number +} + // WriteHeader writes a header into the local chain, given that its parent is // already known. If the total difficulty of the newly inserted header becomes // greater than the current known TD, the canonical chain is re-routed. @@ -122,11 +144,11 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er number = header.Number.Uint64() ) // Calculate the total difficulty of the header - ptd := hc.GetTd(header.ParentHash) + ptd := hc.GetTd(header.ParentHash, number-1) if ptd == nil { return NonStatTy, ParentError(header.ParentHash) } - localTd := hc.GetTd(hc.currentHeaderHash) + localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64()) externTd := new(big.Int).Add(header.Difficulty, ptd) // If the total difficulty is higher than our known, add it to the canonical chain @@ -134,21 +156,25 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { // Delete any canonical number assignments above the new head - for i := number + 1; GetCanonicalHash(hc.chainDb, i) != (common.Hash{}); i++ { + for i := number + 1; ; i++ { + hash := GetCanonicalHash(hc.chainDb, i) + if hash == (common.Hash{}) { + break + } DeleteCanonicalHash(hc.chainDb, i) } // Overwrite any stale canonical number assignments var ( headHash = header.ParentHash - headHeader = hc.GetHeader(headHash) - headNumber = headHeader.Number.Uint64() + headNumber = header.Number.Uint64() - 1 + headHeader = hc.GetHeader(headHash, headNumber) ) for GetCanonicalHash(hc.chainDb, headNumber) != headHash { WriteCanonicalHash(hc.chainDb, headHash, headNumber) headHash = headHeader.ParentHash - headHeader = hc.GetHeader(headHash) - headNumber = headHeader.Number.Uint64() + headNumber = headHeader.Number.Uint64() - 1 + headHeader = hc.GetHeader(headHash, headNumber) } // Extend the canonical chain with the new header if err := WriteCanonicalHash(hc.chainDb, hash, number); err != nil { @@ -164,13 +190,14 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er status = SideStatTy } // Irrelevant of the canonical status, write the header itself to the database - if err := hc.WriteTd(hash, externTd); err != nil { + if err := hc.WriteTd(hash, number, externTd); err != nil { glog.Fatalf("failed to write header total difficulty: %v", err) } if err := WriteHeader(hc.chainDb, header); err != nil { glog.Fatalf("failed to write header contents: %v", err) } hc.headerCache.Add(hash, header) + hc.numberCache.Add(hash, number) return } @@ -239,7 +266,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, w var err error if index == 0 { - err = hc.getValidator().ValidateHeader(header, hc.GetHeader(header.ParentHash), checkPow) + err = hc.getValidator().ValidateHeader(header, hc.GetHeader(header.ParentHash, header.Number.Uint64()-1), checkPow) } else { err = hc.getValidator().ValidateHeader(header, chain[index-1], checkPow) } @@ -300,7 +327,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, w // hash, fetching towards the genesis block. func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash { // Get the origin header from which to fetch - header := hc.GetHeader(hash) + header := hc.GetHeaderByHash(hash) if header == nil { return nil } @@ -308,7 +335,7 @@ func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []co chain := make([]common.Hash, 0, max) for i := uint64(0); i < max; i++ { next := header.ParentHash - if header = hc.GetHeader(next); header == nil { + if header = hc.GetHeader(next, header.Number.Uint64()-1); header == nil { break } chain = append(chain, next) @@ -320,13 +347,13 @@ func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []co } // GetTd retrieves a block's total difficulty in the canonical chain from the -// database by hash, caching it if found. -func (hc *HeaderChain) GetTd(hash common.Hash) *big.Int { +// database by hash and number, caching it if found. +func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { // Short circuit if the td's already in the cache, retrieve otherwise if cached, ok := hc.tdCache.Get(hash); ok { return cached.(*big.Int) } - td := GetTd(hc.chainDb, hash) + td := GetTd(hc.chainDb, hash, number) if td == nil { return nil } @@ -335,24 +362,30 @@ func (hc *HeaderChain) GetTd(hash common.Hash) *big.Int { return td } +// GetTdByHash retrieves a block's total difficulty in the canonical chain from the +// database by hash, caching it if found. +func (hc *HeaderChain) GetTdByHash(hash common.Hash) *big.Int { + return hc.GetTd(hash, hc.GetBlockNumber(hash)) +} + // WriteTd stores a block's total difficulty into the database, also caching it // along the way. -func (hc *HeaderChain) WriteTd(hash common.Hash, td *big.Int) error { - if err := WriteTd(hc.chainDb, hash, td); err != nil { +func (hc *HeaderChain) WriteTd(hash common.Hash, number uint64, td *big.Int) error { + if err := WriteTd(hc.chainDb, hash, number, td); err != nil { return err } hc.tdCache.Add(hash, new(big.Int).Set(td)) return nil } -// GetHeader retrieves a block header from the database by hash, caching it if -// found. -func (hc *HeaderChain) GetHeader(hash common.Hash) *types.Header { +// GetHeader retrieves a block header from the database by hash and number, +// caching it if found. +func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header { // Short circuit if the header's already in the cache, retrieve otherwise if header, ok := hc.headerCache.Get(hash); ok { return header.(*types.Header) } - header := GetHeader(hc.chainDb, hash) + header := GetHeader(hc.chainDb, hash, number) if header == nil { return nil } @@ -361,10 +394,16 @@ func (hc *HeaderChain) GetHeader(hash common.Hash) *types.Header { return header } +// GetHeaderByHash retrieves a block header from the database by hash, caching it if +// found. +func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { + return hc.GetHeader(hash, hc.GetBlockNumber(hash)) +} + // HasHeader checks if a block header is present in the database or not, caching // it if present. func (hc *HeaderChain) HasHeader(hash common.Hash) bool { - return hc.GetHeader(hash) != nil + return hc.GetHeaderByHash(hash) != nil } // GetHeaderByNumber retrieves a block header from the database by number, @@ -374,7 +413,7 @@ func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header { if hash == (common.Hash{}) { return nil } - return hc.GetHeader(hash) + return hc.GetHeader(hash, number) } // CurrentHeader retrieves the current head header of the canonical chain. The @@ -394,7 +433,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { // DeleteCallback is a callback function that is called by SetHead before // each header is deleted. -type DeleteCallback func(common.Hash) +type DeleteCallback func(common.Hash, uint64) // SetHead rewinds the local chain to a new head. Everything above the new head // will be deleted and the new one set. @@ -406,12 +445,13 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { for hc.currentHeader != nil && hc.currentHeader.Number.Uint64() > head { hash := hc.currentHeader.Hash() + num := hc.currentHeader.Number.Uint64() if delFn != nil { - delFn(hash) + delFn(hash, num) } - DeleteHeader(hc.chainDb, hash) - DeleteTd(hc.chainDb, hash) - hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash) + DeleteHeader(hc.chainDb, hash, num) + DeleteTd(hc.chainDb, hash, num) + hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash, hc.currentHeader.Number.Uint64()-1) } // Roll back the canonical chain numbering for i := height; i > head; i-- { @@ -420,6 +460,7 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { // Clear out any stale content from the caches hc.headerCache.Purge() hc.tdCache.Purge() + hc.numberCache.Purge() if hc.currentHeader == nil { hc.currentHeader = hc.genesisHeader diff --git a/core/vm_env.go b/core/vm_env.go index f50140c68..599672382 100644 --- a/core/vm_env.go +++ b/core/vm_env.go @@ -30,7 +30,7 @@ import ( // to query for information. func GetHashFn(ref common.Hash, chain *BlockChain) func(n uint64) common.Hash { return func(n uint64) common.Hash { - for block := chain.GetBlock(ref); block != nil; block = chain.GetBlock(block.ParentHash()) { + for block := chain.GetBlockByHash(ref); block != nil; block = chain.GetBlock(block.ParentHash(), block.NumberU64()-1) { if block.NumberU64() == n { return block.Hash() } diff --git a/eth/api.go b/eth/api.go index a2be81428..9f5f2e677 100644 --- a/eth/api.go +++ b/eth/api.go @@ -594,7 +594,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(blockNr rpc.BlockNumber, fullTx b // GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full // detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByHash(blockHash common.Hash, fullTx bool) (map[string]interface{}, error) { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return s.rpcOutputBlock(block, true, fullTx) } return nil, nil @@ -618,7 +618,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(blockNr rpc.BlockNum // GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. When fullTx is true // all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(blockHash common.Hash, index rpc.HexNumber) (map[string]interface{}, error) { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { uncles := block.Uncles() if index.Int() < 0 || index.Int() >= len(uncles) { glog.V(logger.Debug).Infof("uncle block on index %d not found for block %s", index.Int(), blockHash.Hex()) @@ -640,7 +640,7 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(blockNr rpc.BlockNumber // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(blockHash common.Hash) *rpc.HexNumber { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return rpc.NewHexNumber(len(block.Uncles())) } return nil @@ -814,7 +814,7 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "stateRoot": b.Root(), "miner": b.Coinbase(), "difficulty": rpc.NewHexNumber(b.Difficulty()), - "totalDifficulty": rpc.NewHexNumber(s.bc.GetTd(b.Hash())), + "totalDifficulty": rpc.NewHexNumber(s.bc.GetTd(b.Hash(), b.NumberU64())), "extraData": fmt.Sprintf("0x%x", b.Extra()), "size": rpc.NewHexNumber(b.Size().Int64()), "gasLimit": rpc.NewHexNumber(b.GasLimit()), @@ -1004,7 +1004,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(blockNr rpc. // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(blockHash common.Hash) *rpc.HexNumber { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return rpc.NewHexNumber(len(block.Transactions())) } return nil @@ -1020,7 +1020,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(blockNr r // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(blockHash common.Hash, index rpc.HexNumber) (*RPCTransaction, error) { - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return newRPCTransactionFromBlockIndex(block, index.Int()) } return nil, nil @@ -1080,7 +1080,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByHash(txHash common.Hash) (*RP return nil, nil } - if block := s.bc.GetBlock(blockHash); block != nil { + if block := s.bc.GetBlockByHash(blockHash); block != nil { return newRPCTransaction(block, txHash) } @@ -1705,7 +1705,7 @@ func (api *PrivateDebugAPI) TraceBlockByNumber(number uint64, config *vm.Config) // TraceBlockByHash processes the block by hash. func (api *PrivateDebugAPI) TraceBlockByHash(hash common.Hash, config *vm.Config) BlockTraceResult { // Fetch the block that we aim to reprocess - block := api.eth.BlockChain().GetBlock(hash) + block := api.eth.BlockChain().GetBlockByHash(hash) if block == nil { return BlockTraceResult{Error: fmt.Sprintf("block #%x not found", hash)} } @@ -1745,10 +1745,10 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, config *vm.Config) (b config.Debug = true // make sure debug is set. config.Logger.Collector = collector - if err := core.ValidateHeader(api.config, blockchain.AuxValidator(), block.Header(), blockchain.GetHeader(block.ParentHash()), true, false); err != nil { + if err := core.ValidateHeader(api.config, blockchain.AuxValidator(), block.Header(), blockchain.GetHeader(block.ParentHash(), block.NumberU64()-1), true, false); err != nil { return false, collector.traces, err } - statedb, err := state.New(blockchain.GetBlock(block.ParentHash()).Root(), api.eth.ChainDb()) + statedb, err := state.New(blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), api.eth.ChainDb()) if err != nil { return false, collector.traces, err } @@ -1757,7 +1757,7 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, config *vm.Config) (b if err != nil { return false, collector.traces, err } - if err := validator.ValidateState(block, blockchain.GetBlock(block.ParentHash()), statedb, receipts, usedGas); err != nil { + if err := validator.ValidateState(block, blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1), statedb, receipts, usedGas); err != nil { return false, collector.traces, err } return true, collector.traces, nil @@ -1841,12 +1841,12 @@ func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogC if tx == nil { return nil, fmt.Errorf("transaction %x not found", txHash) } - block := api.eth.BlockChain().GetBlock(blockHash) + block := api.eth.BlockChain().GetBlockByHash(blockHash) if block == nil { return nil, fmt.Errorf("block %x not found", blockHash) } // Create the state database to mutate and eventually trace - parent := api.eth.BlockChain().GetBlock(block.ParentHash()) + parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { return nil, fmt.Errorf("block parent %x not found", block.ParentHash()) } diff --git a/eth/backend.go b/eth/backend.go index bb487650b..006523484 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -18,7 +18,6 @@ package eth import ( - "bytes" "errors" "fmt" "math/big" @@ -47,7 +46,6 @@ import ( "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -104,9 +102,9 @@ type Config struct { } type Ethereum struct { - chainConfig *core.ChainConfig - // Channel for shutting down the ethereum - shutdownChan chan bool + chainConfig *core.ChainConfig + shutdownChan chan bool // Channel for shutting down the ethereum + stopDbUpgrade func() // stop chain db sequential key upgrade // DB interfaces chainDb ethdb.Database // Block chain database @@ -161,6 +159,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err := addMipmapBloomBins(chainDb); err != nil { return nil, err } + stopDbUpgrade := upgradeSequentialKeys(chainDb) dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache, config.DatabaseHandles) if err != nil { @@ -185,7 +184,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { chainDb = config.TestGenesisState } if config.TestGenesisBlock != nil { - core.WriteTd(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.Difficulty()) + core.WriteTd(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64(), config.TestGenesisBlock.Difficulty()) core.WriteBlock(chainDb, config.TestGenesisBlock) core.WriteCanonicalHash(chainDb, config.TestGenesisBlock.Hash(), config.TestGenesisBlock.NumberU64()) core.WriteHeadBlockHash(chainDb, config.TestGenesisBlock.Hash()) @@ -202,6 +201,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth := &Ethereum{ shutdownChan: make(chan bool), + stopDbUpgrade: stopDbUpgrade, chainDb: chainDb, dappDb: dappDb, eventMux: ctx.EventMux, @@ -238,7 +238,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { // load the genesis block or write a new one if no genesis // block is prenent in the database. - genesis := core.GetBlock(chainDb, core.GetCanonicalHash(chainDb, 0)) + genesis := core.GetBlock(chainDb, core.GetCanonicalHash(chainDb, 0), 0) if genesis == nil { genesis, err = core.WriteDefaultGenesisBlock(chainDb) if err != nil { @@ -415,6 +415,9 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { // Stop implements node.Service, terminating all internal goroutines used by the // Ethereum protocol. func (s *Ethereum) Stop() error { + if s.stopDbUpgrade != nil { + s.stopDbUpgrade() + } s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() @@ -526,104 +529,3 @@ func dagFiles(epoch uint64) (string, string) { dag := fmt.Sprintf("full-R%d-%x", ethashRevision, seedHash[:8]) return dag, "full-R" + dag } - -// upgradeChainDatabase ensures that the chain database stores block split into -// separate header and body entries. -func upgradeChainDatabase(db ethdb.Database) error { - // Short circuit if the head block is stored already as separate header and body - data, err := db.Get([]byte("LastBlock")) - if err != nil { - return nil - } - head := common.BytesToHash(data) - - if block := core.GetBlockByHashOld(db, head); block == nil { - return nil - } - // At least some of the database is still the old format, upgrade (skip the head block!) - glog.V(logger.Info).Info("Old database detected, upgrading...") - - if db, ok := db.(*ethdb.LDBDatabase); ok { - blockPrefix := []byte("block-hash-") - for it := db.NewIterator(); it.Next(); { - // Skip anything other than a combined block - if !bytes.HasPrefix(it.Key(), blockPrefix) { - continue - } - // Skip the head block (merge last to signal upgrade completion) - if bytes.HasSuffix(it.Key(), head.Bytes()) { - continue - } - // Load the block, split and serialize (order!) - block := core.GetBlockByHashOld(db, common.BytesToHash(bytes.TrimPrefix(it.Key(), blockPrefix))) - - if err := core.WriteTd(db, block.Hash(), block.DeprecatedTd()); err != nil { - return err - } - if err := core.WriteBody(db, block.Hash(), block.Body()); err != nil { - return err - } - if err := core.WriteHeader(db, block.Header()); err != nil { - return err - } - if err := db.Delete(it.Key()); err != nil { - return err - } - } - // Lastly, upgrade the head block, disabling the upgrade mechanism - current := core.GetBlockByHashOld(db, head) - - if err := core.WriteTd(db, current.Hash(), current.DeprecatedTd()); err != nil { - return err - } - if err := core.WriteBody(db, current.Hash(), current.Body()); err != nil { - return err - } - if err := core.WriteHeader(db, current.Header()); err != nil { - return err - } - } - return nil -} - -func addMipmapBloomBins(db ethdb.Database) (err error) { - const mipmapVersion uint = 2 - - // check if the version is set. We ignore data for now since there's - // only one version so we can easily ignore it for now - var data []byte - data, _ = db.Get([]byte("setting-mipmap-version")) - if len(data) > 0 { - var version uint - if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion { - return nil - } - } - - defer func() { - if err == nil { - var val []byte - val, err = rlp.EncodeToBytes(mipmapVersion) - if err == nil { - err = db.Put([]byte("setting-mipmap-version"), val) - } - return - } - }() - latestBlock := core.GetBlock(db, core.GetHeadBlockHash(db)) - if latestBlock == nil { // clean database - return - } - - tstart := time.Now() - glog.V(logger.Info).Infoln("upgrading db log bloom bins") - for i := uint64(0); i <= latestBlock.NumberU64(); i++ { - hash := core.GetCanonicalHash(db, i) - if (hash == common.Hash{}) { - return fmt.Errorf("chain db corrupted. Could not find block %d.", i) - } - core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash)) - } - glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) - return nil -} diff --git a/eth/backend_test.go b/eth/backend_test.go index c2bed879b..cb94adbf0 100644 --- a/eth/backend_test.go +++ b/eth/backend_test.go @@ -61,7 +61,7 @@ func TestMipmapUpgrade(t *testing.T) { if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { t.Fatalf("failed to insert block number: %v", err) } - if err := core.WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { t.Fatal("error writing block receipts:", err) } } diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go new file mode 100644 index 000000000..12de60fe7 --- /dev/null +++ b/eth/db_upgrade.go @@ -0,0 +1,343 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// Package eth implements the Ethereum protocol. +package eth + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/rlp" +) + +var useSequentialKeys = []byte("dbUpgrade_20160530sequentialKeys") + +// upgradeSequentialKeys checks the chain database version and +// starts a background process to make upgrades if necessary. +// Returns a stop function that blocks until the process has +// been safely stopped. +func upgradeSequentialKeys(db ethdb.Database) (stopFn func()) { + data, _ := db.Get(useSequentialKeys) + if len(data) > 0 && data[0] == 42 { + return nil // already converted + } + + if data, _ := db.Get([]byte("LastHeader")); len(data) == 0 { + db.Put(useSequentialKeys, []byte{42}) + return nil // empty database, nothing to do + } + + glog.V(logger.Info).Infof("Upgrading chain database to use sequential keys") + + stopChn := make(chan struct{}) + stoppedChn := make(chan struct{}) + + go func() { + stopFn := func() bool { + select { + case <-time.After(time.Microsecond * 100): // make sure other processes don't get starved + case <-stopChn: + return true + } + return false + } + + err, stopped := upgradeSequentialCanonicalNumbers(db, stopFn) + if err == nil && !stopped { + err, stopped = upgradeSequentialBlocks(db, stopFn) + } + if err == nil && !stopped { + err, stopped = upgradeSequentialOrphanedReceipts(db, stopFn) + } + if err == nil && !stopped { + glog.V(logger.Info).Infof("Database conversion successful") + db.Put(useSequentialKeys, []byte{42}) + } + if err != nil { + glog.V(logger.Error).Infof("Database conversion failed: %v", err) + } + close(stoppedChn) + }() + + return func() { + close(stopChn) + <-stoppedChn + } +} + +// upgradeSequentialCanonicalNumbers reads all old format canonical numbers from +// the database, writes them in new format and deletes the old ones if successful. +func upgradeSequentialCanonicalNumbers(db ethdb.Database, stopFn func() bool) (error, bool) { + prefix := []byte("block-num-") + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(prefix) + cnt := 0 + for bytes.HasPrefix(it.Key(), prefix) { + keyPtr := it.Key() + if len(keyPtr) < 20 { + cnt++ + if cnt%100000 == 0 { + glog.V(logger.Info).Infof("converting %d canonical numbers...", cnt) + } + number := big.NewInt(0).SetBytes(keyPtr[10:]).Uint64() + newKey := []byte("h12345678n") + binary.BigEndian.PutUint64(newKey[1:9], number) + if err := db.Put(newKey, it.Value()); err != nil { + return err, false + } + if err := db.Delete(keyPtr); err != nil { + return err, false + } + } + + if stopFn() { + return nil, true + } + it.Next() + } + if cnt > 0 { + glog.V(logger.Info).Infof("converted %d canonical numbers...", cnt) + } + return nil, false +} + +// upgradeSequentialBlocks reads all old format block headers, bodies, TDs and block +// receipts from the database, writes them in new format and deletes the old ones +// if successful. +func upgradeSequentialBlocks(db ethdb.Database, stopFn func() bool) (error, bool) { + prefix := []byte("block-") + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(prefix) + cnt := 0 + for bytes.HasPrefix(it.Key(), prefix) { + keyPtr := it.Key() + if len(keyPtr) >= 38 { + cnt++ + if cnt%10000 == 0 { + glog.V(logger.Info).Infof("converting %d blocks...", cnt) + } + // convert header, body, td and block receipts + var keyPrefix [38]byte + copy(keyPrefix[:], keyPtr[0:38]) + hash := keyPrefix[6:38] + if err := upgradeSequentialBlockData(db, hash); err != nil { + return err, false + } + // delete old db entries belonging to this hash + for bytes.HasPrefix(it.Key(), keyPrefix[:]) { + if err := db.Delete(it.Key()); err != nil { + return err, false + } + it.Next() + } + if err := db.Delete(append([]byte("receipts-block-"), hash...)); err != nil { + return err, false + } + } else { + it.Next() + } + + if stopFn() { + return nil, true + } + } + if cnt > 0 { + glog.V(logger.Info).Infof("converted %d blocks...", cnt) + } + return nil, false +} + +// upgradeSequentialOrphanedReceipts removes any old format block receipts from the +// database that did not have a corresponding block +func upgradeSequentialOrphanedReceipts(db ethdb.Database, stopFn func() bool) (error, bool) { + prefix := []byte("receipts-block-") + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(prefix) + cnt := 0 + for bytes.HasPrefix(it.Key(), prefix) { + // phase 2 already converted receipts belonging to existing + // blocks, just remove if there's anything left + cnt++ + if err := db.Delete(it.Key()); err != nil { + return err, false + } + + if stopFn() { + return nil, true + } + it.Next() + } + if cnt > 0 { + glog.V(logger.Info).Infof("removed %d orphaned block receipts...", cnt) + } + return nil, false +} + +// upgradeSequentialBlockData upgrades the header, body, td and block receipts +// database entries belonging to a single hash (doesn't delete old data). +func upgradeSequentialBlockData(db ethdb.Database, hash []byte) error { + // get old chain data and block number + headerRLP, _ := db.Get(append(append([]byte("block-"), hash...), []byte("-header")...)) + if len(headerRLP) == 0 { + return nil + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(headerRLP), header); err != nil { + return err + } + number := header.Number.Uint64() + bodyRLP, _ := db.Get(append(append([]byte("block-"), hash...), []byte("-body")...)) + tdRLP, _ := db.Get(append(append([]byte("block-"), hash...), []byte("-td")...)) + receiptsRLP, _ := db.Get(append([]byte("receipts-block-"), hash...)) + // store new hash -> number association + encNum := make([]byte, 8) + binary.BigEndian.PutUint64(encNum, number) + if err := db.Put(append([]byte("H"), hash...), encNum); err != nil { + return err + } + // store new chain data + if err := db.Put(append(append([]byte("h"), encNum...), hash...), headerRLP); err != nil { + return err + } + if len(tdRLP) != 0 { + if err := db.Put(append(append(append([]byte("h"), encNum...), hash...), []byte("t")...), tdRLP); err != nil { + return err + } + } + if len(bodyRLP) != 0 { + if err := db.Put(append(append([]byte("b"), encNum...), hash...), bodyRLP); err != nil { + return err + } + } + if len(receiptsRLP) != 0 { + if err := db.Put(append(append([]byte("r"), encNum...), hash...), receiptsRLP); err != nil { + return err + } + } + return nil +} + +// upgradeChainDatabase ensures that the chain database stores block split into +// separate header and body entries. +func upgradeChainDatabase(db ethdb.Database) error { + // Short circuit if the head block is stored already as separate header and body + data, err := db.Get([]byte("LastBlock")) + if err != nil { + return nil + } + head := common.BytesToHash(data) + + if block := core.GetBlockByHashOld(db, head); block == nil { + return nil + } + // At least some of the database is still the old format, upgrade (skip the head block!) + glog.V(logger.Info).Info("Old database detected, upgrading...") + + if db, ok := db.(*ethdb.LDBDatabase); ok { + blockPrefix := []byte("block-hash-") + for it := db.NewIterator(); it.Next(); { + // Skip anything other than a combined block + if !bytes.HasPrefix(it.Key(), blockPrefix) { + continue + } + // Skip the head block (merge last to signal upgrade completion) + if bytes.HasSuffix(it.Key(), head.Bytes()) { + continue + } + // Load the block, split and serialize (order!) + block := core.GetBlockByHashOld(db, common.BytesToHash(bytes.TrimPrefix(it.Key(), blockPrefix))) + + if err := core.WriteTd(db, block.Hash(), block.NumberU64(), block.DeprecatedTd()); err != nil { + return err + } + if err := core.WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { + return err + } + if err := core.WriteHeader(db, block.Header()); err != nil { + return err + } + if err := db.Delete(it.Key()); err != nil { + return err + } + } + // Lastly, upgrade the head block, disabling the upgrade mechanism + current := core.GetBlockByHashOld(db, head) + + if err := core.WriteTd(db, current.Hash(), current.NumberU64(), current.DeprecatedTd()); err != nil { + return err + } + if err := core.WriteBody(db, current.Hash(), current.NumberU64(), current.Body()); err != nil { + return err + } + if err := core.WriteHeader(db, current.Header()); err != nil { + return err + } + } + return nil +} + +func addMipmapBloomBins(db ethdb.Database) (err error) { + const mipmapVersion uint = 2 + + // check if the version is set. We ignore data for now since there's + // only one version so we can easily ignore it for now + var data []byte + data, _ = db.Get([]byte("setting-mipmap-version")) + if len(data) > 0 { + var version uint + if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion { + return nil + } + } + + defer func() { + if err == nil { + var val []byte + val, err = rlp.EncodeToBytes(mipmapVersion) + if err == nil { + err = db.Put([]byte("setting-mipmap-version"), val) + } + return + } + }() + latestHash := core.GetHeadBlockHash(db) + latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash)) + if latestBlock == nil { // clean database + return + } + + tstart := time.Now() + glog.V(logger.Info).Infoln("upgrading db log bloom bins") + for i := uint64(0); i <= latestBlock.NumberU64(); i++ { + hash := core.GetCanonicalHash(db, i) + if (hash == common.Hash{}) { + return fmt.Errorf("chain db corrupted. Could not find block %d.", i) + } + core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i)) + } + glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) + return nil +} diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a9c069a92..e9e051ded 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -1824,13 +1824,15 @@ func testFastCriticalRestarts(t *testing.T, protocol int) { for i := 0; i < fsPivotInterval; i++ { tester.peerMissingStates["peer"][headers[hashes[fsMinFullBlocks+i]].Root] = true } + tester.downloader.dropPeer = func(id string) {} // We reuse the same "faulty" peer throughout the test + // Synchronise with the peer a few times and make sure they fail until the retry limit for i := 0; i < fsCriticalTrials; i++ { // Attempt a sync and ensure it fails properly if err := tester.sync("peer", nil, FastSync); err == nil { t.Fatalf("failing fast sync succeeded: %v", err) } - time.Sleep(500 * time.Millisecond) // Make sure no in-flight requests remain + time.Sleep(100 * time.Millisecond) // Make sure no in-flight requests remain // If it's the first failure, pivot should be locked => reenable all others to detect pivot changes if i == 0 { diff --git a/eth/filters/api.go b/eth/filters/api.go index 7278e20b9..393019f8b 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -361,7 +361,7 @@ func (args *NewFilterArgs) UnmarshalJSON(data []byte) error { if len(raw) >= 2 && raw[0] == '0' && (raw[1] == 'x' || raw[1] == 'X') { raw = raw[2:] } - if len(raw) != 2 * common.HashLength { + if len(raw) != 2*common.HashLength { return common.Hash{}, errors.New("invalid topic(s)") } if decAddr, err := hex.DecodeString(raw); err == nil { diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 469dfba4d..995b588fb 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -72,7 +72,8 @@ func (self *Filter) SetTopics(topics [][]common.Hash) { // Run filters logs with the current parameters set func (self *Filter) Find() vm.Logs { - latestBlock := core.GetBlock(self.db, core.GetHeadBlockHash(self.db)) + latestHash := core.GetHeadBlockHash(self.db) + latestBlock := core.GetBlock(self.db, latestHash, core.GetBlockNumber(self.db, latestHash)) var beginBlockNo uint64 = uint64(self.begin) if self.begin == -1 { beginBlockNo = latestBlock.NumberU64() @@ -127,7 +128,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { for i := start; i <= end; i++ { hash := core.GetCanonicalHash(self.db, i) if hash != (common.Hash{}) { - block = core.GetBlock(self.db, hash) + block = core.GetBlock(self.db, hash, i) } else { // block not found return logs } @@ -137,7 +138,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { if self.bloomFilter(block) { // Get the logs of the block var ( - receipts = core.GetBlockReceipts(self.db, block.Hash()) + receipts = core.GetBlockReceipts(self.db, block.Hash(), i) unfiltered vm.Logs ) for _, receipt := range receipts { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index b0f88ffeb..a95adfce7 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -94,7 +94,7 @@ func BenchmarkMipmaps(b *testing.B) { if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { b.Fatalf("failed to insert block number: %v", err) } - if err := core.WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { b.Fatal("error writing block receipts:", err) } } @@ -196,7 +196,7 @@ func TestFilters(t *testing.T) { if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { t.Fatalf("failed to insert block number: %v", err) } - if err := core.WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { t.Fatal("error writing block receipts:", err) } } diff --git a/eth/gasprice.go b/eth/gasprice.go index e0de89e62..ef203f8fe 100644 --- a/eth/gasprice.go +++ b/eth/gasprice.go @@ -166,7 +166,7 @@ func (self *GasPriceOracle) processBlock(block *types.Block) { func (self *GasPriceOracle) lowestPrice(block *types.Block) *big.Int { gasUsed := big.NewInt(0) - receipts := core.GetBlockReceipts(self.eth.ChainDb(), block.Hash()) + receipts := core.GetBlockReceipts(self.eth.ChainDb(), block.Hash(), block.NumberU64()) if len(receipts) > 0 { if cgu := receipts[len(receipts)-1].CumulativeGasUsed; cgu != nil { gasUsed = receipts[len(receipts)-1].CumulativeGasUsed diff --git a/eth/handler.go b/eth/handler.go index 1e4dc1289..47a36cc0b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -152,9 +152,9 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, return nil, errIncompatibleConfig } // Construct the different synchronisation mechanisms - manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeader, - blockchain.GetBlock, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, - blockchain.GetTd, blockchain.InsertHeaderChain, manager.insertChain, blockchain.InsertReceiptChain, blockchain.Rollback, + manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeaderByHash, + blockchain.GetBlockByHash, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, + blockchain.GetTdByHash, blockchain.InsertHeaderChain, manager.insertChain, blockchain.InsertReceiptChain, blockchain.Rollback, manager.removePeer) validator := func(block *types.Block, parent *types.Block) error { @@ -167,7 +167,7 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, atomic.StoreUint32(&manager.synced, 1) // Mark initial sync done on any fetcher import return manager.insertChain(blocks) } - manager.fetcher = fetcher.New(blockchain.GetBlock, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) + manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) if blockchain.Genesis().Hash().Hex() == defaultGenesisHash && networkId == 1 { glog.V(logger.Debug).Infoln("Bad Block Reporting is enabled") @@ -382,7 +382,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Retrieve the requested block, stopping if enough was found - if block := pm.blockchain.GetBlock(hash); block != nil { + if block := pm.blockchain.GetBlockByHash(hash); block != nil { blocks = append(blocks, block) bytes += block.Size() } @@ -425,13 +425,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Retrieve the next header satisfying the query var origin *types.Header if hashMode { - origin = pm.blockchain.GetHeader(query.Origin.Hash) + origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) } else { origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) } if origin == nil { break } + number := origin.Number.Uint64() headers = append(headers, origin) bytes += estHeaderRlpSize @@ -440,8 +441,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { case query.Origin.Hash != (common.Hash{}) && query.Reverse: // Hash based traversal towards the genesis block for i := 0; i < int(query.Skip)+1; i++ { - if header := pm.blockchain.GetHeader(query.Origin.Hash); header != nil { + if header := pm.blockchain.GetHeader(query.Origin.Hash, number); header != nil { query.Origin.Hash = header.ParentHash + number-- } else { unknown = true break @@ -602,9 +604,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Retrieve the requested block's receipts, skipping if unknown to us - results := core.GetBlockReceipts(pm.chaindb, hash) + results := core.GetBlockReceipts(pm.chaindb, hash, core.GetBlockNumber(pm.chaindb, hash)) if results == nil { - if header := pm.blockchain.GetHeader(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { continue } } @@ -697,7 +699,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Update the peers total difficulty if needed, schedule a download if gapped if request.TD.Cmp(p.Td()) > 0 { p.SetTd(request.TD) - td := pm.blockchain.GetTd(pm.blockchain.CurrentBlock().Hash()) + currentBlock := pm.blockchain.CurrentBlock() + td := pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) if request.TD.Cmp(new(big.Int).Add(td, request.Block.Difficulty())) > 0 { go pm.synchronise(p) } @@ -738,8 +741,8 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { if propagate { // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) var td *big.Int - if parent := pm.blockchain.GetBlock(block.ParentHash()); parent != nil { - td = new(big.Int).Add(block.Difficulty(), pm.blockchain.GetTd(block.ParentHash())) + if parent := pm.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { + td = new(big.Int).Add(block.Difficulty(), pm.blockchain.GetTd(block.ParentHash(), block.NumberU64()-1)) } else { glog.V(logger.Error).Infof("propagating dangling block #%d [%x]", block.NumberU64(), hash[:4]) return @@ -807,10 +810,11 @@ type EthNodeInfo struct { // NodeInfo retrieves some protocol metadata about the running host node. func (self *ProtocolManager) NodeInfo() *EthNodeInfo { + currentBlock := self.blockchain.CurrentBlock() return &EthNodeInfo{ Network: self.networkId, - Difficulty: self.blockchain.GetTd(self.blockchain.CurrentBlock().Hash()), + Difficulty: self.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), Genesis: self.blockchain.Genesis().Hash(), - Head: self.blockchain.CurrentBlock().Hash(), + Head: currentBlock.Hash(), } } diff --git a/eth/handler_test.go b/eth/handler_test.go index 9e593f040..8418c28b2 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -91,7 +91,7 @@ func testGetBlockHashes(t *testing.T, protocol int) { // Assemble the hash response we would like to receive resp := make([]common.Hash, tt.result) if len(resp) > 0 { - from := pm.blockchain.GetBlock(tt.origin).NumberU64() - 1 + from := pm.blockchain.GetBlockByHash(tt.origin).NumberU64() - 1 for j := 0; j < len(resp); j++ { resp[j] = pm.blockchain.GetBlockByNumber(uint64(int(from) - j)).Hash() } @@ -204,7 +204,7 @@ func testGetBlocks(t *testing.T, protocol int) { for j, hash := range tt.explicit { hashes = append(hashes, hash) if tt.available[j] && len(blocks) < tt.expected { - blocks = append(blocks, pm.blockchain.GetBlock(hash)) + blocks = append(blocks, pm.blockchain.GetBlockByHash(hash)) } } // Send the hash request and verify the response @@ -339,7 +339,7 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Collect the headers to expect in the response headers := []*types.Header{} for _, hash := range tt.expect { - headers = append(headers, pm.blockchain.GetBlock(hash).Header()) + headers = append(headers, pm.blockchain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response p2p.Send(peer.app, 0x03, tt.query) @@ -420,7 +420,7 @@ func testGetBlockBodies(t *testing.T, protocol int) { for j, hash := range tt.explicit { hashes = append(hashes, hash) if tt.available[j] && len(bodies) < tt.expected { - block := pm.blockchain.GetBlock(hash) + block := pm.blockchain.GetBlockByHash(hash) bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) } } @@ -572,7 +572,7 @@ func testGetReceipt(t *testing.T, protocol int) { block := pm.blockchain.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash())) + receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response p2p.Send(peer.app, 0x0f, hashes) diff --git a/eth/sync.go b/eth/sync.go index 52f7e90e7..23cf18c8d 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -162,7 +162,8 @@ func (pm *ProtocolManager) synchronise(peer *peer) { return } // Make sure the peer's TD is higher than our own. If not drop. - td := pm.blockchain.GetTd(pm.blockchain.CurrentBlock().Hash()) + currentBlock := pm.blockchain.CurrentBlock() + td := pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) if peer.Td().Cmp(td) <= 0 { return } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 5b1a9b23c..9fc5fc4fe 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -22,9 +22,9 @@ import ( _ "net/http/pprof" "runtime" - "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "gopkg.in/urfave/cli.v1" ) var ( diff --git a/miner/worker.go b/miner/worker.go index fe759560c..09cf6b6aa 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -272,7 +272,7 @@ func (self *worker) wait() { go self.mux.Post(core.NewMinedBlockEvent{Block: block}) } else { work.state.Commit() - parent := self.chain.GetBlock(block.ParentHash()) + parent := self.chain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { glog.V(logger.Error).Infoln("Invalid block found during mining") continue @@ -319,7 +319,7 @@ func (self *worker) wait() { self.mux.Post(core.ChainHeadEvent{Block: block}) self.mux.Post(logs) } - if err := core.WriteBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil { + if err := core.WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { glog.V(logger.Warn).Infoln("error writing block receipts:", err) } }(block, work.state.Logs(), work.receipts) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index b92c183e1..d9a5eec08 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -164,7 +164,7 @@ func runBlockTest(homesteadBlock *big.Int, test *BlockTest) error { return fmt.Errorf("InsertPreState: %v", err) } - core.WriteTd(db, test.Genesis.Hash(), test.Genesis.Difficulty()) + core.WriteTd(db, test.Genesis.Hash(), 0, test.Genesis.Difficulty()) core.WriteBlock(db, test.Genesis) core.WriteCanonicalHash(db, test.Genesis.Hash(), test.Genesis.NumberU64()) core.WriteHeadBlockHash(db, test.Genesis.Hash()) @@ -412,7 +412,7 @@ func (test *BlockTest) ValidateImportedHeaders(cm *core.BlockChain, validBlocks // block-by-block, so we can only validate imported headers after // all blocks have been processed by ChainManager, as they may not // be part of the longest chain until last block is imported. - for b := cm.CurrentBlock(); b != nil && b.NumberU64() != 0; b = cm.GetBlock(b.Header().ParentHash) { + for b := cm.CurrentBlock(); b != nil && b.NumberU64() != 0; b = cm.GetBlockByHash(b.Header().ParentHash) { bHash := common.Bytes2Hex(b.Hash().Bytes()) // hex without 0x prefix if err := validateHeader(bmap[bHash].BlockHeader, b.Header()); err != nil { return fmt.Errorf("Imported block header validation failed: %v", err) |