From 3b9808f23ca4eb1621a92aad80de5c89269f17fe Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 13:29:34 +0200 Subject: cmd/geth, cmd/utils: don't use Ethereum for import, export and upgradedb The blockchain commands don't need the full stack. With this change, p2p, miner, downloader, etc are no longer started for blockchain operations. --- cmd/utils/cmd.go | 49 +++++++++++++++++++++---------------------------- cmd/utils/flags.go | 44 ++++++++++++++++++-------------------------- 2 files changed, 39 insertions(+), 54 deletions(-) (limited to 'cmd/utils') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 39b4e46da..550ac1c51 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -167,7 +167,7 @@ func FormatTransactionData(data string) []byte { } func ImportChain(chainmgr *core.ChainManager, fn string) error { - fmt.Printf("importing blockchain '%s'\n", fn) + glog.Infoln("Importing blockchain", fn) fh, err := os.OpenFile(fn, os.O_RDONLY, os.ModePerm) if err != nil { return err @@ -176,43 +176,36 @@ func ImportChain(chainmgr *core.ChainManager, fn string) error { chainmgr.Reset() stream := rlp.NewStream(fh, 0) - var i, n int batchSize := 2500 blocks := make(types.Blocks, batchSize) - - for ; ; i++ { - var b types.Block - if err := stream.Decode(&b); err == io.EOF { - break - } else if err != nil { - return fmt.Errorf("at block %d: %v", i, err) - } - - blocks[n] = &b - n++ - - if n == batchSize { - if _, err := chainmgr.InsertChain(blocks); err != nil { - return fmt.Errorf("invalid block %v", err) + n := 0 + for { + // Load a batch of RLP blocks. + i := 0 + for ; i < batchSize; i++ { + var b types.Block + if err := stream.Decode(&b); err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("at block %d: %v", n, err) } - n = 0 - blocks = make(types.Blocks, batchSize) + blocks[i] = &b + n++ } - } - - if n > 0 { - if _, err := chainmgr.InsertChain(blocks[:n]); err != nil { - return fmt.Errorf("invalid block %v", err) + if i == 0 { + break + } + // Import the batch. + if _, err := chainmgr.InsertChain(blocks[:i]); err != nil { + return fmt.Errorf("invalid block %d: %v", n, err) } } - - fmt.Printf("imported %d blocks\n", i) return nil } func ExportChain(chainmgr *core.ChainManager, fn string) error { - fmt.Printf("exporting blockchain '%s'\n", fn) + glog.Infoln("Exporting blockchain to", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { return err @@ -221,6 +214,6 @@ func ExportChain(chainmgr *core.ChainManager, fn string) error { if err := chainmgr.Export(fh); err != nil { return err } - fmt.Printf("exported blockchain\n") + glog.Infoln("Exported blockchain to", fn) return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 155110ddc..176a546f1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -283,20 +283,10 @@ func GetNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { } func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { - // Set verbosity on glog - glog.SetV(ctx.GlobalInt(VerbosityFlag.Name)) - glog.CopyStandardLogTo("INFO") - // Set the log type - //glog.SetToStderr(ctx.GlobalBool(LogToStdErrFlag.Name)) - glog.SetToStderr(true) - // Set the log dir - glog.SetLogDir(ctx.GlobalString(LogFileFlag.Name)) - customName := ctx.GlobalString(IdentityFlag.Name) if len(customName) > 0 { clientID += "/" + customName } - return ð.Config{ Name: common.MakeName(clientID, version), DataDir: ctx.GlobalString(DataDirFlag.Name), @@ -327,32 +317,34 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { } } -func GetChain(ctx *cli.Context) (*core.ChainManager, common.Database, common.Database) { - dataDir := ctx.GlobalString(DataDirFlag.Name) +// SetupLogger configures glog from the logging-related command line flags. +func SetupLogger(ctx *cli.Context) { + glog.SetV(ctx.GlobalInt(VerbosityFlag.Name)) + glog.CopyStandardLogTo("INFO") + glog.SetToStderr(true) + glog.SetLogDir(ctx.GlobalString(LogFileFlag.Name)) +} - blockDb, err := ethdb.NewLDBDatabase(filepath.Join(dataDir, "blockchain")) - if err != nil { +func GetChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { + dd := ctx.GlobalString(DataDirFlag.Name) + var err error + if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "blockchain")); err != nil { Fatalf("Could not open database: %v", err) } - - stateDb, err := ethdb.NewLDBDatabase(filepath.Join(dataDir, "state")) - if err != nil { + if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "state")); err != nil { Fatalf("Could not open database: %v", err) } - - extraDb, err := ethdb.NewLDBDatabase(filepath.Join(dataDir, "extra")) - if err != nil { + if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "extra")); err != nil { Fatalf("Could not open database: %v", err) } eventMux := new(event.TypeMux) pow := ethash.New() - chainManager := core.NewChainManager(blockDb, stateDb, pow, eventMux) - txPool := core.NewTxPool(eventMux, chainManager.State, chainManager.GasLimit) - blockProcessor := core.NewBlockProcessor(stateDb, extraDb, pow, txPool, chainManager, eventMux) - chainManager.SetProcessor(blockProcessor) - - return chainManager, blockDb, stateDb + chain = core.NewChainManager(blockDB, stateDB, pow, eventMux) + txpool := core.NewTxPool(eventMux, chain.State, chain.GasLimit) + proc := core.NewBlockProcessor(stateDB, extraDB, pow, txpool, chain, eventMux) + chain.SetProcessor(proc) + return chain, blockDB, stateDB, extraDB } func GetAccountManager(ctx *cli.Context) *accounts.Manager { -- cgit v1.2.3 From 74706a0f029fe74ea15e366904d827da03091fef Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 14:50:31 +0200 Subject: cmd/geth, cmd/utils: rename utils.Get* -> utils.Make* The renaming should make it clearer that these functions create a new instance for every call. @obscuren suggested this renaming a while ago. --- cmd/utils/flags.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'cmd/utils') diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 176a546f1..d319055b1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -256,7 +256,8 @@ var ( } ) -func GetNAT(ctx *cli.Context) nat.Interface { +// MakeNAT creates a port mapper from set command line flags. +func MakeNAT(ctx *cli.Context) nat.Interface { natif, err := nat.Parse(ctx.GlobalString(NATFlag.Name)) if err != nil { Fatalf("Option %s: %v", NATFlag.Name, err) @@ -264,7 +265,8 @@ func GetNAT(ctx *cli.Context) nat.Interface { return natif } -func GetNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { +// MakeNodeKey creates a node key from set command line flags. +func MakeNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { hex, file := ctx.GlobalString(NodeKeyHexFlag.Name), ctx.GlobalString(NodeKeyFileFlag.Name) var err error switch { @@ -282,6 +284,7 @@ func GetNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { return key } +// MakeEthConfig creates ethereum options from set command line flags. func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { customName := ctx.GlobalString(IdentityFlag.Name) if len(customName) > 0 { @@ -299,15 +302,15 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { LogJSON: ctx.GlobalString(LogJSONFlag.Name), Etherbase: ctx.GlobalString(EtherbaseFlag.Name), MinerThreads: ctx.GlobalInt(MinerThreadsFlag.Name), - AccountManager: GetAccountManager(ctx), + AccountManager: MakeAccountManager(ctx), VmDebug: ctx.GlobalBool(VMDebugFlag.Name), MaxPeers: ctx.GlobalInt(MaxPeersFlag.Name), MaxPendingPeers: ctx.GlobalInt(MaxPendingPeersFlag.Name), Port: ctx.GlobalString(ListenPortFlag.Name), - NAT: GetNAT(ctx), + NAT: MakeNAT(ctx), NatSpec: ctx.GlobalBool(NatspecEnabledFlag.Name), Discovery: !ctx.GlobalBool(NoDiscoverFlag.Name), - NodeKey: GetNodeKey(ctx), + NodeKey: MakeNodeKey(ctx), Shh: ctx.GlobalBool(WhisperEnabledFlag.Name), Dial: true, BootNodes: ctx.GlobalString(BootnodesFlag.Name), @@ -325,7 +328,8 @@ func SetupLogger(ctx *cli.Context) { glog.SetLogDir(ctx.GlobalString(LogFileFlag.Name)) } -func GetChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { +// MakeChain creates a chain manager from set command line flags. +func MakeChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { dd := ctx.GlobalString(DataDirFlag.Name) var err error if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "blockchain")); err != nil { @@ -347,7 +351,8 @@ func GetChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, ext return chain, blockDB, stateDB, extraDB } -func GetAccountManager(ctx *cli.Context) *accounts.Manager { +// MakeChain creates an account manager from set command line flags. +func MakeAccountManager(ctx *cli.Context) *accounts.Manager { dataDir := ctx.GlobalString(DataDirFlag.Name) ks := crypto.NewKeyStorePassphrase(filepath.Join(dataDir, "keystore")) return accounts.NewManager(ks) -- cgit v1.2.3 From 705beb4c25f976f6bce40818bd835e235c2babf4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 15:48:07 +0200 Subject: cmd/utils: print errors only once if stdout and stderr are the same file --- cmd/utils/cmd.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cmd/utils') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 550ac1c51..06c240bd4 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -125,10 +125,17 @@ func initDataDir(Datadir string) { } } -// Fatalf formats a message to standard output and exits the program. +// Fatalf formats a message to standard error and exits the program. +// The message is also printed to standard output if standard error +// is redirected to a different file. func Fatalf(format string, args ...interface{}) { - fmt.Fprintf(os.Stderr, "Fatal: "+format+"\n", args...) - fmt.Fprintf(os.Stdout, "Fatal: "+format+"\n", args...) + w := io.MultiWriter(os.Stdout, os.Stderr) + outf, _ := os.Stdout.Stat() + errf, _ := os.Stderr.Stat() + if outf != nil && errf != nil && os.SameFile(outf, errf) { + w = os.Stderr + } + fmt.Fprintf(w, "Fatal: "+format+"\n", args...) logger.Flush() os.Exit(1) } -- cgit v1.2.3 From 67effb94b6fadac7b207cb5333c91f578326762e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 16:02:08 +0200 Subject: cmd/geth, cmd/utils: make chain importing interruptible Interrupting import with Ctrl-C could cause database corruption because the signal wasn't handled. utils.ImportChain now checks for a queued interrupt on every batch. --- cmd/utils/cmd.go | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'cmd/utils') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 06c240bd4..2e2b627df 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -173,22 +173,47 @@ func FormatTransactionData(data string) []byte { return d } -func ImportChain(chainmgr *core.ChainManager, fn string) error { +func ImportChain(chain *core.ChainManager, fn string) error { + // Watch for Ctrl-C while the import is running. + // If a signal is received, the import will stop at the next batch. + interrupt := make(chan os.Signal, 1) + stop := make(chan struct{}) + signal.Notify(interrupt, os.Interrupt) + defer signal.Stop(interrupt) + defer close(interrupt) + go func() { + if _, ok := <-interrupt; ok { + glog.Info("caught interrupt during import, will stop at next batch") + } + close(stop) + }() + checkInterrupt := func() bool { + select { + case <-stop: + return true + default: + return false + } + } + glog.Infoln("Importing blockchain", fn) - fh, err := os.OpenFile(fn, os.O_RDONLY, os.ModePerm) + fh, err := os.Open(fn) if err != nil { return err } defer fh.Close() - - chainmgr.Reset() stream := rlp.NewStream(fh, 0) + // Remove all existing blocks and start the import. + chain.Reset() batchSize := 2500 blocks := make(types.Blocks, batchSize) n := 0 for { // Load a batch of RLP blocks. + if checkInterrupt() { + return fmt.Errorf("interrupted") + } i := 0 for ; i < batchSize; i++ { var b types.Block @@ -204,7 +229,10 @@ func ImportChain(chainmgr *core.ChainManager, fn string) error { break } // Import the batch. - if _, err := chainmgr.InsertChain(blocks[:i]); err != nil { + if checkInterrupt() { + return fmt.Errorf("interrupted") + } + if _, err := chain.InsertChain(blocks[:i]); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } } -- cgit v1.2.3 From a8bc2181c94f5d3a9455c4fa526f8722a21ecb04 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 17:35:08 +0200 Subject: cmd/utils: skip batches with known blocks during import This makes block importing restartable. --- cmd/utils/cmd.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'cmd/utils') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 2e2b627df..2a5e2ec6a 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -204,12 +204,11 @@ func ImportChain(chain *core.ChainManager, fn string) error { defer fh.Close() stream := rlp.NewStream(fh, 0) - // Remove all existing blocks and start the import. - chain.Reset() + // Run actual the import. batchSize := 2500 blocks := make(types.Blocks, batchSize) n := 0 - for { + for batch := 0; ; batch++ { // Load a batch of RLP blocks. if checkInterrupt() { return fmt.Errorf("interrupted") @@ -232,6 +231,11 @@ func ImportChain(chain *core.ChainManager, fn string) error { if checkInterrupt() { return fmt.Errorf("interrupted") } + if hasAllBlocks(chain, blocks[:i]) { + glog.Infof("skipping batch %d, all blocks present [%x / %x]", + batch, blocks[0].Hash().Bytes()[:4], blocks[i-1].Hash().Bytes()[:4]) + continue + } if _, err := chain.InsertChain(blocks[:i]); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } @@ -239,6 +243,15 @@ func ImportChain(chain *core.ChainManager, fn string) error { return nil } +func hasAllBlocks(chain *core.ChainManager, bs []*types.Block) bool { + for _, b := range bs { + if !chain.HasBlock(b.Hash()) { + return false + } + } + return true +} + func ExportChain(chainmgr *core.ChainManager, fn string) error { glog.Infoln("Exporting blockchain to", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) -- cgit v1.2.3 From e1fe75e3b637758f99ddbcaeb01eafa1a0b6455e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 May 2015 01:16:57 +0200 Subject: cmd/utils: use constant for import batch size --- cmd/utils/cmd.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'cmd/utils') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 2a5e2ec6a..e5413973d 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -40,6 +40,10 @@ import ( "github.com/peterh/liner" ) +const ( + importBatchSize = 2500 +) + var interruptCallbacks = []func(os.Signal){} // Register interrupt handlers callbacks @@ -205,8 +209,7 @@ func ImportChain(chain *core.ChainManager, fn string) error { stream := rlp.NewStream(fh, 0) // Run actual the import. - batchSize := 2500 - blocks := make(types.Blocks, batchSize) + blocks := make(types.Blocks, importBatchSize) n := 0 for batch := 0; ; batch++ { // Load a batch of RLP blocks. @@ -214,7 +217,7 @@ func ImportChain(chain *core.ChainManager, fn string) error { return fmt.Errorf("interrupted") } i := 0 - for ; i < batchSize; i++ { + for ; i < importBatchSize; i++ { var b types.Block if err := stream.Decode(&b); err == io.EOF { break -- cgit v1.2.3