From 3b9808f23ca4eb1621a92aad80de5c89269f17fe Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 13:29:34 +0200 Subject: cmd/geth, cmd/utils: don't use Ethereum for import, export and upgradedb The blockchain commands don't need the full stack. With this change, p2p, miner, downloader, etc are no longer started for blockchain operations. --- cmd/utils/cmd.go | 49 +++++++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) (limited to 'cmd/utils/cmd.go') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 39b4e46da..550ac1c51 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -167,7 +167,7 @@ func FormatTransactionData(data string) []byte { } func ImportChain(chainmgr *core.ChainManager, fn string) error { - fmt.Printf("importing blockchain '%s'\n", fn) + glog.Infoln("Importing blockchain", fn) fh, err := os.OpenFile(fn, os.O_RDONLY, os.ModePerm) if err != nil { return err @@ -176,43 +176,36 @@ func ImportChain(chainmgr *core.ChainManager, fn string) error { chainmgr.Reset() stream := rlp.NewStream(fh, 0) - var i, n int batchSize := 2500 blocks := make(types.Blocks, batchSize) - - for ; ; i++ { - var b types.Block - if err := stream.Decode(&b); err == io.EOF { - break - } else if err != nil { - return fmt.Errorf("at block %d: %v", i, err) - } - - blocks[n] = &b - n++ - - if n == batchSize { - if _, err := chainmgr.InsertChain(blocks); err != nil { - return fmt.Errorf("invalid block %v", err) + n := 0 + for { + // Load a batch of RLP blocks. + i := 0 + for ; i < batchSize; i++ { + var b types.Block + if err := stream.Decode(&b); err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("at block %d: %v", n, err) } - n = 0 - blocks = make(types.Blocks, batchSize) + blocks[i] = &b + n++ } - } - - if n > 0 { - if _, err := chainmgr.InsertChain(blocks[:n]); err != nil { - return fmt.Errorf("invalid block %v", err) + if i == 0 { + break + } + // Import the batch. + if _, err := chainmgr.InsertChain(blocks[:i]); err != nil { + return fmt.Errorf("invalid block %d: %v", n, err) } } - - fmt.Printf("imported %d blocks\n", i) return nil } func ExportChain(chainmgr *core.ChainManager, fn string) error { - fmt.Printf("exporting blockchain '%s'\n", fn) + glog.Infoln("Exporting blockchain to", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { return err @@ -221,6 +214,6 @@ func ExportChain(chainmgr *core.ChainManager, fn string) error { if err := chainmgr.Export(fh); err != nil { return err } - fmt.Printf("exported blockchain\n") + glog.Infoln("Exported blockchain to", fn) return nil } -- cgit v1.2.3 From 705beb4c25f976f6bce40818bd835e235c2babf4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 15:48:07 +0200 Subject: cmd/utils: print errors only once if stdout and stderr are the same file --- cmd/utils/cmd.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cmd/utils/cmd.go') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 550ac1c51..06c240bd4 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -125,10 +125,17 @@ func initDataDir(Datadir string) { } } -// Fatalf formats a message to standard output and exits the program. +// Fatalf formats a message to standard error and exits the program. +// The message is also printed to standard output if standard error +// is redirected to a different file. func Fatalf(format string, args ...interface{}) { - fmt.Fprintf(os.Stderr, "Fatal: "+format+"\n", args...) - fmt.Fprintf(os.Stdout, "Fatal: "+format+"\n", args...) + w := io.MultiWriter(os.Stdout, os.Stderr) + outf, _ := os.Stdout.Stat() + errf, _ := os.Stderr.Stat() + if outf != nil && errf != nil && os.SameFile(outf, errf) { + w = os.Stderr + } + fmt.Fprintf(w, "Fatal: "+format+"\n", args...) logger.Flush() os.Exit(1) } -- cgit v1.2.3 From 67effb94b6fadac7b207cb5333c91f578326762e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 16:02:08 +0200 Subject: cmd/geth, cmd/utils: make chain importing interruptible Interrupting import with Ctrl-C could cause database corruption because the signal wasn't handled. utils.ImportChain now checks for a queued interrupt on every batch. --- cmd/utils/cmd.go | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'cmd/utils/cmd.go') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 06c240bd4..2e2b627df 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -173,22 +173,47 @@ func FormatTransactionData(data string) []byte { return d } -func ImportChain(chainmgr *core.ChainManager, fn string) error { +func ImportChain(chain *core.ChainManager, fn string) error { + // Watch for Ctrl-C while the import is running. + // If a signal is received, the import will stop at the next batch. + interrupt := make(chan os.Signal, 1) + stop := make(chan struct{}) + signal.Notify(interrupt, os.Interrupt) + defer signal.Stop(interrupt) + defer close(interrupt) + go func() { + if _, ok := <-interrupt; ok { + glog.Info("caught interrupt during import, will stop at next batch") + } + close(stop) + }() + checkInterrupt := func() bool { + select { + case <-stop: + return true + default: + return false + } + } + glog.Infoln("Importing blockchain", fn) - fh, err := os.OpenFile(fn, os.O_RDONLY, os.ModePerm) + fh, err := os.Open(fn) if err != nil { return err } defer fh.Close() - - chainmgr.Reset() stream := rlp.NewStream(fh, 0) + // Remove all existing blocks and start the import. + chain.Reset() batchSize := 2500 blocks := make(types.Blocks, batchSize) n := 0 for { // Load a batch of RLP blocks. + if checkInterrupt() { + return fmt.Errorf("interrupted") + } i := 0 for ; i < batchSize; i++ { var b types.Block @@ -204,7 +229,10 @@ func ImportChain(chainmgr *core.ChainManager, fn string) error { break } // Import the batch. - if _, err := chainmgr.InsertChain(blocks[:i]); err != nil { + if checkInterrupt() { + return fmt.Errorf("interrupted") + } + if _, err := chain.InsertChain(blocks[:i]); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } } -- cgit v1.2.3 From a8bc2181c94f5d3a9455c4fa526f8722a21ecb04 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 17:35:08 +0200 Subject: cmd/utils: skip batches with known blocks during import This makes block importing restartable. --- cmd/utils/cmd.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'cmd/utils/cmd.go') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 2e2b627df..2a5e2ec6a 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -204,12 +204,11 @@ func ImportChain(chain *core.ChainManager, fn string) error { defer fh.Close() stream := rlp.NewStream(fh, 0) - // Remove all existing blocks and start the import. - chain.Reset() + // Run actual the import. batchSize := 2500 blocks := make(types.Blocks, batchSize) n := 0 - for { + for batch := 0; ; batch++ { // Load a batch of RLP blocks. if checkInterrupt() { return fmt.Errorf("interrupted") @@ -232,6 +231,11 @@ func ImportChain(chain *core.ChainManager, fn string) error { if checkInterrupt() { return fmt.Errorf("interrupted") } + if hasAllBlocks(chain, blocks[:i]) { + glog.Infof("skipping batch %d, all blocks present [%x / %x]", + batch, blocks[0].Hash().Bytes()[:4], blocks[i-1].Hash().Bytes()[:4]) + continue + } if _, err := chain.InsertChain(blocks[:i]); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } @@ -239,6 +243,15 @@ func ImportChain(chain *core.ChainManager, fn string) error { return nil } +func hasAllBlocks(chain *core.ChainManager, bs []*types.Block) bool { + for _, b := range bs { + if !chain.HasBlock(b.Hash()) { + return false + } + } + return true +} + func ExportChain(chainmgr *core.ChainManager, fn string) error { glog.Infoln("Exporting blockchain to", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) -- cgit v1.2.3 From e1fe75e3b637758f99ddbcaeb01eafa1a0b6455e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 May 2015 01:16:57 +0200 Subject: cmd/utils: use constant for import batch size --- cmd/utils/cmd.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'cmd/utils/cmd.go') diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 2a5e2ec6a..e5413973d 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -40,6 +40,10 @@ import ( "github.com/peterh/liner" ) +const ( + importBatchSize = 2500 +) + var interruptCallbacks = []func(os.Signal){} // Register interrupt handlers callbacks @@ -205,8 +209,7 @@ func ImportChain(chain *core.ChainManager, fn string) error { stream := rlp.NewStream(fh, 0) // Run actual the import. - batchSize := 2500 - blocks := make(types.Blocks, batchSize) + blocks := make(types.Blocks, importBatchSize) n := 0 for batch := 0; ; batch++ { // Load a batch of RLP blocks. @@ -214,7 +217,7 @@ func ImportChain(chain *core.ChainManager, fn string) error { return fmt.Errorf("interrupted") } i := 0 - for ; i < batchSize; i++ { + for ; i < importBatchSize; i++ { var b types.Block if err := stream.Decode(&b); err == io.EOF { break -- cgit v1.2.3