diff options
81 files changed, 2969 insertions, 1104 deletions
diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index c939ae670..e4b37a12e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -6,11 +6,6 @@ ], "Deps": [ { - "ImportPath": "code.google.com/p/go-uuid/uuid", - "Comment": "null-12", - "Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9" - }, - { "ImportPath": "github.com/codegangsta/cli", "Comment": "1.2.0-95-g9b2bd2b", "Rev": "9b2bd2b3489748d4d0a204fa4eb2ee9e89e0ebc6" @@ -21,7 +16,8 @@ }, { "ImportPath": "github.com/ethereum/ethash", - "Rev": "227ec953eae56f4f6c7f5e7dc93b4bbebf0cda2e" + "Comment": "v23.1-234-g062e40a", + "Rev": "062e40a1a1671f5a5102862b56e4c56f68a732f5" }, { "ImportPath": "github.com/fatih/color", @@ -62,6 +58,10 @@ "Rev": "675ffd907b7401b8a709a5ef2249978af5616bb2" }, { + "ImportPath": "github.com/pborman/uuid", + "Rev": "cccd189d45f7ac3368a0d127efb7f4d08ae0b655" + }, + { "ImportPath": "github.com/peterh/liner", "Rev": "29f6a646557d83e2b6e9ba05c45fbea9c006dbe8" }, diff --git a/Godeps/_workspace/src/github.com/ethereum/ethash/setup.py b/Godeps/_workspace/src/github.com/ethereum/ethash/setup.py index 18aa20f6d..18aa20f6d 100755..100644 --- a/Godeps/_workspace/src/github.com/ethereum/ethash/setup.py +++ b/Godeps/_workspace/src/github.com/ethereum/ethash/setup.py diff --git a/Godeps/_workspace/src/github.com/ethereum/ethash/src/libethash/endian.h b/Godeps/_workspace/src/github.com/ethereum/ethash/src/libethash/endian.h index e32b1c539..849325a59 100644 --- a/Godeps/_workspace/src/github.com/ethereum/ethash/src/libethash/endian.h +++ b/Godeps/_workspace/src/github.com/ethereum/ethash/src/libethash/endian.h @@ -41,8 +41,8 @@ #define ethash_swap_u64(input_) swap64(input_) #else // posix #include <byteswap.h> -#define ethash_swap_u32(input_) __bswap_32(input_) -#define ethash_swap_u64(input_) __bswap_64(input_) +#define ethash_swap_u32(input_) bswap_32(input_) +#define ethash_swap_u64(input_) bswap_64(input_) #endif diff --git a/Godeps/_workspace/src/github.com/ethereum/ethash/test/c/test.sh b/Godeps/_workspace/src/github.com/ethereum/ethash/test/c/test.sh index 92b6b8b66..92b6b8b66 100755..100644 --- a/Godeps/_workspace/src/github.com/ethereum/ethash/test/c/test.sh +++ b/Godeps/_workspace/src/github.com/ethereum/ethash/test/c/test.sh diff --git a/Godeps/_workspace/src/github.com/ethereum/ethash/test/python/test.sh b/Godeps/_workspace/src/github.com/ethereum/ethash/test/python/test.sh index 05c66b550..05c66b550 100755..100644 --- a/Godeps/_workspace/src/github.com/ethereum/ethash/test/python/test.sh +++ b/Godeps/_workspace/src/github.com/ethereum/ethash/test/python/test.sh diff --git a/Godeps/_workspace/src/github.com/ethereum/ethash/test/test.sh b/Godeps/_workspace/src/github.com/ethereum/ethash/test/test.sh index aaeaa878c..aaeaa878c 100755..100644 --- a/Godeps/_workspace/src/github.com/ethereum/ethash/test/test.sh +++ b/Godeps/_workspace/src/github.com/ethereum/ethash/test/test.sh diff --git a/Godeps/_workspace/src/github.com/pborman/uuid/CONTRIBUTORS b/Godeps/_workspace/src/github.com/pborman/uuid/CONTRIBUTORS new file mode 100644 index 000000000..b382a04ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/pborman/uuid/CONTRIBUTORS @@ -0,0 +1 @@ +Paul Borman <borman@google.com> diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/LICENSE b/Godeps/_workspace/src/github.com/pborman/uuid/LICENSE index ab6b011a1..5dc68268d 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/LICENSE +++ b/Godeps/_workspace/src/github.com/pborman/uuid/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 Google Inc. All rights reserved. +Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/dce.go b/Godeps/_workspace/src/github.com/pborman/uuid/dce.go index 50a0f2d09..50a0f2d09 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/dce.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/dce.go diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/doc.go b/Godeps/_workspace/src/github.com/pborman/uuid/doc.go index d8bd013e6..d8bd013e6 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/doc.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/doc.go diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/hash.go b/Godeps/_workspace/src/github.com/pborman/uuid/hash.go index cdd4192fd..cdd4192fd 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/hash.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/hash.go diff --git a/Godeps/_workspace/src/github.com/pborman/uuid/json.go b/Godeps/_workspace/src/github.com/pborman/uuid/json.go new file mode 100644 index 000000000..760580a50 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pborman/uuid/json.go @@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "errors" + +func (u UUID) MarshalJSON() ([]byte, error) { + if len(u) == 0 { + return []byte(`""`), nil + } + return []byte(`"` + u.String() + `"`), nil +} + +func (u *UUID) UnmarshalJSON(data []byte) error { + if len(data) == 0 || string(data) == `""` { + return nil + } + if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { + return errors.New("invalid UUID format") + } + data = data[1 : len(data)-1] + uu := Parse(string(data)) + if uu == nil { + return errors.New("invalid UUID format") + } + *u = uu + return nil +} diff --git a/Godeps/_workspace/src/github.com/pborman/uuid/json_test.go b/Godeps/_workspace/src/github.com/pborman/uuid/json_test.go new file mode 100644 index 000000000..b5eae0924 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pborman/uuid/json_test.go @@ -0,0 +1,32 @@ +// Copyright 2014 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/json" + "reflect" + "testing" +) + +var testUUID = Parse("f47ac10b-58cc-0372-8567-0e02b2c3d479") + +func TestJSON(t *testing.T) { + type S struct { + ID1 UUID + ID2 UUID + } + s1 := S{ID1: testUUID} + data, err := json.Marshal(&s1) + if err != nil { + t.Fatal(err) + } + var s2 S + if err := json.Unmarshal(data, &s2); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&s1, &s2) { + t.Errorf("got %#v, want %#v", s2, s1) + } +} diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/node.go b/Godeps/_workspace/src/github.com/pborman/uuid/node.go index dd0a8ac18..dd0a8ac18 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/node.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/node.go diff --git a/Godeps/_workspace/src/github.com/pborman/uuid/seq_test.go b/Godeps/_workspace/src/github.com/pborman/uuid/seq_test.go new file mode 100644 index 000000000..3b3d1430d --- /dev/null +++ b/Godeps/_workspace/src/github.com/pborman/uuid/seq_test.go @@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "flag" + "runtime" + "testing" + "time" +) + +// This test is only run when --regressions is passed on the go test line. +var regressions = flag.Bool("regressions", false, "run uuid regression tests") + +// TestClockSeqRace tests for a particular race condition of returning two +// identical Version1 UUIDs. The duration of 1 minute was chosen as the race +// condition, before being fixed, nearly always occured in under 30 seconds. +func TestClockSeqRace(t *testing.T) { + if !*regressions { + t.Skip("skipping regression tests") + } + duration := time.Minute + + done := make(chan struct{}) + defer close(done) + + ch := make(chan UUID, 10000) + ncpu := runtime.NumCPU() + switch ncpu { + case 0, 1: + // We can't run the test effectively. + t.Skip("skipping race test, only one CPU detected") + return + default: + runtime.GOMAXPROCS(ncpu) + } + for i := 0; i < ncpu; i++ { + go func() { + for { + select { + case <-done: + return + case ch <- NewUUID(): + } + } + }() + } + + uuids := make(map[string]bool) + cnt := 0 + start := time.Now() + for u := range ch { + s := u.String() + if uuids[s] { + t.Errorf("duplicate uuid after %d in %v: %s", cnt, time.Since(start), s) + return + } + uuids[s] = true + if time.Since(start) > duration { + return + } + cnt++ + } +} diff --git a/Godeps/_workspace/src/github.com/pborman/uuid/sql.go b/Godeps/_workspace/src/github.com/pborman/uuid/sql.go new file mode 100644 index 000000000..2d7679e2a --- /dev/null +++ b/Godeps/_workspace/src/github.com/pborman/uuid/sql.go @@ -0,0 +1,40 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "errors" + "fmt" +) + +// Scan implements sql.Scanner so UUIDs can be read from databases transparently +// Currently, database types that map to string and []byte are supported. Please +// consult database-specific driver documentation for matching types. +func (uuid *UUID) Scan(src interface{}) error { + switch src.(type) { + case string: + // see uuid.Parse for required string format + parsed := Parse(src.(string)) + + if parsed == nil { + return errors.New("Scan: invalid UUID format") + } + + *uuid = parsed + case []byte: + // assumes a simple slice of bytes, just check validity and store + u := UUID(src.([]byte)) + + if u.Variant() == Invalid { + return errors.New("Scan: invalid UUID format") + } + + *uuid = u + default: + return fmt.Errorf("Scan: unable to scan type %T into UUID", src) + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/pborman/uuid/sql_test.go b/Godeps/_workspace/src/github.com/pborman/uuid/sql_test.go new file mode 100644 index 000000000..d643567ee --- /dev/null +++ b/Godeps/_workspace/src/github.com/pborman/uuid/sql_test.go @@ -0,0 +1,53 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "strings" + "testing" +) + +func TestScan(t *testing.T) { + var stringTest string = "f47ac10b-58cc-0372-8567-0e02b2c3d479" + var byteTest []byte = Parse(stringTest) + var badTypeTest int = 6 + var invalidTest string = "f47ac10b-58cc-0372-8567-0e02b2c3d4" + var invalidByteTest []byte = Parse(invalidTest) + + var uuid UUID + err := (&uuid).Scan(stringTest) + if err != nil { + t.Fatal(err) + } + + err = (&uuid).Scan(byteTest) + if err != nil { + t.Fatal(err) + } + + err = (&uuid).Scan(badTypeTest) + if err == nil { + t.Error("int correctly parsed and shouldn't have") + } + if !strings.Contains(err.Error(), "unable to scan type") { + t.Error("attempting to parse an int returned an incorrect error message") + } + + err = (&uuid).Scan(invalidTest) + if err == nil { + t.Error("invalid uuid was parsed without error") + } + if !strings.Contains(err.Error(), "invalid UUID") { + t.Error("attempting to parse an invalid UUID returned an incorrect error message") + } + + err = (&uuid).Scan(invalidByteTest) + if err == nil { + t.Error("invalid byte uuid was parsed without error") + } + if !strings.Contains(err.Error(), "invalid UUID") { + t.Error("attempting to parse an invalid byte UUID returned an incorrect error message") + } +} diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/time.go b/Godeps/_workspace/src/github.com/pborman/uuid/time.go index b9369c200..7ebc9bef1 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/time.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/time.go @@ -40,15 +40,15 @@ func (t Time) UnixTime() (sec, nsec int64) { } // GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and -// adjusts the clock sequence as needed. An error is returned if the current -// time cannot be determined. -func GetTime() (Time, error) { +// clock sequence as well as adjusting the clock sequence as needed. An error +// is returned if the current time cannot be determined. +func GetTime() (Time, uint16, error) { defer mu.Unlock() mu.Lock() return getTime() } -func getTime() (Time, error) { +func getTime() (Time, uint16, error) { t := timeNow() // If we don't have a clock sequence already, set one. @@ -63,7 +63,7 @@ func getTime() (Time, error) { clock_seq = ((clock_seq + 1) & 0x3fff) | 0x8000 } lasttime = now - return Time(now), nil + return Time(now), clock_seq, nil } // ClockSequence returns the current clock sequence, generating one if not diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/util.go b/Godeps/_workspace/src/github.com/pborman/uuid/util.go index de40b102c..de40b102c 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/util.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/util.go diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/uuid.go b/Godeps/_workspace/src/github.com/pborman/uuid/uuid.go index 2920fae63..2920fae63 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/uuid.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/uuid.go diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/uuid_test.go b/Godeps/_workspace/src/github.com/pborman/uuid/uuid_test.go index 417ebeb26..417ebeb26 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/uuid_test.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/uuid_test.go diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/version1.go b/Godeps/_workspace/src/github.com/pborman/uuid/version1.go index 63580044b..0127eacfa 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/version1.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/version1.go @@ -19,7 +19,7 @@ func NewUUID() UUID { SetNodeInterface("") } - now, err := GetTime() + now, seq, err := GetTime() if err != nil { return nil } @@ -34,7 +34,7 @@ func NewUUID() UUID { binary.BigEndian.PutUint32(uuid[0:], time_low) binary.BigEndian.PutUint16(uuid[4:], time_mid) binary.BigEndian.PutUint16(uuid[6:], time_hi) - binary.BigEndian.PutUint16(uuid[8:], clock_seq) + binary.BigEndian.PutUint16(uuid[8:], seq) copy(uuid[10:], nodeID) return uuid diff --git a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/version4.go b/Godeps/_workspace/src/github.com/pborman/uuid/version4.go index b3d4a368d..b3d4a368d 100644 --- a/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid/version4.go +++ b/Godeps/_workspace/src/github.com/pborman/uuid/version4.go diff --git a/cmd/geth/blocktestcmd.go b/cmd/geth/blocktestcmd.go index 4eff82e3d..a667cfd60 100644 --- a/cmd/geth/blocktestcmd.go +++ b/cmd/geth/blocktestcmd.go @@ -22,7 +22,6 @@ import ( "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/tests" @@ -103,7 +102,7 @@ func runBlockTest(ctx *cli.Context) { func runOneBlockTest(ctx *cli.Context, test *tests.BlockTest) (*eth.Ethereum, error) { cfg := utils.MakeEthConfig(ClientIdentifier, Version, ctx) - cfg.NewDB = func(path string) (common.Database, error) { return ethdb.NewMemDatabase() } + cfg.NewDB = func(path string) (ethdb.Database, error) { return ethdb.NewMemDatabase() } cfg.MaxPeers = 0 // disable network cfg.Shh = false // disable whisper cfg.NAT = nil // disable port mapping diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c42045918..c5bc4b66a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger/glog" ) @@ -191,7 +192,7 @@ func hashish(x string) bool { return err != nil } -func closeAll(dbs ...common.Database) { +func closeAll(dbs ...ethdb.Database) { for _, db := range dbs { db.Close() } diff --git a/cmd/geth/js_test.go b/cmd/geth/js_test.go index 67c36dfe7..2fd5a531d 100644 --- a/cmd/geth/js_test.go +++ b/cmd/geth/js_test.go @@ -103,7 +103,7 @@ func testREPL(t *testing.T, config func(*eth.Config)) (string, *testjethre, *eth Name: "test", SolcPath: testSolcPath, PowTest: true, - NewDB: func(path string) (common.Database, error) { return db, nil }, + NewDB: func(path string) (ethdb.Database, error) { return db, nil }, } if config != nil { config(conf) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index aacb588fe..b54d85c22 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -58,6 +58,11 @@ var ( gitCommit string // set via linker flagg nodeNameVersion string app *cli.App + + ExtraDataFlag = cli.StringFlag{ + Name: "extradata", + Usage: "Extra data for the miner", + } ) func init() { @@ -308,6 +313,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.IPCPathFlag, utils.ExecFlag, utils.WhisperEnabledFlag, + utils.DevModeFlag, utils.VMDebugFlag, utils.VMForceJitFlag, utils.VMJitCacheFlag, @@ -330,6 +336,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.GpobaseStepDownFlag, utils.GpobaseStepUpFlag, utils.GpobaseCorrectionFactorFlag, + ExtraDataFlag, } app.Before = func(ctx *cli.Context) error { utils.SetupLogger(ctx) @@ -353,6 +360,14 @@ func main() { } } +// makeExtra resolves extradata for the miner from a flag or returns a default. +func makeExtra(ctx *cli.Context) []byte { + if ctx.GlobalIsSet(ExtraDataFlag.Name) { + return []byte(ctx.GlobalString(ExtraDataFlag.Name)) + } + return makeDefaultExtra() +} + func makeDefaultExtra() []byte { var clientInfo = struct { Version uint @@ -381,7 +396,7 @@ func run(ctx *cli.Context) { } cfg := utils.MakeEthConfig(ClientIdentifier, nodeNameVersion, ctx) - cfg.ExtraData = makeDefaultExtra() + cfg.ExtraData = makeExtra(ctx) ethereum, err := eth.New(cfg) if err != nil { @@ -429,6 +444,8 @@ func console(ctx *cli.Context) { utils.CheckLegalese(ctx.GlobalString(utils.DataDirFlag.Name)) cfg := utils.MakeEthConfig(ClientIdentifier, nodeNameVersion, ctx) + cfg.ExtraData = makeExtra(ctx) + ethereum, err := eth.New(cfg) if err != nil { utils.Fatalf("%v", err) @@ -527,17 +544,16 @@ func blockRecovery(ctx *cli.Context) { var block *types.Block if arg[0] == '#' { - block = core.GetBlockByNumber(blockDb, common.String2Big(arg[1:]).Uint64()) + block = core.GetBlock(blockDb, core.GetCanonicalHash(blockDb, common.String2Big(arg[1:]).Uint64())) } else { - block = core.GetBlockByHash(blockDb, common.HexToHash(arg)) + block = core.GetBlock(blockDb, common.HexToHash(arg)) } if block == nil { glog.Fatalln("block not found. Recovery failed") } - err = core.WriteHead(blockDb, block) - if err != nil { + if err = core.WriteHeadBlockHash(blockDb, block.Hash()); err != nil { glog.Fatalln("block write err", err) } glog.Infof("Recovery succesful. New HEAD %x\n", block.Hash()) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 80805ca22..b45ef0af2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -121,6 +121,10 @@ var ( Name: "genesis", Usage: "Inserts/Overwrites the genesis block (json format)", } + DevModeFlag = cli.BoolFlag{ + Name: "dev", + Usage: "Developer mode. This mode creates a private network and sets several debugging flags", + } IdentityFlag = cli.StringFlag{ Name: "identity", Usage: "Custom node name", @@ -410,7 +414,7 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { glog.V(logger.Error).Infoln("WARNING: No etherbase set and no accounts found as default") } - return ð.Config{ + cfg := ð.Config{ Name: common.MakeName(clientID, version), DataDir: ctx.GlobalString(DataDirFlag.Name), GenesisNonce: ctx.GlobalInt(GenesisNonceFlag.Name), @@ -447,6 +451,33 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { SolcPath: ctx.GlobalString(SolcPathFlag.Name), AutoDAG: ctx.GlobalBool(AutoDAGFlag.Name) || ctx.GlobalBool(MiningEnabledFlag.Name), } + + if ctx.GlobalBool(DevModeFlag.Name) { + if !ctx.GlobalIsSet(VMDebugFlag.Name) { + cfg.VmDebug = true + } + if !ctx.GlobalIsSet(MaxPeersFlag.Name) { + cfg.MaxPeers = 0 + } + if !ctx.GlobalIsSet(GasPriceFlag.Name) { + cfg.GasPrice = new(big.Int) + } + if !ctx.GlobalIsSet(ListenPortFlag.Name) { + cfg.Port = "0" // auto port + } + if !ctx.GlobalIsSet(WhisperEnabledFlag.Name) { + cfg.Shh = true + } + if !ctx.GlobalIsSet(DataDirFlag.Name) { + cfg.DataDir = os.TempDir() + "/ethereum_dev_mode" + } + cfg.PowTest = true + cfg.DevMode = true + + glog.V(logger.Info).Infoln("dev mode enabled") + } + + return cfg } // SetupLogger configures glog from the logging-related command line flags. @@ -477,7 +508,7 @@ func SetupEth(ctx *cli.Context) { } // MakeChain creates a chain manager from set command line flags. -func MakeChain(ctx *cli.Context) (chain *core.ChainManager, chainDb common.Database) { +func MakeChain(ctx *cli.Context) (chain *core.ChainManager, chainDb ethdb.Database) { datadir := ctx.GlobalString(DataDirFlag.Name) cache := ctx.GlobalInt(CacheFlag.Name) diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index 67815a726..aca3a1fc2 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -34,15 +34,10 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" ) -const ( - // flair = "Christian <c@ethdev.com> and Lefteris <lefteris@ethdev.com> (c) 2014-2015" - flair = "" - languageVersion = "0" -) - var ( - versionRegExp = regexp.MustCompile("[0-9]+.[0-9]+.[0-9]+") - params = []string{ + versionRegexp = regexp.MustCompile("[0-9]+\\.[0-9]+\\.[0-9]+") + legacyRegexp = regexp.MustCompile("0\\.(9\\..*|1\\.[01])") + paramsLegacy = []string{ "--binary", // Request to output the contract in binary (hexadecimal). "file", // "--json-abi", // Request to output the contract's JSON ABI interface. @@ -54,6 +49,15 @@ var ( "--add-std", "1", } + paramsNew = []string{ + "--bin", // Request to output the contract in binary (hexadecimal). + "--abi", // Request to output the contract's JSON ABI interface. + "--userdoc", // Request to output the contract's Natspec user documentation. + "--devdoc", // Request to output the contract's Natspec developer documentation. + "--add-std", // include standard lib contracts + "--optimize", // code optimizer switched on + "-o", // output directory + } ) type Contract struct { @@ -66,14 +70,17 @@ type ContractInfo struct { Language string `json:"language"` LanguageVersion string `json:"languageVersion"` CompilerVersion string `json:"compilerVersion"` + CompilerOptions string `json:"compilerOptions"` AbiDefinition interface{} `json:"abiDefinition"` UserDoc interface{} `json:"userDoc"` DeveloperDoc interface{} `json:"developerDoc"` } type Solidity struct { - solcPath string - version string + solcPath string + version string + fullVersion string + legacy bool } func New(solcPath string) (sol *Solidity, err error) { @@ -94,17 +101,22 @@ func New(solcPath string) (sol *Solidity, err error) { return } - version := versionRegExp.FindString(out.String()) + fullVersion := out.String() + version := versionRegexp.FindString(fullVersion) + legacy := legacyRegexp.MatchString(version) + sol = &Solidity{ - solcPath: solcPath, - version: version, + solcPath: solcPath, + version: version, + fullVersion: fullVersion, + legacy: legacy, } glog.V(logger.Info).Infoln(sol.Info()) return } func (sol *Solidity) Info() string { - return fmt.Sprintf("solc v%s\nSolidity Compiler: %s\n%s", sol.version, sol.solcPath, flair) + return fmt.Sprintf("%s\npath: %s", sol.fullVersion, sol.solcPath) } func (sol *Solidity) Version() string { @@ -127,6 +139,15 @@ func (sol *Solidity) Compile(source string) (map[string]*Contract, error) { // Assemble the compiler command, change to the temp folder and capture any errors stderr := new(bytes.Buffer) + var params []string + if sol.legacy { + params = paramsLegacy + } else { + params = paramsNew + params = append(params, wd) + } + compilerOptions := strings.Join(params, " ") + cmd := exec.Command(sol.solcPath, params...) cmd.Dir = wd cmd.Stdin = strings.NewReader(source) @@ -136,7 +157,7 @@ func (sol *Solidity) Compile(source string) (map[string]*Contract, error) { return nil, fmt.Errorf("solc: %v\n%s", err, string(stderr.Bytes())) } // Sanity check that something was actually built - matches, _ := filepath.Glob(wd + "/*.binary") + matches, _ := filepath.Glob(wd + "/*\\.bin*") if len(matches) < 1 { return nil, fmt.Errorf("solc: no build results found") } @@ -148,7 +169,11 @@ func (sol *Solidity) Compile(source string) (map[string]*Contract, error) { // Parse the individual compilation results (code binary, ABI definitions, user and dev docs) var binary []byte - if binary, err = ioutil.ReadFile(filepath.Join(wd, base+".binary")); err != nil { + binext := ".bin" + if sol.legacy { + binext = ".binary" + } + if binary, err = ioutil.ReadFile(filepath.Join(wd, base+binext)); err != nil { return nil, fmt.Errorf("solc: error reading compiler output for code: %v", err) } @@ -178,8 +203,9 @@ func (sol *Solidity) Compile(source string) (map[string]*Contract, error) { Info: ContractInfo{ Source: source, Language: "Solidity", - LanguageVersion: languageVersion, + LanguageVersion: sol.version, CompilerVersion: sol.version, + CompilerOptions: compilerOptions, AbiDefinition: abi, UserDoc: userdoc, DeveloperDoc: devdoc, diff --git a/common/compiler/solidity_test.go b/common/compiler/solidity_test.go index 3303bc15a..258a78f52 100644 --- a/common/compiler/solidity_test.go +++ b/common/compiler/solidity_test.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" ) -const solcVersion = "0.9.23" +const solcVersion = "0.1.1" var ( source = ` @@ -37,18 +37,18 @@ contract test { } } ` - code = "0x605880600c6000396000f3006000357c010000000000000000000000000000000000000000000000000000000090048063c6888fa114602e57005b603d6004803590602001506047565b8060005260206000f35b60006007820290506053565b91905056" - info = `{"source":"\ncontract test {\n /// @notice Will multiply ` + "`a`" + ` by 7.\n function multiply(uint a) returns(uint d) {\n return a * 7;\n }\n}\n","language":"Solidity","languageVersion":"0","compilerVersion":"0.9.23","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}` + code = "0x6060604052606d8060116000396000f30060606040526000357c010000000000000000000000000000000000000000000000000000000090048063c6888fa1146037576035565b005b6046600480359060200150605c565b6040518082815260200191505060405180910390f35b60006007820290506068565b91905056" + info = `{"source":"\ncontract test {\n /// @notice Will multiply ` + "`a`" + ` by 7.\n function multiply(uint a) returns(uint d) {\n return a * 7;\n }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --natspec-user file --natspec-dev file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}` - infohash = common.HexToHash("0xea782f674eb898e477c20e8a7cf11c2c28b09fa68b5278732104f7a101aed255") + infohash = common.HexToHash("0x9f3803735e7f16120c5a140ab3f02121fd3533a9655c69b33a10e78752cc49b0") ) func TestCompiler(t *testing.T) { sol, err := New("") if err != nil { - t.Skip("solc not found: skip") + t.Skipf("solc not found: %v", err) } else if sol.Version() != solcVersion { - t.Skip("WARNING: skipping due to a newer version of solc found (%v, expect %v)", sol.Version(), solcVersion) + t.Skipf("WARNING: a newer version of solc found (%v, expect %v)", sol.Version(), solcVersion) } contracts, err := sol.Compile(source) if err != nil { @@ -83,7 +83,7 @@ func TestCompileError(t *testing.T) { func TestNoCompiler(t *testing.T) { _, err := New("/path/to/solc") if err != nil { - t.Log("solidity quits with error: %v", err) + t.Logf("solidity quits with error: %v", err) } else { t.Errorf("no solc installed, but got no error") } @@ -111,4 +111,4 @@ func TestSaveInfo(t *testing.T) { if cinfohash != infohash { t.Errorf("content hash for info is incorrect. expected %v, got %v", infohash.Hex(), cinfohash.Hex()) } -}
\ No newline at end of file +} diff --git a/common/natspec/natspec_e2e_test.go b/common/natspec/natspec_e2e_test.go index fc8ca6af2..ea28b457e 100644 --- a/common/natspec/natspec_e2e_test.go +++ b/common/natspec/natspec_e2e_test.go @@ -143,7 +143,7 @@ func testEth(t *testing.T) (ethereum *eth.Ethereum, err error) { MaxPeers: 0, PowTest: true, Etherbase: common.HexToAddress(testAddress), - NewDB: func(path string) (common.Database, error) { return db, nil }, + NewDB: func(path string) (ethdb.Database, error) { return db, nil }, }) if err != nil { diff --git a/core/bench_test.go b/core/bench_test.go index baae8a7a5..d05b7d30b 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -144,7 +144,7 @@ func genUncles(i int, gen *BlockGen) { func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Create the database in memory or in a temporary directory. - var db common.Database + var db ethdb.Database if !disk { db, _ = ethdb.NewMemDatabase() } else { diff --git a/core/block_cache.go b/core/block_cache.go deleted file mode 100644 index 0fd711448..000000000 --- a/core/block_cache.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package core - -import ( - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" -) - -// BlockCache implements a caching mechanism specifically for blocks and uses FILO to pop -type BlockCache struct { - size int - - hashes []common.Hash - blocks map[common.Hash]*types.Block - - mu sync.RWMutex -} - -// Creates and returns a `BlockCache` with `size`. If `size` is smaller than 1 it will panic -func NewBlockCache(size int) *BlockCache { - if size < 1 { - panic("block cache size not allowed to be smaller than 1") - } - - bc := &BlockCache{size: size} - bc.Clear() - return bc -} - -func (bc *BlockCache) Clear() { - bc.blocks = make(map[common.Hash]*types.Block) - bc.hashes = nil - -} - -func (bc *BlockCache) Push(block *types.Block) { - bc.mu.Lock() - defer bc.mu.Unlock() - - if len(bc.hashes) == bc.size { - delete(bc.blocks, bc.hashes[0]) - - // XXX There are a few other options on solving this - // 1) use a poller / GC like mechanism to clean up untracked objects - // 2) copy as below - // re-use the slice and remove the reference to bc.hashes[0] - // this will allow the element to be garbage collected. - copy(bc.hashes, bc.hashes[1:]) - } else { - bc.hashes = append(bc.hashes, common.Hash{}) - } - - hash := block.Hash() - bc.blocks[hash] = block - bc.hashes[len(bc.hashes)-1] = hash -} - -func (bc *BlockCache) Delete(hash common.Hash) { - bc.mu.Lock() - defer bc.mu.Unlock() - - if _, ok := bc.blocks[hash]; ok { - delete(bc.blocks, hash) - for i, h := range bc.hashes { - if hash == h { - bc.hashes = bc.hashes[:i+copy(bc.hashes[i:], bc.hashes[i+1:])] - // or ? => bc.hashes = append(bc.hashes[:i], bc.hashes[i+1]...) - - break - } - } - } -} - -func (bc *BlockCache) Get(hash common.Hash) *types.Block { - bc.mu.RLock() - defer bc.mu.RUnlock() - - if block, haz := bc.blocks[hash]; haz { - return block - } - - return nil -} - -func (bc *BlockCache) Has(hash common.Hash) bool { - bc.mu.RLock() - defer bc.mu.RUnlock() - - _, ok := bc.blocks[hash] - return ok -} - -func (bc *BlockCache) Each(cb func(int, *types.Block)) { - bc.mu.Lock() - defer bc.mu.Unlock() - - i := 0 - for _, block := range bc.blocks { - cb(i, block) - i++ - } -} diff --git a/core/block_cache_test.go b/core/block_cache_test.go deleted file mode 100644 index ef826d5bd..000000000 --- a/core/block_cache_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package core - -import ( - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" -) - -func newChain(size int) (chain []*types.Block) { - var parentHash common.Hash - for i := 0; i < size; i++ { - head := &types.Header{ParentHash: parentHash, Number: big.NewInt(int64(i))} - block := types.NewBlock(head, nil, nil, nil) - chain = append(chain, block) - parentHash = block.Hash() - } - return chain -} - -func insertChainCache(cache *BlockCache, chain []*types.Block) { - for _, block := range chain { - cache.Push(block) - } -} - -func TestNewBlockCache(t *testing.T) { - chain := newChain(3) - cache := NewBlockCache(2) - insertChainCache(cache, chain) - - if cache.hashes[0] != chain[1].Hash() { - t.Error("oldest block incorrect") - } -} - -func TestInclusion(t *testing.T) { - chain := newChain(3) - cache := NewBlockCache(3) - insertChainCache(cache, chain) - - for _, block := range chain { - if b := cache.Get(block.Hash()); b == nil { - t.Errorf("getting %x failed", block.Hash()) - } - } -} - -func TestDeletion(t *testing.T) { - chain := newChain(3) - cache := NewBlockCache(3) - insertChainCache(cache, chain) - - cache.Delete(chain[1].Hash()) - - if cache.Has(chain[1].Hash()) { - t.Errorf("expected %x not to be included") - } -} diff --git a/core/block_processor.go b/core/block_processor.go index 1238fda7b..238b2db95 100644 --- a/core/block_processor.go +++ b/core/block_processor.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -41,7 +42,7 @@ const ( ) type BlockProcessor struct { - chainDb common.Database + chainDb ethdb.Database // Mutex for locking the block processor. Blocks can only be handled one at a time mutex sync.Mutex // Canonical block chain @@ -68,7 +69,7 @@ type GasPool interface { SubGas(gas, price *big.Int) error } -func NewBlockProcessor(db common.Database, pow pow.PoW, chainManager *ChainManager, eventMux *event.TypeMux) *BlockProcessor { +func NewBlockProcessor(db ethdb.Database, pow pow.PoW, chainManager *ChainManager, eventMux *event.TypeMux) *BlockProcessor { sm := &BlockProcessor{ chainDb: db, mem: make(map[string]*big.Int), @@ -213,7 +214,7 @@ func (sm *BlockProcessor) processWithParent(block, parent *types.Block) (logs st txs := block.Transactions() // Block validation - if err = ValidateHeader(sm.Pow, header, parent, false, false); err != nil { + if err = ValidateHeader(sm.Pow, header, parent.Header(), false, false); err != nil { return } @@ -337,7 +338,7 @@ func (sm *BlockProcessor) VerifyUncles(statedb *state.StateDB, block, parent *ty return UncleError("uncle[%d](%x)'s parent is not ancestor (%x)", i, hash[:4], uncle.ParentHash[0:4]) } - if err := ValidateHeader(sm.Pow, uncle, ancestors[uncle.ParentHash], true, true); err != nil { + if err := ValidateHeader(sm.Pow, uncle, ancestors[uncle.ParentHash].Header(), true, true); err != nil { return ValidationError(fmt.Sprintf("uncle[%d](%x) header invalid: %v", i, hash[:4], err)) } } @@ -367,52 +368,50 @@ func (sm *BlockProcessor) GetLogs(block *types.Block) (logs state.Logs, err erro } // See YP section 4.3.4. "Block Header Validity" -// Validates a block. Returns an error if the block is invalid. -func ValidateHeader(pow pow.PoW, block *types.Header, parent *types.Block, checkPow, uncle bool) error { - if big.NewInt(int64(len(block.Extra))).Cmp(params.MaximumExtraDataSize) == 1 { - return fmt.Errorf("Block extra data too long (%d)", len(block.Extra)) +// Validates a header. Returns an error if the header is invalid. +func ValidateHeader(pow pow.PoW, header *types.Header, parent *types.Header, checkPow, uncle bool) error { + if big.NewInt(int64(len(header.Extra))).Cmp(params.MaximumExtraDataSize) == 1 { + return fmt.Errorf("Header extra data too long (%d)", len(header.Extra)) } if uncle { - if block.Time.Cmp(common.MaxBig) == 1 { + if header.Time.Cmp(common.MaxBig) == 1 { return BlockTSTooBigErr } } else { - if block.Time.Cmp(big.NewInt(time.Now().Unix())) == 1 { + if header.Time.Cmp(big.NewInt(time.Now().Unix())) == 1 { return BlockFutureErr } } - if block.Time.Cmp(parent.Time()) != 1 { + if header.Time.Cmp(parent.Time) != 1 { return BlockEqualTSErr } - expd := CalcDifficulty(block.Time.Uint64(), parent.Time().Uint64(), parent.Number(), parent.Difficulty()) - if expd.Cmp(block.Difficulty) != 0 { - return fmt.Errorf("Difficulty check failed for block %v, %v", block.Difficulty, expd) + expd := CalcDifficulty(header.Time.Uint64(), parent.Time.Uint64(), parent.Number, parent.Difficulty) + if expd.Cmp(header.Difficulty) != 0 { + return fmt.Errorf("Difficulty check failed for header %v, %v", header.Difficulty, expd) } - var a, b *big.Int - a = parent.GasLimit() - a = a.Sub(a, block.GasLimit) + a := new(big.Int).Set(parent.GasLimit) + a = a.Sub(a, header.GasLimit) a.Abs(a) - b = parent.GasLimit() + b := new(big.Int).Set(parent.GasLimit) b = b.Div(b, params.GasLimitBoundDivisor) - if !(a.Cmp(b) < 0) || (block.GasLimit.Cmp(params.MinGasLimit) == -1) { - return fmt.Errorf("GasLimit check failed for block %v (%v > %v)", block.GasLimit, a, b) + if !(a.Cmp(b) < 0) || (header.GasLimit.Cmp(params.MinGasLimit) == -1) { + return fmt.Errorf("GasLimit check failed for header %v (%v > %v)", header.GasLimit, a, b) } - num := parent.Number() - num.Sub(block.Number, num) + num := new(big.Int).Set(parent.Number) + num.Sub(header.Number, num) if num.Cmp(big.NewInt(1)) != 0 { return BlockNumberErr } if checkPow { - // Verify the nonce of the block. Return an error if it's not valid - if !pow.Verify(types.NewBlockWithHeader(block)) { - return ValidationError("Block's nonce is invalid (= %x)", block.Nonce) + // Verify the nonce of the header. Return an error if it's not valid + if !pow.Verify(types.NewBlockWithHeader(header)) { + return ValidationError("Header's nonce is invalid (= %x)", header.Nonce) } } - return nil } diff --git a/core/block_processor_test.go b/core/block_processor_test.go index e0b2d3313..538cf4ee5 100644 --- a/core/block_processor_test.go +++ b/core/block_processor_test.go @@ -48,13 +48,13 @@ func TestNumber(t *testing.T) { statedb := state.New(chain.Genesis().Root(), chain.chainDb) header := makeHeader(chain.Genesis(), statedb) header.Number = big.NewInt(3) - err := ValidateHeader(pow, header, chain.Genesis(), false, false) + err := ValidateHeader(pow, header, chain.Genesis().Header(), false, false) if err != BlockNumberErr { t.Errorf("expected block number error, got %q", err) } header = makeHeader(chain.Genesis(), statedb) - err = ValidateHeader(pow, header, chain.Genesis(), false, false) + err = ValidateHeader(pow, header, chain.Genesis().Header(), false, false) if err == BlockNumberErr { t.Errorf("didn't expect block number error") } diff --git a/core/chain_makers.go b/core/chain_makers.go index b009e0c28..d3b7c42b6 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/pow" ) @@ -142,7 +143,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // Blocks created by GenerateChain do not contain valid proof of work // values. Inserting them into ChainManager requires use of FakePow or // a similar non-validating proof of work implementation. -func GenerateChain(parent *types.Block, db common.Database, n int, gen func(int, *BlockGen)) []*types.Block { +func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, *BlockGen)) []*types.Block { statedb := state.New(parent.Root(), db) blocks := make(types.Blocks, n) genblock := func(i int, h *types.Header) *types.Block { @@ -158,7 +159,6 @@ func GenerateChain(parent *types.Block, db common.Database, n int, gen func(int, for i := 0; i < n; i++ { header := makeHeader(parent, statedb) block := genblock(i, header) - block.Td = CalcTD(block, parent) blocks[i] = block parent = block } @@ -186,7 +186,7 @@ func makeHeader(parent *types.Block, state *state.StateDB) *types.Header { // newCanonical creates a new deterministic canonical chain by running // InsertChain on the result of makeChain. -func newCanonical(n int, db common.Database) (*BlockProcessor, error) { +func newCanonical(n int, db ethdb.Database) (*BlockProcessor, error) { evmux := &event.TypeMux{} WriteTestNetGenesisBlock(db, 0) @@ -202,7 +202,7 @@ func newCanonical(n int, db common.Database) (*BlockProcessor, error) { return bman, err } -func makeChain(parent *types.Block, n int, db common.Database, seed int) []*types.Block { +func makeChain(parent *types.Block, n int, db ethdb.Database, seed int) []*types.Block { return GenerateChain(parent, db, n, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)}) }) diff --git a/core/chain_manager.go b/core/chain_manager.go index c8127951e..1218b1a6e 100644 --- a/core/chain_manager.go +++ b/core/chain_manager.go @@ -30,11 +30,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/pow" + "github.com/ethereum/go-ethereum/rlp" "github.com/hashicorp/golang-lru" ) @@ -48,6 +50,9 @@ var ( ) const ( + headerCacheLimit = 512 + bodyCacheLimit = 256 + tdCacheLimit = 1024 blockCacheLimit = 256 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 @@ -56,7 +61,7 @@ const ( type ChainManager struct { //eth EthManager - chainDb common.Database + chainDb ethdb.Database processor types.BlockProcessor eventMux *event.TypeMux genesisBlock *types.Block @@ -68,10 +73,13 @@ type ChainManager struct { checkpoint int // checkpoint counts towards the new checkpoint td *big.Int currentBlock *types.Block - lastBlockHash common.Hash currentGasLimit *big.Int - cache *lru.Cache // cache is the LRU caching + headerCache *lru.Cache // Cache for the most recent block headers + bodyCache *lru.Cache // Cache for the most recent block bodies + bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format + tdCache *lru.Cache // Cache for the most recent block total difficulties + blockCache *lru.Cache // Cache for the most recent entire blocks futureBlocks *lru.Cache // future blocks are blocks added for later processing quit chan struct{} @@ -83,14 +91,25 @@ type ChainManager struct { pow pow.PoW } -func NewChainManager(chainDb common.Database, pow pow.PoW, mux *event.TypeMux) (*ChainManager, error) { - cache, _ := lru.New(blockCacheLimit) +func NewChainManager(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*ChainManager, error) { + headerCache, _ := lru.New(headerCacheLimit) + bodyCache, _ := lru.New(bodyCacheLimit) + bodyRLPCache, _ := lru.New(bodyCacheLimit) + tdCache, _ := lru.New(tdCacheLimit) + blockCache, _ := lru.New(blockCacheLimit) + futureBlocks, _ := lru.New(maxFutureBlocks) + bc := &ChainManager{ - chainDb: chainDb, - eventMux: mux, - quit: make(chan struct{}), - cache: cache, - pow: pow, + chainDb: chainDb, + eventMux: mux, + quit: make(chan struct{}), + headerCache: headerCache, + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + tdCache: tdCache, + blockCache: blockCache, + futureBlocks: futureBlocks, + pow: pow, } bc.genesisBlock = bc.GetBlockByNumber(0) @@ -105,11 +124,9 @@ func NewChainManager(chainDb common.Database, pow pow.PoW, mux *event.TypeMux) ( } glog.V(logger.Info).Infoln("WARNING: Wrote default ethereum genesis block") } - if err := bc.setLastState(); err != nil { return nil, err } - // Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain for hash, _ := range BadHashes { if block := bc.GetBlock(hash); block != nil { @@ -123,14 +140,8 @@ func NewChainManager(chainDb common.Database, pow pow.PoW, mux *event.TypeMux) ( glog.V(logger.Error).Infoln("Chain reorg was successfull. Resuming normal operation") } } - // Take ownership of this particular state - - bc.futureBlocks, _ = lru.New(maxFutureBlocks) - bc.makeCache() - go bc.update() - return bc, nil } @@ -139,14 +150,16 @@ func (bc *ChainManager) SetHead(head *types.Block) { defer bc.mu.Unlock() for block := bc.currentBlock; block != nil && block.Hash() != head.Hash(); block = bc.GetBlock(block.ParentHash()) { - bc.removeBlock(block) + DeleteBlock(bc.chainDb, block.Hash()) } + bc.headerCache.Purge() + bc.bodyCache.Purge() + bc.bodyRLPCache.Purge() + bc.blockCache.Purge() + bc.futureBlocks.Purge() - bc.cache, _ = lru.New(blockCacheLimit) bc.currentBlock = head - bc.makeCache() - - bc.setTotalDifficulty(head.Td) + bc.setTotalDifficulty(bc.GetTd(head.Hash())) bc.insert(head) bc.setLastState() } @@ -169,7 +182,7 @@ func (self *ChainManager) LastBlockHash() common.Hash { self.mu.RLock() defer self.mu.RUnlock() - return self.lastBlockHash + return self.currentBlock.Hash() } func (self *ChainManager) CurrentBlock() *types.Block { @@ -199,13 +212,13 @@ func (bc *ChainManager) recover() bool { if len(data) != 0 { block := bc.GetBlock(common.BytesToHash(data)) if block != nil { - err := bc.chainDb.Put([]byte("LastBlock"), block.Hash().Bytes()) - if err != nil { - glog.Fatalln("db write err:", err) + if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil { + glog.Fatalf("failed to write database head number: %v", err) + } + if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil { + glog.Fatalf("failed to write database head hash: %v", err) } - bc.currentBlock = block - bc.lastBlockHash = block.Hash() return true } } @@ -213,14 +226,13 @@ func (bc *ChainManager) recover() bool { } func (bc *ChainManager) setLastState() error { - data, _ := bc.chainDb.Get([]byte("LastBlock")) - if len(data) != 0 { - block := bc.GetBlock(common.BytesToHash(data)) + head := GetHeadBlockHash(bc.chainDb) + if head != (common.Hash{}) { + block := bc.GetBlock(head) if block != nil { bc.currentBlock = block - bc.lastBlockHash = block.Hash() } else { - glog.Infof("LastBlock (%x) not found. Recovering...\n", data) + glog.Infof("LastBlock (%x) not found. Recovering...\n", head) if bc.recover() { glog.Infof("Recover successful") } else { @@ -230,7 +242,7 @@ func (bc *ChainManager) setLastState() error { } else { bc.Reset() } - bc.td = bc.currentBlock.Td + bc.td = bc.GetTd(bc.currentBlock.Hash()) bc.currentGasLimit = CalcGasLimit(bc.currentBlock) if glog.V(logger.Info) { @@ -240,63 +252,37 @@ func (bc *ChainManager) setLastState() error { return nil } -func (bc *ChainManager) makeCache() { - bc.cache, _ = lru.New(blockCacheLimit) - // load in last `blockCacheLimit` - 1 blocks. Last block is the current. - bc.cache.Add(bc.genesisBlock.Hash(), bc.genesisBlock) - for _, block := range bc.GetBlocksFromHash(bc.currentBlock.Hash(), blockCacheLimit) { - bc.cache.Add(block.Hash(), block) - } -} - +// Reset purges the entire blockchain, restoring it to its genesis state. func (bc *ChainManager) Reset() { - bc.mu.Lock() - defer bc.mu.Unlock() - - for block := bc.currentBlock; block != nil; block = bc.GetBlock(block.ParentHash()) { - bc.removeBlock(block) - } - - bc.cache, _ = lru.New(blockCacheLimit) - - // Prepare the genesis block - err := WriteBlock(bc.chainDb, bc.genesisBlock) - if err != nil { - glog.Fatalln("db err:", err) - } - - bc.insert(bc.genesisBlock) - bc.currentBlock = bc.genesisBlock - bc.makeCache() - - bc.setTotalDifficulty(common.Big("0")) + bc.ResetWithGenesisBlock(bc.genesisBlock) } -func (bc *ChainManager) removeBlock(block *types.Block) { - bc.chainDb.Delete(append(blockHashPre, block.Hash().Bytes()...)) -} - -func (bc *ChainManager) ResetWithGenesisBlock(gb *types.Block) { +// ResetWithGenesisBlock purges the entire blockchain, restoring it to the +// specified genesis state. +func (bc *ChainManager) ResetWithGenesisBlock(genesis *types.Block) { bc.mu.Lock() defer bc.mu.Unlock() + // Dump the entire block chain and purge the caches for block := bc.currentBlock; block != nil; block = bc.GetBlock(block.ParentHash()) { - bc.removeBlock(block) + DeleteBlock(bc.chainDb, block.Hash()) } + bc.headerCache.Purge() + bc.bodyCache.Purge() + bc.bodyRLPCache.Purge() + bc.blockCache.Purge() + bc.futureBlocks.Purge() - // Prepare the genesis block - gb.Td = gb.Difficulty() - bc.genesisBlock = gb - - err := WriteBlock(bc.chainDb, bc.genesisBlock) - if err != nil { - glog.Fatalln("db err:", err) + // Prepare the genesis block and reinitialize the chain + if err := WriteTd(bc.chainDb, genesis.Hash(), genesis.Difficulty()); err != nil { + glog.Fatalf("failed to write genesis block TD: %v", err) + } + if err := WriteBlock(bc.chainDb, genesis); err != nil { + glog.Fatalf("failed to write genesis block: %v", err) } - bc.insert(bc.genesisBlock) bc.currentBlock = bc.genesisBlock - bc.makeCache() - bc.td = gb.Difficulty() + bc.setTotalDifficulty(genesis.Difficulty()) } // Export writes the active chain to the given writer. @@ -335,23 +321,23 @@ func (self *ChainManager) ExportN(w io.Writer, first uint64, last uint64) error // insert injects a block into the current chain block chain. Note, this function // assumes that the `mu` mutex is held! func (bc *ChainManager) insert(block *types.Block) { - err := WriteHead(bc.chainDb, block) - if err != nil { - glog.Fatal("db write fail:", err) + // Add the block to the canonical chain number scheme and mark as the head + if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil { + glog.Fatalf("failed to insert block number: %v", err) } - + if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil { + glog.Fatalf("failed to insert block number: %v", err) + } + // Add a new restore point if we reached some limit bc.checkpoint++ if bc.checkpoint > checkpointLimit { - err = bc.chainDb.Put([]byte("checkpoint"), block.Hash().Bytes()) - if err != nil { - glog.Fatal("db write fail:", err) + if err := bc.chainDb.Put([]byte("checkpoint"), block.Hash().Bytes()); err != nil { + glog.Fatalf("failed to create checkpoint: %v", err) } - bc.checkpoint = 0 } - + // Update the internal internal state with the head block bc.currentBlock = block - bc.lastBlockHash = block.Hash() } // Accessors @@ -359,61 +345,141 @@ func (bc *ChainManager) Genesis() *types.Block { return bc.genesisBlock } -// Block fetching methods -func (bc *ChainManager) HasBlock(hash common.Hash) bool { - if bc.cache.Contains(hash) { - return true +// HasHeader checks if a block header is present in the database or not, caching +// it if present. +func (bc *ChainManager) HasHeader(hash common.Hash) bool { + return bc.GetHeader(hash) != nil +} + +// GetHeader retrieves a block header from the database by hash, caching it if +// found. +func (self *ChainManager) GetHeader(hash common.Hash) *types.Header { + // Short circuit if the header's already in the cache, retrieve otherwise + if header, ok := self.headerCache.Get(hash); ok { + return header.(*types.Header) + } + header := GetHeader(self.chainDb, hash) + if header == nil { + return nil } + // Cache the found header for next time and return + self.headerCache.Add(header.Hash(), header) + return header +} - data, _ := bc.chainDb.Get(append(blockHashPre, hash[:]...)) - return len(data) != 0 +// GetHeaderByNumber retrieves a block header from the database by number, +// caching it (associated with its hash) if found. +func (self *ChainManager) GetHeaderByNumber(number uint64) *types.Header { + hash := GetCanonicalHash(self.chainDb, number) + if hash == (common.Hash{}) { + return nil + } + return self.GetHeader(hash) } -func (self *ChainManager) GetBlockHashesFromHash(hash common.Hash, max uint64) (chain []common.Hash) { - block := self.GetBlock(hash) - if block == nil { - return +// GetBody retrieves a block body (transactions and uncles) from the database by +// hash, caching it if found. +func (self *ChainManager) GetBody(hash common.Hash) *types.Body { + // Short circuit if the body's already in the cache, retrieve otherwise + if cached, ok := self.bodyCache.Get(hash); ok { + body := cached.(*types.Body) + return body } - // XXX Could be optimised by using a different database which only holds hashes (i.e., linked list) - for i := uint64(0); i < max; i++ { - block = self.GetBlock(block.ParentHash()) - if block == nil { - break - } + body := GetBody(self.chainDb, hash) + if body == nil { + return nil + } + // Cache the found body for next time and return + self.bodyCache.Add(hash, body) + return body +} - chain = append(chain, block.Hash()) - if block.Number().Cmp(common.Big0) <= 0 { - break - } +// GetBodyRLP retrieves a block body in RLP encoding from the database by hash, +// caching it if found. +func (self *ChainManager) GetBodyRLP(hash common.Hash) rlp.RawValue { + // Short circuit if the body's already in the cache, retrieve otherwise + if cached, ok := self.bodyRLPCache.Get(hash); ok { + return cached.(rlp.RawValue) } + body := GetBodyRLP(self.chainDb, hash) + if len(body) == 0 { + return nil + } + // Cache the found body for next time and return + self.bodyRLPCache.Add(hash, body) + return body +} - return +// GetTd retrieves a block's total difficulty in the canonical chain from the +// database by hash, caching it if found. +func (self *ChainManager) GetTd(hash common.Hash) *big.Int { + // Short circuit if the td's already in the cache, retrieve otherwise + if cached, ok := self.tdCache.Get(hash); ok { + return cached.(*big.Int) + } + td := GetTd(self.chainDb, hash) + if td == nil { + return nil + } + // Cache the found body for next time and return + self.tdCache.Add(hash, td) + return td +} + +// HasBlock checks if a block is fully present in the database or not, caching +// it if present. +func (bc *ChainManager) HasBlock(hash common.Hash) bool { + return bc.GetBlock(hash) != nil } +// GetBlock retrieves a block from the database by hash, caching it if found. func (self *ChainManager) GetBlock(hash common.Hash) *types.Block { - if block, ok := self.cache.Get(hash); ok { + // Short circuit if the block's already in the cache, retrieve otherwise + if block, ok := self.blockCache.Get(hash); ok { return block.(*types.Block) } - - block := GetBlockByHash(self.chainDb, hash) + block := GetBlock(self.chainDb, hash) if block == nil { return nil } - - // Add the block to the cache - self.cache.Add(hash, (*types.Block)(block)) - - return (*types.Block)(block) + // Cache the found block for next time and return + self.blockCache.Add(block.Hash(), block) + return block } -func (self *ChainManager) GetBlockByNumber(num uint64) *types.Block { - self.mu.RLock() - defer self.mu.RUnlock() - - return self.getBlockByNumber(num) +// GetBlockByNumber retrieves a block from the database by number, caching it +// (associated with its hash) if found. +func (self *ChainManager) GetBlockByNumber(number uint64) *types.Block { + hash := GetCanonicalHash(self.chainDb, number) + if hash == (common.Hash{}) { + return nil + } + return self.GetBlock(hash) +} +// GetBlockHashesFromHash retrieves a number of block hashes starting at a given +// hash, fetching towards the genesis block. +func (self *ChainManager) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash { + // Get the origin header from which to fetch + header := self.GetHeader(hash) + if header == nil { + return nil + } + // Iterate the headers until enough is collected or the genesis reached + chain := make([]common.Hash, 0, max) + for i := uint64(0); i < max; i++ { + if header = self.GetHeader(header.ParentHash); header == nil { + break + } + chain = append(chain, header.Hash()) + if header.Number.Cmp(common.Big0) == 0 { + break + } + } + return chain } +// [deprecated by eth/62] // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. func (self *ChainManager) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { for i := 0; i < n; i++ { @@ -427,11 +493,6 @@ func (self *ChainManager) GetBlocksFromHash(hash common.Hash, n int) (blocks []* return } -// non blocking version -func (self *ChainManager) getBlockByNumber(num uint64) *types.Block { - return GetBlockByNumber(self.chainDb, num) -} - func (self *ChainManager) GetUnclesInChain(block *types.Block, length int) (uncles []*types.Header) { for i := 0; block != nil && i < length; i++ { uncles = append(uncles, block.Uncles()...) @@ -487,15 +548,25 @@ const ( SideStatTy ) -// WriteBlock writes the block to the chain (or pending queue) -func (self *ChainManager) WriteBlock(block *types.Block, queued bool) (status writeStatus, err error) { +// WriteBlock writes the block to the chain. +func (self *ChainManager) WriteBlock(block *types.Block) (status writeStatus, err error) { self.wg.Add(1) defer self.wg.Done() + // Calculate the total difficulty of the block + ptd := self.GetTd(block.ParentHash()) + if ptd == nil { + return NonStatTy, ParentError(block.ParentHash()) + } + td := new(big.Int).Add(block.Difficulty(), ptd) + + self.mu.RLock() cblock := self.currentBlock + self.mu.RUnlock() + // Compare the TD of the last known block in the canonical chain to make sure it's greater. // At this point it's possible that a different chain (fork) becomes the new canonical chain. - if block.Td.Cmp(self.Td()) > 0 { + if td.Cmp(self.Td()) > 0 { // chain fork if block.ParentHash() != cblock.Hash() { // during split we merge two different chains and create the new canonical chain @@ -503,12 +574,10 @@ func (self *ChainManager) WriteBlock(block *types.Block, queued bool) (status wr if err != nil { return NonStatTy, err } - status = SplitStatTy } - self.mu.Lock() - self.setTotalDifficulty(block.Td) + self.setTotalDifficulty(td) self.insert(block) self.mu.Unlock() @@ -517,9 +586,11 @@ func (self *ChainManager) WriteBlock(block *types.Block, queued bool) (status wr status = SideStatTy } - err = WriteBlock(self.chainDb, block) - if err != nil { - glog.Fatalln("db err:", err) + if err := WriteTd(self.chainDb, block.Hash(), td); err != nil { + glog.Fatalf("failed to write block total difficulty: %v", err) + } + if err := WriteBlock(self.chainDb, block); err != nil { + glog.Fatalf("filed to write block contents: %v", err) } // Delete from future blocks self.futureBlocks.Remove(block.Hash()) @@ -578,11 +649,6 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) { blockErr(block, err) return i, err } - - // Setting block.Td regardless of error (known for example) prevents errors down the line - // in the protocol handler - block.Td = new(big.Int).Set(CalcTD(block, self.GetBlock(block.ParentHash()))) - // Call in to the block processor and check for errors. It's likely that if one block fails // all others will fail too (unless a known block is returned). logs, receipts, err := self.processor.Process(block) @@ -622,7 +688,7 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) { txcount += len(block.Transactions()) // write the block to the chain and get the status - status, err := self.WriteBlock(block, true) + status, err := self.WriteBlock(block) if err != nil { return i, err } @@ -755,12 +821,11 @@ out: case ChainEvent: // We need some control over the mining operation. Acquiring locks and waiting for the miner to create new block takes too long // and in most cases isn't even necessary. - if self.lastBlockHash == event.Hash { + if self.currentBlock.Hash() == event.Hash { self.currentGasLimit = CalcGasLimit(event.Block) self.eventMux.Post(ChainHeadEvent{event.Block}) } } - self.eventMux.Post(event) } } diff --git a/core/chain_manager_test.go b/core/chain_manager_test.go index 002dcbe44..67ca41f00 100644 --- a/core/chain_manager_test.go +++ b/core/chain_manager_test.go @@ -46,7 +46,7 @@ func thePow() pow.PoW { return pow } -func theChainManager(db common.Database, t *testing.T) *ChainManager { +func theChainManager(db ethdb.Database, t *testing.T) *ChainManager { var eventMux event.TypeMux WriteTestNetGenesisBlock(db, 0) chainMan, err := NewChainManager(db, thePow(), &eventMux) @@ -77,6 +77,7 @@ func testFork(t *testing.T, bman *BlockProcessor, i, N int, f func(td1, td2 *big bi1 := bman.bc.GetBlockByNumber(uint64(i)).Hash() bi2 := bman2.bc.GetBlockByNumber(uint64(i)).Hash() if bi1 != bi2 { + fmt.Printf("%+v\n%+v\n\n", bi1, bi2) t.Fatal("chains do not have the same hash at height", i) } bman2.bc.SetProcessor(bman2) @@ -110,7 +111,6 @@ func printChain(bc *ChainManager) { // process blocks against a chain func testChain(chainB types.Blocks, bman *BlockProcessor) (*big.Int, error) { - td := new(big.Int) for _, block := range chainB { _, _, err := bman.bc.processor.Process(block) if err != nil { @@ -119,17 +119,12 @@ func testChain(chainB types.Blocks, bman *BlockProcessor) (*big.Int, error) { } return nil, err } - parent := bman.bc.GetBlock(block.ParentHash()) - block.Td = CalcTD(block, parent) - td = block.Td - bman.bc.mu.Lock() - { - WriteBlock(bman.bc.chainDb, block) - } + WriteTd(bman.bc.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), bman.bc.GetTd(block.ParentHash()))) + WriteBlock(bman.bc.chainDb, block) bman.bc.mu.Unlock() } - return td, nil + return bman.bc.GetTd(chainB[len(chainB)-1].Hash()), nil } func loadChain(fn string, t *testing.T) (types.Blocks, error) { @@ -385,10 +380,14 @@ func makeChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block return chain } -func chm(genesis *types.Block, db common.Database) *ChainManager { +func chm(genesis *types.Block, db ethdb.Database) *ChainManager { var eventMux event.TypeMux bc := &ChainManager{chainDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}} - bc.cache, _ = lru.New(100) + bc.headerCache, _ = lru.New(100) + bc.bodyCache, _ = lru.New(100) + bc.bodyRLPCache, _ = lru.New(100) + bc.tdCache, _ = lru.New(100) + bc.blockCache, _ = lru.New(100) bc.futureBlocks, _ = lru.New(100) bc.processor = bproc{} bc.ResetWithGenesisBlock(genesis) diff --git a/core/chain_util.go b/core/chain_util.go index 84b462ce3..33d94cebd 100644 --- a/core/chain_util.go +++ b/core/chain_util.go @@ -19,10 +19,10 @@ package core import ( "bytes" "math/big" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/params" @@ -30,9 +30,18 @@ import ( ) var ( - blockHashPre = []byte("block-hash-") - blockNumPre = []byte("block-num-") + headHeaderKey = []byte("LastHeader") + headBlockKey = []byte("LastBlock") + + blockPrefix = []byte("block-") + blockNumPrefix = []byte("block-num-") + + headerSuffix = []byte("-header") + bodySuffix = []byte("-body") + tdSuffix = []byte("-td") + ExpDiffPeriod = big.NewInt(100000) + blockHashPre = []byte("block-hash-") // [deprecated by eth/63] ) // CalcDifficulty is the difficulty adjustment algorithm. It returns @@ -69,16 +78,6 @@ func CalcDifficulty(time, parentTime uint64, parentNumber, parentDiff *big.Int) return diff } -// CalcTD computes the total difficulty of block. -func CalcTD(block, parent *types.Block) *big.Int { - if parent == nil { - return block.Difficulty() - } - d := block.Difficulty() - d.Add(d, parent.Td) - return d -} - // CalcGasLimit computes the gas limit of the next block after parent. // The result may be modified by the caller. // This is miner strategy, not consensus protocol. @@ -112,68 +111,238 @@ func CalcGasLimit(parent *types.Block) *big.Int { return gl } -// GetBlockByHash returns the block corresponding to the hash or nil if not found -func GetBlockByHash(db common.Database, hash common.Hash) *types.Block { - data, _ := db.Get(append(blockHashPre, hash[:]...)) +// GetCanonicalHash retrieves a hash assigned to a canonical block number. +func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { + data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// GetHeadHeaderHash retrieves the hash of the current canonical head block's +// header. The difference between this and GetHeadBlockHash is that whereas the +// last block hash is only updated upon a full block import, the last header +// hash is updated already at header import, allowing head tracking for the +// fast synchronization mechanism. +func GetHeadHeaderHash(db ethdb.Database) common.Hash { + data, _ := db.Get(headHeaderKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// GetHeadBlockHash retrieves the hash of the current canonical head block. +func GetHeadBlockHash(db ethdb.Database) common.Hash { + data, _ := db.Get(headBlockKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil +// if the header's not found. +func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { + data, _ := db.Get(append(append(blockPrefix, hash[:]...), headerSuffix...)) + return data +} + +// GetHeader retrieves the block header corresponding to the hash, nil if none +// found. +func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { + data := GetHeaderRLP(db, hash) if len(data) == 0 { return nil } - var block types.StorageBlock - if err := rlp.Decode(bytes.NewReader(data), &block); err != nil { - glog.V(logger.Error).Infof("invalid block RLP for hash %x: %v", hash, err) + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + glog.V(logger.Error).Infof("invalid block header RLP for hash %x: %v", hash, err) return nil } - return (*types.Block)(&block) + return header +} + +// GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. +func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { + data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...)) + return data } -// GetBlockByHash returns the canonical block by number or nil if not found -func GetBlockByNumber(db common.Database, number uint64) *types.Block { - key, _ := db.Get(append(blockNumPre, big.NewInt(int64(number)).Bytes()...)) - if len(key) == 0 { +// GetBody retrieves the block body (transactons, uncles) corresponding to the +// hash, nil if none found. +func GetBody(db ethdb.Database, hash common.Hash) *types.Body { + data := GetBodyRLP(db, hash) + if len(data) == 0 { + return nil + } + body := new(types.Body) + if err := rlp.Decode(bytes.NewReader(data), body); err != nil { + glog.V(logger.Error).Infof("invalid block body RLP for hash %x: %v", hash, err) return nil } + return body +} - return GetBlockByHash(db, common.BytesToHash(key)) +// GetTd retrieves a block's total difficulty corresponding to the hash, nil if +// none found. +func GetTd(db ethdb.Database, hash common.Hash) *big.Int { + data, _ := db.Get(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) + if len(data) == 0 { + return nil + } + td := new(big.Int) + if err := rlp.Decode(bytes.NewReader(data), td); err != nil { + glog.V(logger.Error).Infof("invalid block total difficulty RLP for hash %x: %v", hash, err) + return nil + } + return td } -// WriteCanonNumber writes the canonical hash for the given block -func WriteCanonNumber(db common.Database, block *types.Block) error { - key := append(blockNumPre, block.Number().Bytes()...) - err := db.Put(key, block.Hash().Bytes()) - if err != nil { +// GetBlock retrieves an entire block corresponding to the hash, assembling it +// back from the stored header and body. +func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { + // Retrieve the block header and body contents + header := GetHeader(db, hash) + if header == nil { + return nil + } + body := GetBody(db, hash) + if body == nil { + return nil + } + // Reassemble the block and return + return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles) +} + +// WriteCanonicalHash stores the canonical hash for the given block number. +func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { + key := append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...) + if err := db.Put(key, hash.Bytes()); err != nil { + glog.Fatalf("failed to store number to hash mapping into database: %v", err) return err } return nil } -// WriteHead force writes the current head -func WriteHead(db common.Database, block *types.Block) error { - err := WriteCanonNumber(db, block) - if err != nil { +// WriteHeadHeaderHash stores the head header's hash. +func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { + if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { + glog.Fatalf("failed to store last header's hash into database: %v", err) return err } - err = db.Put([]byte("LastBlock"), block.Hash().Bytes()) - if err != nil { + return nil +} + +// WriteHeadBlockHash stores the head block's hash. +func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { + if err := db.Put(headBlockKey, hash.Bytes()); err != nil { + glog.Fatalf("failed to store last block's hash into database: %v", err) return err } return nil } -// WriteBlock writes a block to the database -func WriteBlock(db common.Database, block *types.Block) error { - tstart := time.Now() +// WriteHeader serializes a block header into the database. +func WriteHeader(db ethdb.Database, header *types.Header) error { + data, err := rlp.EncodeToBytes(header) + if err != nil { + return err + } + key := append(append(blockPrefix, header.Hash().Bytes()...), headerSuffix...) + if err := db.Put(key, data); err != nil { + glog.Fatalf("failed to store header into database: %v", err) + return err + } + glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, header.Hash().Bytes()[:4]) + return nil +} - enc, _ := rlp.EncodeToBytes((*types.StorageBlock)(block)) - key := append(blockHashPre, block.Hash().Bytes()...) - err := db.Put(key, enc) +// WriteBody serializes the body of a block into the database. +func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error { + data, err := rlp.EncodeToBytes(body) if err != nil { - glog.Fatal("db write fail:", err) return err } + key := append(append(blockPrefix, hash.Bytes()...), bodySuffix...) + if err := db.Put(key, data); err != nil { + glog.Fatalf("failed to store block body into database: %v", err) + return err + } + glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4]) + return nil +} - if glog.V(logger.Debug) { - glog.Infof("wrote block #%v %s. Took %v\n", block.Number(), common.PP(block.Hash().Bytes()), time.Since(tstart)) +// WriteTd serializes the total difficulty of a block into the database. +func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { + data, err := rlp.EncodeToBytes(td) + if err != nil { + return err } + key := append(append(blockPrefix, hash.Bytes()...), tdSuffix...) + if err := db.Put(key, data); err != nil { + glog.Fatalf("failed to store block total difficulty into database: %v", err) + return err + } + glog.V(logger.Debug).Infof("stored block total difficulty [%x…]: %v", hash.Bytes()[:4], td) + return nil +} +// WriteBlock serializes a block into the database, header and body separately. +func WriteBlock(db ethdb.Database, block *types.Block) error { + // Store the body first to retain database consistency + if err := WriteBody(db, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil { + return err + } + // Store the header too, signaling full block ownership + if err := WriteHeader(db, block.Header()); err != nil { + return err + } return nil } + +// DeleteCanonicalHash removes the number to hash canonical mapping. +func DeleteCanonicalHash(db ethdb.Database, number uint64) { + db.Delete(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) +} + +// DeleteHeader removes all block header data associated with a hash. +func DeleteHeader(db ethdb.Database, hash common.Hash) { + db.Delete(append(append(blockPrefix, hash.Bytes()...), headerSuffix...)) +} + +// DeleteBody removes all block body data associated with a hash. +func DeleteBody(db ethdb.Database, hash common.Hash) { + db.Delete(append(append(blockPrefix, hash.Bytes()...), bodySuffix...)) +} + +// DeleteTd removes all block total difficulty data associated with a hash. +func DeleteTd(db ethdb.Database, hash common.Hash) { + db.Delete(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +} + +// DeleteBlock removes all block data associated with a hash. +func DeleteBlock(db ethdb.Database, hash common.Hash) { + DeleteHeader(db, hash) + DeleteBody(db, hash) + DeleteTd(db, hash) +} + +// [deprecated by eth/63] +// GetBlockByHashOld returns the old combined block corresponding to the hash +// or nil if not found. This method is only used by the upgrade mechanism to +// access the old combined block representation. It will be dropped after the +// network transitions to eth/63. +func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block { + data, _ := db.Get(append(blockHashPre, hash[:]...)) + if len(data) == 0 { + return nil + } + var block types.StorageBlock + if err := rlp.Decode(bytes.NewReader(data), &block); err != nil { + glog.V(logger.Error).Infof("invalid block RLP for hash %x: %v", hash, err) + return nil + } + return (*types.Block)(&block) +} diff --git a/core/chain_util_test.go b/core/chain_util_test.go index 4bbe81194..3f0446715 100644 --- a/core/chain_util_test.go +++ b/core/chain_util_test.go @@ -23,6 +23,10 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" ) type diffTest struct { @@ -75,3 +79,242 @@ func TestDifficulty(t *testing.T) { } } } + +// Tests block header storage and retrieval operations. +func TestHeaderStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + + // Create a test header to move around the database and make sure it's really new + header := &types.Header{Extra: []byte("test header")} + if entry := GetHeader(db, header.Hash()); entry != nil { + t.Fatalf("Non existent header returned: %v", entry) + } + // Write and verify the header in the database + if err := WriteHeader(db, header); err != nil { + t.Fatalf("Failed to write header into database: %v", err) + } + if entry := GetHeader(db, header.Hash()); entry == nil { + t.Fatalf("Stored header not found") + } else if entry.Hash() != header.Hash() { + t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, header) + } + if entry := GetHeaderRLP(db, header.Hash()); entry == nil { + t.Fatalf("Stored header RLP not found") + } else { + hasher := sha3.NewKeccak256() + hasher.Write(entry) + + if hash := common.BytesToHash(hasher.Sum(nil)); hash != header.Hash() { + t.Fatalf("Retrieved RLP header mismatch: have %v, want %v", entry, header) + } + } + // Delete the header and verify the execution + DeleteHeader(db, header.Hash()) + if entry := GetHeader(db, header.Hash()); entry != nil { + t.Fatalf("Deleted header returned: %v", entry) + } +} + +// Tests block body storage and retrieval operations. +func TestBodyStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + + // Create a test body to move around the database and make sure it's really new + body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} + + hasher := sha3.NewKeccak256() + rlp.Encode(hasher, body) + hash := common.BytesToHash(hasher.Sum(nil)) + + if entry := GetBody(db, hash); entry != nil { + t.Fatalf("Non existent body returned: %v", entry) + } + // Write and verify the body in the database + if err := WriteBody(db, hash, body); err != nil { + t.Fatalf("Failed to write body into database: %v", err) + } + if entry := GetBody(db, hash); entry == nil { + t.Fatalf("Stored body not found") + } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { + t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) + } + if entry := GetBodyRLP(db, hash); entry == nil { + t.Fatalf("Stored body RLP not found") + } else { + hasher := sha3.NewKeccak256() + hasher.Write(entry) + + if calc := common.BytesToHash(hasher.Sum(nil)); calc != hash { + t.Fatalf("Retrieved RLP body mismatch: have %v, want %v", entry, body) + } + } + // Delete the body and verify the execution + DeleteBody(db, hash) + if entry := GetBody(db, hash); entry != nil { + t.Fatalf("Deleted body returned: %v", entry) + } +} + +// Tests block storage and retrieval operations. +func TestBlockStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + + // Create a test block to move around the database and make sure it's really new + block := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block")}) + if entry := GetBlock(db, block.Hash()); entry != nil { + t.Fatalf("Non existent block returned: %v", entry) + } + if entry := GetHeader(db, block.Hash()); entry != nil { + t.Fatalf("Non existent header returned: %v", entry) + } + if entry := GetBody(db, block.Hash()); entry != nil { + t.Fatalf("Non existent body returned: %v", entry) + } + // Write and verify the block in the database + if err := WriteBlock(db, block); err != nil { + t.Fatalf("Failed to write block into database: %v", err) + } + if entry := GetBlock(db, block.Hash()); entry == nil { + t.Fatalf("Stored block not found") + } else if entry.Hash() != block.Hash() { + t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) + } + if entry := GetHeader(db, block.Hash()); entry == nil { + t.Fatalf("Stored header not found") + } else if entry.Hash() != block.Header().Hash() { + t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) + } + if entry := GetBody(db, block.Hash()); entry == nil { + t.Fatalf("Stored body not found") + } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) { + t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, &types.Body{block.Transactions(), block.Uncles()}) + } + // Delete the block and verify the execution + DeleteBlock(db, block.Hash()) + if entry := GetBlock(db, block.Hash()); entry != nil { + t.Fatalf("Deleted block returned: %v", entry) + } + if entry := GetHeader(db, block.Hash()); entry != nil { + t.Fatalf("Deleted header returned: %v", entry) + } + if entry := GetBody(db, block.Hash()); entry != nil { + t.Fatalf("Deleted body returned: %v", entry) + } +} + +// Tests that partial block contents don't get reassembled into full blocks. +func TestPartialBlockStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + block := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block")}) + + // Store a header and check that it's not recognized as a block + if err := WriteHeader(db, block.Header()); err != nil { + t.Fatalf("Failed to write header into database: %v", err) + } + if entry := GetBlock(db, block.Hash()); entry != nil { + t.Fatalf("Non existent block returned: %v", entry) + } + DeleteHeader(db, block.Hash()) + + // Store a body and check that it's not recognized as a block + if err := WriteBody(db, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil { + t.Fatalf("Failed to write body into database: %v", err) + } + if entry := GetBlock(db, block.Hash()); entry != nil { + t.Fatalf("Non existent block returned: %v", entry) + } + DeleteBody(db, block.Hash()) + + // Store a header and a body separately and check reassembly + if err := WriteHeader(db, block.Header()); err != nil { + t.Fatalf("Failed to write header into database: %v", err) + } + if err := WriteBody(db, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil { + t.Fatalf("Failed to write body into database: %v", err) + } + if entry := GetBlock(db, block.Hash()); entry == nil { + t.Fatalf("Stored block not found") + } else if entry.Hash() != block.Hash() { + t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) + } +} + +// Tests block total difficulty storage and retrieval operations. +func TestTdStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + + // Create a test TD to move around the database and make sure it's really new + hash, td := common.Hash{}, big.NewInt(314) + if entry := GetTd(db, hash); entry != nil { + t.Fatalf("Non existent TD returned: %v", entry) + } + // Write and verify the TD in the database + if err := WriteTd(db, hash, td); err != nil { + t.Fatalf("Failed to write TD into database: %v", err) + } + if entry := GetTd(db, hash); entry == nil { + t.Fatalf("Stored TD not found") + } else if entry.Cmp(td) != 0 { + t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td) + } + // Delete the TD and verify the execution + DeleteTd(db, hash) + if entry := GetTd(db, hash); entry != nil { + t.Fatalf("Deleted TD returned: %v", entry) + } +} + +// Tests that canonical numbers can be mapped to hashes and retrieved. +func TestCanonicalMappingStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + + // Create a test canonical number and assinged hash to move around + hash, number := common.Hash{0: 0xff}, uint64(314) + if entry := GetCanonicalHash(db, number); entry != (common.Hash{}) { + t.Fatalf("Non existent canonical mapping returned: %v", entry) + } + // Write and verify the TD in the database + if err := WriteCanonicalHash(db, hash, number); err != nil { + t.Fatalf("Failed to write canonical mapping into database: %v", err) + } + if entry := GetCanonicalHash(db, number); entry == (common.Hash{}) { + t.Fatalf("Stored canonical mapping not found") + } else if entry != hash { + t.Fatalf("Retrieved canonical mapping mismatch: have %v, want %v", entry, hash) + } + // Delete the TD and verify the execution + DeleteCanonicalHash(db, number) + if entry := GetCanonicalHash(db, number); entry != (common.Hash{}) { + t.Fatalf("Deleted canonical mapping returned: %v", entry) + } +} + +// Tests that head headers and head blocks can be assigned, individually. +func TestHeadStorage(t *testing.T) { + db, _ := ethdb.NewMemDatabase() + + blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")}) + blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")}) + + // Check that no head entries are in a pristine database + if entry := GetHeadHeaderHash(db); entry != (common.Hash{}) { + t.Fatalf("Non head header entry returned: %v", entry) + } + if entry := GetHeadBlockHash(db); entry != (common.Hash{}) { + t.Fatalf("Non head block entry returned: %v", entry) + } + // Assign separate entries for the head header and block + if err := WriteHeadHeaderHash(db, blockHead.Hash()); err != nil { + t.Fatalf("Failed to write head header hash: %v", err) + } + if err := WriteHeadBlockHash(db, blockFull.Hash()); err != nil { + t.Fatalf("Failed to write head block hash: %v", err) + } + // Check that both heads are present, and different (i.e. two heads maintained) + if entry := GetHeadHeaderHash(db); entry != blockHead.Hash() { + t.Fatalf("Head header hash mismatch: have %v, want %v", entry, blockHead.Hash()) + } + if entry := GetHeadBlockHash(db); entry != blockFull.Hash() { + t.Fatalf("Head block hash mismatch: have %v, want %v", entry, blockFull.Hash()) + } +} diff --git a/core/genesis.go b/core/genesis.go index 7d4e03c99..727e2c75f 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -27,13 +27,14 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/params" ) // WriteGenesisBlock writes the genesis block to the database as block number 0 -func WriteGenesisBlock(chainDb common.Database, reader io.Reader) (*types.Block, error) { +func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, error) { contents, err := ioutil.ReadAll(reader) if err != nil { return nil, err @@ -82,34 +83,35 @@ func WriteGenesisBlock(chainDb common.Database, reader io.Reader) (*types.Block, Coinbase: common.HexToAddress(genesis.Coinbase), Root: statedb.Root(), }, nil, nil, nil) - block.Td = difficulty - if block := GetBlockByHash(chainDb, block.Hash()); block != nil { + if block := GetBlock(chainDb, block.Hash()); block != nil { glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number") - err := WriteCanonNumber(chainDb, block) + err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) if err != nil { return nil, err } return block, nil } - statedb.Sync() - err = WriteBlock(chainDb, block) - if err != nil { + if err := WriteTd(chainDb, block.Hash(), difficulty); err != nil { return nil, err } - err = WriteHead(chainDb, block) - if err != nil { + if err := WriteBlock(chainDb, block); err != nil { + return nil, err + } + if err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()); err != nil { + return nil, err + } + if err := WriteHeadBlockHash(chainDb, block.Hash()); err != nil { return nil, err } - return block, nil } // GenesisBlockForTesting creates a block in which addr has the given wei balance. // The state trie of the block is written to db. -func GenesisBlockForTesting(db common.Database, addr common.Address, balance *big.Int) *types.Block { +func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { statedb := state.New(common.Hash{}, db) obj := statedb.GetOrNewStateObject(addr) obj.SetBalance(balance) @@ -120,11 +122,10 @@ func GenesisBlockForTesting(db common.Database, addr common.Address, balance *bi GasLimit: params.GenesisGasLimit, Root: statedb.Root(), }, nil, nil, nil) - block.Td = params.GenesisDifficulty return block } -func WriteGenesisBlockForTesting(db common.Database, addr common.Address, balance *big.Int) *types.Block { +func WriteGenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { testGenesis := fmt.Sprintf(`{ "nonce":"0x%x", "gasLimit":"0x%x", @@ -137,7 +138,7 @@ func WriteGenesisBlockForTesting(db common.Database, addr common.Address, balanc return block } -func WriteTestNetGenesisBlock(chainDb common.Database, nonce uint64) (*types.Block, error) { +func WriteTestNetGenesisBlock(chainDb ethdb.Database, nonce uint64) (*types.Block, error) { testGenesis := fmt.Sprintf(`{ "nonce":"0x%x", "gasLimit":"0x%x", diff --git a/core/helper_test.go b/core/helper_test.go index b21f31d7c..81ea6fc22 100644 --- a/core/helper_test.go +++ b/core/helper_test.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" // "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" ) @@ -32,7 +32,7 @@ type TestManager struct { // stateManager *StateManager eventMux *event.TypeMux - db common.Database + db ethdb.Database txPool *TxPool blockChain *ChainManager Blocks []*types.Block @@ -74,7 +74,7 @@ func (tm *TestManager) EventMux() *event.TypeMux { // return nil // } -func (tm *TestManager) Db() common.Database { +func (tm *TestManager) Db() ethdb.Database { return tm.db } diff --git a/core/manager.go b/core/manager.go index 8b0401b03..0f108a6de 100644 --- a/core/manager.go +++ b/core/manager.go @@ -18,7 +18,7 @@ package core import ( "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" ) @@ -28,7 +28,7 @@ type Backend interface { BlockProcessor() *BlockProcessor ChainManager() *ChainManager TxPool() *TxPool - ChainDb() common.Database - DappDb() common.Database + ChainDb() ethdb.Database + DappDb() ethdb.Database EventMux() *event.TypeMux } diff --git a/core/state/state_object.go b/core/state/state_object.go index 0af0fbd5a..353f2357b 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" @@ -56,7 +57,7 @@ func (self Storage) Copy() Storage { type StateObject struct { // State database for storing state changes - db common.Database + db ethdb.Database trie *trie.SecureTrie // Address belonging to this account @@ -87,11 +88,7 @@ type StateObject struct { dirty bool } -func (self *StateObject) Reset() { - self.storage = make(Storage) -} - -func NewStateObject(address common.Address, db common.Database) *StateObject { +func NewStateObject(address common.Address, db ethdb.Database) *StateObject { object := &StateObject{db: db, address: address, balance: new(big.Int), gasPool: new(big.Int), dirty: true} object.trie = trie.NewSecure((common.Hash{}).Bytes(), db) object.storage = make(Storage) @@ -100,7 +97,7 @@ func NewStateObject(address common.Address, db common.Database) *StateObject { return object } -func NewStateObjectFromBytes(address common.Address, data []byte, db common.Database) *StateObject { +func NewStateObjectFromBytes(address common.Address, data []byte, db ethdb.Database) *StateObject { // TODO clean me up var extobject struct { Nonce uint64 @@ -184,14 +181,6 @@ func (self *StateObject) Update() { } } -func (c *StateObject) GetInstr(pc *big.Int) *common.Value { - if int64(len(c.code)-1) < pc.Int64() { - return common.NewValue(0) - } - - return common.NewValueFromBytes([]byte{c.code[pc.Int64()]}) -} - func (c *StateObject) AddBalance(amount *big.Int) { c.SetBalance(new(big.Int).Add(c.balance, amount)) @@ -268,10 +257,6 @@ func (self *StateObject) Copy() *StateObject { return stateObject } -func (self *StateObject) Set(stateObject *StateObject) { - *self = *stateObject -} - // // Attribute accessors // @@ -280,20 +265,11 @@ func (self *StateObject) Balance() *big.Int { return self.balance } -func (c *StateObject) N() *big.Int { - return big.NewInt(int64(c.nonce)) -} - // Returns the address of the contract/account func (c *StateObject) Address() common.Address { return c.address } -// Returns the initialization Code -func (c *StateObject) Init() Code { - return c.initCode -} - func (self *StateObject) Trie() *trie.SecureTrie { return self.trie } @@ -311,11 +287,6 @@ func (self *StateObject) SetCode(code []byte) { self.dirty = true } -func (self *StateObject) SetInitCode(code []byte) { - self.initCode = code - self.dirty = true -} - func (self *StateObject) SetNonce(nonce uint64) { self.nonce = nonce self.dirty = true @@ -354,19 +325,6 @@ func (c *StateObject) CodeHash() common.Bytes { return crypto.Sha3(c.code) } -func (c *StateObject) RlpDecode(data []byte) { - decoder := common.NewValueFromBytes(data) - c.nonce = decoder.Get(0).Uint() - c.balance = decoder.Get(1).BigInt() - c.trie = trie.NewSecure(decoder.Get(2).Bytes(), c.db) - c.storage = make(map[string]common.Hash) - c.gasPool = new(big.Int) - - c.codeHash = decoder.Get(3).Bytes() - - c.code, _ = c.db.Get(c.codeHash) -} - // Storage change object. Used by the manifest for notifying changes to // the sub channels. type StorageState struct { diff --git a/core/state/statedb.go b/core/state/statedb.go index 577f7162e..24f97e32a 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,10 +18,10 @@ package state import ( - "bytes" "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/trie" @@ -33,7 +33,7 @@ import ( // * Contracts // * Accounts type StateDB struct { - db common.Database + db ethdb.Database trie *trie.SecureTrie root common.Hash @@ -48,7 +48,7 @@ type StateDB struct { } // Create a new state from a given trie -func New(root common.Hash, db common.Database) *StateDB { +func New(root common.Hash, db ethdb.Database) *StateDB { trie := trie.NewSecure(root[:], db) return &StateDB{root: root, db: db, trie: trie, stateObjects: make(map[string]*StateObject), refund: new(big.Int), logs: make(map[common.Hash]Logs)} } @@ -276,10 +276,6 @@ func (self *StateDB) CreateAccount(addr common.Address) *StateObject { // Setting, copying of the state methods // -func (s *StateDB) Cmp(other *StateDB) bool { - return bytes.Equal(s.trie.Root(), other.trie.Root()) -} - func (self *StateDB) Copy() *StateDB { state := New(common.Hash{}, self.db) state.trie = self.trie @@ -311,22 +307,6 @@ func (s *StateDB) Root() common.Hash { return common.BytesToHash(s.trie.Root()) } -func (s *StateDB) Trie() *trie.SecureTrie { - return s.trie -} - -// Resets the trie and all siblings -func (s *StateDB) Reset() { - s.trie.Reset() - - // Reset all nested states - for _, stateObject := range s.stateObjects { - stateObject.Reset() - } - - s.Empty() -} - // Syncs the trie and all siblings func (s *StateDB) Sync() { // Sync all nested states diff --git a/core/transaction_util.go b/core/transaction_util.go index ce2ceac46..69c6bc36f 100644 --- a/core/transaction_util.go +++ b/core/transaction_util.go @@ -32,7 +32,7 @@ var ( ) // PutTransactions stores the transactions in the given database -func PutTransactions(db common.Database, block *types.Block, txs types.Transactions) { +func PutTransactions(db ethdb.Database, block *types.Block, txs types.Transactions) { batch := new(leveldb.Batch) _, batchWrite := db.(*ethdb.LDBDatabase) @@ -78,7 +78,7 @@ func PutTransactions(db common.Database, block *types.Block, txs types.Transacti } // PutReceipts stores the receipts in the current database -func PutReceipts(db common.Database, receipts types.Receipts) error { +func PutReceipts(db ethdb.Database, receipts types.Receipts) error { batch := new(leveldb.Batch) _, batchWrite := db.(*ethdb.LDBDatabase) @@ -108,7 +108,7 @@ func PutReceipts(db common.Database, receipts types.Receipts) error { } // GetReceipt returns a receipt by hash -func GetReceipt(db common.Database, txHash common.Hash) *types.Receipt { +func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { data, _ := db.Get(append(receiptsPre, txHash[:]...)) if len(data) == 0 { return nil @@ -124,7 +124,7 @@ func GetReceipt(db common.Database, txHash common.Hash) *types.Receipt { // GetBlockReceipts returns the receipts generated by the transactions // included in block's given hash. -func GetBlockReceipts(db common.Database, hash common.Hash) types.Receipts { +func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { data, _ := db.Get(append(blockReceiptsPre, hash[:]...)) if len(data) == 0 { return nil @@ -141,7 +141,7 @@ func GetBlockReceipts(db common.Database, hash common.Hash) types.Receipts { // PutBlockReceipts stores the block's transactions associated receipts // and stores them by block hash in a single slice. This is required for // forks and chain reorgs -func PutBlockReceipts(db common.Database, block *types.Block, receipts types.Receipts) error { +func PutBlockReceipts(db ethdb.Database, block *types.Block, receipts types.Receipts) error { rs := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { rs[i] = (*types.ReceiptForStorage)(receipt) diff --git a/core/types/block.go b/core/types/block.go index fd81db04c..7a84045a6 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -117,6 +117,13 @@ func rlpHash(x interface{}) (h common.Hash) { return h } +// Body is a simple (mutable, non-safe) data container for storing and moving +// a block's data contents (transactions and uncles) together. +type Body struct { + Transactions []*Transaction + Uncles []*Header +} + type Block struct { header *Header uncles []*Header @@ -129,12 +136,20 @@ type Block struct { // Td is used by package core to store the total difficulty // of the chain up to and including the block. - Td *big.Int + td *big.Int // ReceivedAt is used by package eth to track block propagation time. ReceivedAt time.Time } +// DeprecatedTd is an old relic for extracting the TD of a block. It is in the +// code solely to facilitate upgrading the database from the old format to the +// new, after which it should be deleted. Do not use! +func (b *Block) DeprecatedTd() *big.Int { + return b.td +} + +// [deprecated by eth/63] // StorageBlock defines the RLP encoding of a Block stored in the // state database. The StorageBlock encoding contains fields that // would otherwise need to be recomputed. @@ -147,6 +162,7 @@ type extblock struct { Uncles []*Header } +// [deprecated by eth/63] // "storage" block encoding. used for database. type storageblock struct { Header *Header @@ -168,7 +184,7 @@ var ( // are ignored and set to values derived from the given txs, uncles // and receipts. func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt) *Block { - b := &Block{header: copyHeader(header), Td: new(big.Int)} + b := &Block{header: copyHeader(header), td: new(big.Int)} // TODO: panic if len(txs) != len(receipts) if len(txs) == 0 { @@ -268,24 +284,16 @@ func (b *Block) EncodeRLP(w io.Writer) error { }) } +// [deprecated by eth/63] func (b *StorageBlock) DecodeRLP(s *rlp.Stream) error { var sb storageblock if err := s.Decode(&sb); err != nil { return err } - b.header, b.uncles, b.transactions, b.Td = sb.Header, sb.Uncles, sb.Txs, sb.TD + b.header, b.uncles, b.transactions, b.td = sb.Header, sb.Uncles, sb.Txs, sb.TD return nil } -func (b *StorageBlock) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, storageblock{ - Header: b.header, - Txs: b.transactions, - Uncles: b.uncles, - TD: b.Td, - }) -} - // TODO: copies func (b *Block) Uncles() []*Header { return b.uncles } func (b *Block) Transactions() Transactions { return b.transactions } @@ -356,7 +364,6 @@ func (b *Block) WithMiningResult(nonce uint64, mixDigest common.Hash) *Block { transactions: b.transactions, receipts: b.receipts, uncles: b.uncles, - Td: b.Td, } } @@ -386,7 +393,7 @@ func (b *Block) Hash() common.Hash { } func (b *Block) String() string { - str := fmt.Sprintf(`Block(#%v): Size: %v TD: %v { + str := fmt.Sprintf(`Block(#%v): Size: %v { MinerHash: %x %v Transactions: @@ -394,7 +401,7 @@ Transactions: Uncles: %v } -`, b.Number(), b.Size(), b.Td, b.header.HashNoNonce(), b.header, b.transactions, b.uncles) +`, b.Number(), b.Size(), b.header.HashNoNonce(), b.header, b.transactions, b.uncles) return str } diff --git a/core/types/transaction.go b/core/types/transaction.go index 28a7e02b3..8260d7423 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -33,10 +33,6 @@ import ( var ErrInvalidSig = errors.New("invalid v, r, s values") -func IsContractAddr(addr []byte) bool { - return len(addr) == 0 -} - type Transaction struct { data txdata // caches diff --git a/core/vm/errors.go b/core/vm/errors.go index 24567e9a1..e2fc84065 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -25,20 +25,3 @@ import ( var OutOfGasError = errors.New("Out of gas") var DepthError = fmt.Errorf("Max call depth exceeded (%d)", params.CallCreateDepth) - -type StackError struct { - req, has int -} - -func StackErr(req, has int) StackError { - return StackError{req, has} -} - -func (self StackError) Error() string { - return fmt.Sprintf("stack error! require %v, have %v", self.req, self.has) -} - -func IsStackErr(err error) bool { - _, ok := err.(StackError) - return ok -} diff --git a/crypto/crypto.go b/crypto/crypto.go index a474d6f13..b3a8d730b 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -33,12 +33,12 @@ import ( "encoding/json" "errors" - "code.google.com/p/go-uuid/uuid" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/crypto/secp256k1" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/rlp" + "github.com/pborman/uuid" "golang.org/x/crypto/pbkdf2" "golang.org/x/crypto/ripemd160" ) diff --git a/crypto/key.go b/crypto/key.go index d80b99759..35139b67f 100644 --- a/crypto/key.go +++ b/crypto/key.go @@ -23,8 +23,8 @@ import ( "encoding/json" "io" - "code.google.com/p/go-uuid/uuid" "github.com/ethereum/go-ethereum/common" + "github.com/pborman/uuid" ) const ( diff --git a/crypto/key_store_passphrase.go b/crypto/key_store_passphrase.go index f21af8dd9..c7ee00987 100644 --- a/crypto/key_store_passphrase.go +++ b/crypto/key_store_passphrase.go @@ -36,9 +36,9 @@ import ( "io" "reflect" - "code.google.com/p/go-uuid/uuid" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/randentropy" + "github.com/pborman/uuid" "golang.org/x/crypto/pbkdf2" "golang.org/x/crypto/scrypt" ) diff --git a/eth/backend.go b/eth/backend.go index ad2a2c1f9..349dfa613 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -18,6 +18,7 @@ package eth import ( + "bytes" "crypto/ecdsa" "encoding/json" "fmt" @@ -73,6 +74,8 @@ var ( ) type Config struct { + DevMode bool + Name string NetworkId int GenesisNonce int @@ -125,7 +128,7 @@ type Config struct { // NewDB is used to create databases. // If nil, the default is to create leveldb databases on disk. - NewDB func(path string) (common.Database, error) + NewDB func(path string) (ethdb.Database, error) } func (cfg *Config) parseBootNodes() []*discover.Node { @@ -207,11 +210,8 @@ type Ethereum struct { shutdownChan chan bool // DB interfaces - chainDb common.Database // Block chain databe - dappDb common.Database // Dapp database - - // Closed when databases are flushed and closed - databasesClosed chan bool + chainDb ethdb.Database // Block chain database + dappDb ethdb.Database // Dapp database //*** SERVICES *** // State manager for processing new blocks and managing the over all states @@ -264,14 +264,10 @@ func New(config *Config) (*Ethereum, error) { newdb := config.NewDB if newdb == nil { - newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path, config.DatabaseCache) } - } - - // attempt to merge database together, upgrading from an old version - if err := mergeDatabases(config.DataDir, newdb); err != nil { - return nil, err + newdb = func(path string) (ethdb.Database, error) { return ethdb.NewLDBDatabase(path, config.DatabaseCache) } } + // Open the chain database and perform any upgrades needed chainDb, err := newdb(filepath.Join(config.DataDir, "chaindata")) if err != nil { return nil, fmt.Errorf("blockchain db err: %v", err) @@ -279,6 +275,10 @@ func New(config *Config) (*Ethereum, error) { if db, ok := chainDb.(*ethdb.LDBDatabase); ok { db.Meter("eth/db/chaindata/") } + if err := upgradeChainDatabase(chainDb); err != nil { + return nil, err + } + dappDb, err := newdb(filepath.Join(config.DataDir, "dapp")) if err != nil { return nil, fmt.Errorf("dapp db err: %v", err) @@ -303,18 +303,23 @@ func New(config *Config) (*Ethereum, error) { glog.V(logger.Info).Infof("Successfully wrote genesis block. New genesis hash = %x\n", block.Hash()) } - if config.Olympic { + // different modes + switch { + case config.Olympic: + glog.V(logger.Error).Infoln("Starting Olympic network") + fallthrough + case config.DevMode: _, err := core.WriteTestNetGenesisBlock(chainDb, 42) if err != nil { return nil, err } - glog.V(logger.Error).Infoln("Starting Olympic network") } - // This is for testing only. if config.GenesisBlock != nil { + core.WriteTd(chainDb, config.GenesisBlock.Hash(), config.GenesisBlock.Difficulty()) core.WriteBlock(chainDb, config.GenesisBlock) - core.WriteHead(chainDb, config.GenesisBlock) + core.WriteCanonicalHash(chainDb, config.GenesisBlock.Hash(), config.GenesisBlock.NumberU64()) + core.WriteHeadBlockHash(chainDb, config.GenesisBlock.Hash()) } if !config.SkipBcVersionCheck { @@ -329,7 +334,6 @@ func New(config *Config) (*Ethereum, error) { eth := &Ethereum{ shutdownChan: make(chan bool), - databasesClosed: make(chan bool), chainDb: chainDb, dappDb: dappDb, eventMux: &event.TypeMux{}, @@ -519,8 +523,8 @@ func (s *Ethereum) BlockProcessor() *core.BlockProcessor { return s.blockProcess func (s *Ethereum) TxPool() *core.TxPool { return s.txPool } func (s *Ethereum) Whisper() *whisper.Whisper { return s.whisper } func (s *Ethereum) EventMux() *event.TypeMux { return s.eventMux } -func (s *Ethereum) ChainDb() common.Database { return s.chainDb } -func (s *Ethereum) DappDb() common.Database { return s.dappDb } +func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } +func (s *Ethereum) DappDb() ethdb.Database { return s.dappDb } func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) PeerCount() int { return s.net.PeerCount() } func (s *Ethereum) Peers() []*p2p.Peer { return s.net.Peers() } @@ -541,8 +545,6 @@ func (s *Ethereum) Start() error { if err != nil { return err } - // periodically flush databases - go s.syncDatabases() if s.AutoDAG { s.StartAutoDAG() @@ -558,32 +560,6 @@ func (s *Ethereum) Start() error { return nil } -// sync databases every minute. If flushing fails we exit immediatly. The system -// may not continue under any circumstances. -func (s *Ethereum) syncDatabases() { - ticker := time.NewTicker(1 * time.Minute) -done: - for { - select { - case <-ticker.C: - // don't change the order of database flushes - if err := s.dappDb.Flush(); err != nil { - glog.Fatalf("fatal error: flush dappDb: %v (Restart your node. We are aware of this issue)\n", err) - } - if err := s.chainDb.Flush(); err != nil { - glog.Fatalf("fatal error: flush chainDb: %v (Restart your node. We are aware of this issue)\n", err) - } - case <-s.shutdownChan: - break done - } - } - - s.chainDb.Close() - s.dappDb.Close() - - close(s.databasesClosed) -} - func (s *Ethereum) StartForTest() { jsonlogger.LogJson(&logger.LogStarting{ ClientString: s.net.Name, @@ -614,12 +590,13 @@ func (s *Ethereum) Stop() { } s.StopAutoDAG() + s.chainDb.Close() + s.dappDb.Close() close(s.shutdownChan) } // This function will wait for a shutdown and resumes main thread execution func (s *Ethereum) WaitForShutdown() { - <-s.databasesClosed <-s.shutdownChan } @@ -709,7 +686,7 @@ func dagFiles(epoch uint64) (string, string) { return dag, "full-R" + dag } -func saveBlockchainVersion(db common.Database, bcVersion int) { +func saveBlockchainVersion(db ethdb.Database, bcVersion int) { d, _ := db.Get([]byte("BlockchainVersion")) blockchainVersion := common.NewValue(d).Uint() @@ -718,74 +695,61 @@ func saveBlockchainVersion(db common.Database, bcVersion int) { } } -// mergeDatabases when required merge old database layout to one single database -func mergeDatabases(datadir string, newdb func(path string) (common.Database, error)) error { - // Check if already upgraded - data := filepath.Join(datadir, "chaindata") - if _, err := os.Stat(data); !os.IsNotExist(err) { - return nil - } - // make sure it's not just a clean path - chainPath := filepath.Join(datadir, "blockchain") - if _, err := os.Stat(chainPath); os.IsNotExist(err) { +// upgradeChainDatabase ensures that the chain database stores block split into +// separate header and body entries. +func upgradeChainDatabase(db ethdb.Database) error { + // Short circuit if the head block is stored already as separate header and body + data, err := db.Get([]byte("LastBlock")) + if err != nil { return nil } - glog.Infoln("Database upgrade required. Upgrading...") + head := common.BytesToHash(data) - database, err := newdb(data) - if err != nil { - return fmt.Errorf("creating data db err: %v", err) + if block := core.GetBlockByHashOld(db, head); block == nil { + return nil } - defer database.Close() + // At least some of the database is still the old format, upgrade (skip the head block!) + glog.V(logger.Info).Info("Old database detected, upgrading...") - // Migrate blocks - chainDb, err := newdb(chainPath) - if err != nil { - return fmt.Errorf("state db err: %v", err) - } - defer chainDb.Close() + if db, ok := db.(*ethdb.LDBDatabase); ok { + blockPrefix := []byte("block-hash-") + for it := db.NewIterator(); it.Next(); { + // Skip anything other than a combined block + if !bytes.HasPrefix(it.Key(), blockPrefix) { + continue + } + // Skip the head block (merge last to signal upgrade completion) + if bytes.HasSuffix(it.Key(), head.Bytes()) { + continue + } + // Load the block, split and serialize (order!) + block := core.GetBlockByHashOld(db, common.BytesToHash(bytes.TrimPrefix(it.Key(), blockPrefix))) - if chain, ok := chainDb.(*ethdb.LDBDatabase); ok { - glog.Infoln("Merging blockchain database...") - it := chain.NewIterator() - for it.Next() { - database.Put(it.Key(), it.Value()) + if err := core.WriteTd(db, block.Hash(), block.DeprecatedTd()); err != nil { + return err + } + if err := core.WriteBody(db, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil { + return err + } + if err := core.WriteHeader(db, block.Header()); err != nil { + return err + } + if err := db.Delete(it.Key()); err != nil { + return err + } } - it.Release() - } - - // Migrate state - stateDb, err := newdb(filepath.Join(datadir, "state")) - if err != nil { - return fmt.Errorf("state db err: %v", err) - } - defer stateDb.Close() + // Lastly, upgrade the head block, disabling the upgrade mechanism + current := core.GetBlockByHashOld(db, head) - if state, ok := stateDb.(*ethdb.LDBDatabase); ok { - glog.Infoln("Merging state database...") - it := state.NewIterator() - for it.Next() { - database.Put(it.Key(), it.Value()) + if err := core.WriteTd(db, current.Hash(), current.DeprecatedTd()); err != nil { + return err } - it.Release() - } - - // Migrate transaction / receipts - extraDb, err := newdb(filepath.Join(datadir, "extra")) - if err != nil { - return fmt.Errorf("state db err: %v", err) - } - defer extraDb.Close() - - if extra, ok := extraDb.(*ethdb.LDBDatabase); ok { - glog.Infoln("Merging transaction database...") - - it := extra.NewIterator() - for it.Next() { - database.Put(it.Key(), it.Value()) + if err := core.WriteBody(db, current.Hash(), &types.Body{current.Transactions(), current.Uncles()}); err != nil { + return err + } + if err := core.WriteHeader(db, current.Header()); err != nil { + return err } - it.Release() } - return nil } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 73f95bf64..f038e24e4 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -87,6 +87,9 @@ type blockRetrievalFn func(common.Hash) *types.Block // headRetrievalFn is a callback type for retrieving the head block from the local chain. type headRetrievalFn func() *types.Block +// tdRetrievalFn is a callback type for retrieving the total difficulty of a local block. +type tdRetrievalFn func(common.Hash) *big.Int + // chainInsertFn is a callback type to insert a batch of blocks into the local chain. type chainInsertFn func(types.Blocks) (int, error) @@ -127,15 +130,15 @@ type Downloader struct { interrupt int32 // Atomic boolean to signal termination // Statistics - importStart time.Time // Instance when the last blocks were taken from the cache - importQueue []*Block // Previously taken blocks to check import progress - importDone int // Number of taken blocks already imported from the last batch - importLock sync.Mutex + syncStatsOrigin uint64 // Origin block number where syncing started at + syncStatsHeight uint64 // Highest block number known when syncing started + syncStatsLock sync.RWMutex // Lock protecting the sync stats fields // Callbacks hasBlock hashCheckFn // Checks if a block is present in the chain getBlock blockRetrievalFn // Retrieves a block from the chain headBlock headRetrievalFn // Retrieves the head block from the chain + getTd tdRetrievalFn // Retrieves the TD of a block from the chain insertChain chainInsertFn // Injects a batch of blocks into the chain dropPeer peerDropFn // Drops a peer for misbehaving @@ -157,6 +160,7 @@ type Downloader struct { cancelLock sync.RWMutex // Lock to protect the cancel channel in delivers // Testing hooks + syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch chainInsertHook func([]*Block) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) } @@ -168,7 +172,7 @@ type Block struct { } // New creates a new downloader to fetch hashes and blocks from remote peers. -func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, headBlock headRetrievalFn, insertChain chainInsertFn, dropPeer peerDropFn) *Downloader { +func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, headBlock headRetrievalFn, getTd tdRetrievalFn, insertChain chainInsertFn, dropPeer peerDropFn) *Downloader { return &Downloader{ mux: mux, queue: newQueue(), @@ -176,6 +180,7 @@ func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, he hasBlock: hasBlock, getBlock: getBlock, headBlock: headBlock, + getTd: getTd, insertChain: insertChain, dropPeer: dropPeer, newPeerCh: make(chan *peer, 1), @@ -187,27 +192,14 @@ func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, he } } -// Stats retrieves the current status of the downloader. -func (d *Downloader) Stats() (pending int, cached int, importing int, estimate time.Duration) { - // Fetch the download status - pending, cached = d.queue.Size() - - // Figure out the import progress - d.importLock.Lock() - defer d.importLock.Unlock() +// Boundaries retrieves the synchronisation boundaries, specifically the origin +// block where synchronisation started at (may have failed/suspended) and the +// latest known block which the synchonisation targets. +func (d *Downloader) Boundaries() (uint64, uint64) { + d.syncStatsLock.RLock() + defer d.syncStatsLock.RUnlock() - for len(d.importQueue) > 0 && d.hasBlock(d.importQueue[0].RawBlock.Hash()) { - d.importQueue = d.importQueue[1:] - d.importDone++ - } - importing = len(d.importQueue) - - // Make an estimate on the total sync - estimate = 0 - if d.importDone > 0 { - estimate = time.Since(d.importStart) / time.Duration(d.importDone) * time.Duration(pending+cached+importing) - } - return + return d.syncStatsOrigin, d.syncStatsHeight } // Synchronising returns whether the downloader is currently retrieving blocks. @@ -328,14 +320,29 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e switch { case p.version == eth61: - // Old eth/61, use forward, concurrent hash and block retrieval algorithm - number, err := d.findAncestor61(p) + // Look up the sync boundaries: the common ancestor and the target block + latest, err := d.fetchHeight61(p) if err != nil { return err } + origin, err := d.findAncestor61(p) + if err != nil { + return err + } + d.syncStatsLock.Lock() + if d.syncStatsHeight <= origin || d.syncStatsOrigin > origin { + d.syncStatsOrigin = origin + } + d.syncStatsHeight = latest + d.syncStatsLock.Unlock() + + // Initiate the sync using a concurrent hash and block retrieval algorithm + if d.syncInitHook != nil { + d.syncInitHook(origin, latest) + } errc := make(chan error, 2) - go func() { errc <- d.fetchHashes61(p, td, number+1) }() - go func() { errc <- d.fetchBlocks61(number + 1) }() + go func() { errc <- d.fetchHashes61(p, td, origin+1) }() + go func() { errc <- d.fetchBlocks61(origin + 1) }() // If any fetcher fails, cancel the other if err := <-errc; err != nil { @@ -346,14 +353,29 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e return <-errc case p.version >= eth62: - // New eth/62, use forward, concurrent header and block body retrieval algorithm - number, err := d.findAncestor(p) + // Look up the sync boundaries: the common ancestor and the target block + latest, err := d.fetchHeight(p) if err != nil { return err } + origin, err := d.findAncestor(p) + if err != nil { + return err + } + d.syncStatsLock.Lock() + if d.syncStatsHeight <= origin || d.syncStatsOrigin > origin { + d.syncStatsOrigin = origin + } + d.syncStatsHeight = latest + d.syncStatsLock.Unlock() + + // Initiate the sync using a concurrent hash and block retrieval algorithm + if d.syncInitHook != nil { + d.syncInitHook(origin, latest) + } errc := make(chan error, 2) - go func() { errc <- d.fetchHeaders(p, td, number+1) }() - go func() { errc <- d.fetchBodies(number + 1) }() + go func() { errc <- d.fetchHeaders(p, td, origin+1) }() + go func() { errc <- d.fetchBodies(origin + 1) }() // If any fetcher fails, cancel the other if err := <-errc; err != nil { @@ -396,6 +418,50 @@ func (d *Downloader) Terminate() { d.cancel() } +// fetchHeight61 retrieves the head block of the remote peer to aid in estimating +// the total time a pending synchronisation would take. +func (d *Downloader) fetchHeight61(p *peer) (uint64, error) { + glog.V(logger.Debug).Infof("%v: retrieving remote chain height", p) + + // Request the advertised remote head block and wait for the response + go p.getBlocks([]common.Hash{p.head}) + + timeout := time.After(blockSoftTTL) + for { + select { + case <-d.cancelCh: + return 0, errCancelBlockFetch + + case <-d.headerCh: + // Out of bounds eth/62 block headers received, ignore them + + case <-d.bodyCh: + // Out of bounds eth/62 block bodies received, ignore them + + case <-d.hashCh: + // Out of bounds hashes received, ignore them + + case blockPack := <-d.blockCh: + // Discard anything not from the origin peer + if blockPack.peerId != p.id { + glog.V(logger.Debug).Infof("Received blocks from incorrect peer(%s)", blockPack.peerId) + break + } + // Make sure the peer actually gave something valid + blocks := blockPack.blocks + if len(blocks) != 1 { + glog.V(logger.Debug).Infof("%v: invalid number of head blocks: %d != 1", p, len(blocks)) + return 0, errBadPeer + } + return blocks[0].NumberU64(), nil + + case <-timeout: + glog.V(logger.Debug).Infof("%v: head block timeout", p) + return 0, errTimeout + } + } +} + // findAncestor61 tries to locate the common ancestor block of the local chain and // a remote peers blockchain. In the general case when our node was in sync and // on the correct chain, checking the top N blocks should already get us a match. @@ -582,7 +648,7 @@ func (d *Downloader) fetchHashes61(p *peer, td *big.Int, from uint64) error { // L: Sync begins, and finds common ancestor at 11 // L: Request new hashes up from 11 (R's TD was higher, it must have something) // R: Nothing to give - if !gotHashes && td.Cmp(d.headBlock().Td) > 0 { + if !gotHashes && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 { return errStallingPeer } return nil @@ -771,6 +837,50 @@ func (d *Downloader) fetchBlocks61(from uint64) error { } } +// fetchHeight retrieves the head header of the remote peer to aid in estimating +// the total time a pending synchronisation would take. +func (d *Downloader) fetchHeight(p *peer) (uint64, error) { + glog.V(logger.Debug).Infof("%v: retrieving remote chain height", p) + + // Request the advertised remote head block and wait for the response + go p.getRelHeaders(p.head, 1, 0, false) + + timeout := time.After(headerTTL) + for { + select { + case <-d.cancelCh: + return 0, errCancelBlockFetch + + case headerPack := <-d.headerCh: + // Discard anything not from the origin peer + if headerPack.peerId != p.id { + glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId) + break + } + // Make sure the peer actually gave something valid + headers := headerPack.headers + if len(headers) != 1 { + glog.V(logger.Debug).Infof("%v: invalid number of head headers: %d != 1", p, len(headers)) + return 0, errBadPeer + } + return headers[0].Number.Uint64(), nil + + case <-d.bodyCh: + // Out of bounds block bodies received, ignore them + + case <-d.hashCh: + // Out of bounds eth/61 hashes received, ignore them + + case <-d.blockCh: + // Out of bounds eth/61 blocks received, ignore them + + case <-timeout: + glog.V(logger.Debug).Infof("%v: head header timeout", p) + return 0, errTimeout + } + } +} + // findAncestor tries to locate the common ancestor block of the local chain and // a remote peers blockchain. In the general case when our node was in sync and // on the correct chain, checking the top N blocks should already get us a match. @@ -958,7 +1068,7 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error { // L: Sync begins, and finds common ancestor at 11 // L: Request new headers up from 11 (R's TD was higher, it must have something) // R: Nothing to give - if !gotHeaders && td.Cmp(d.headBlock().Td) > 0 { + if !gotHeaders && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 { return errStallingPeer } return nil @@ -968,7 +1078,7 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error { // Otherwise insert all the new headers, aborting in case of junk glog.V(logger.Detail).Infof("%v: inserting %d headers from #%d", p, len(headerPack.headers), from) - inserts := d.queue.Insert(headerPack.headers) + inserts := d.queue.Insert(headerPack.headers, from) if len(inserts) != len(headerPack.headers) { glog.V(logger.Debug).Infof("%v: stale headers", p) return errBadPeer @@ -1198,16 +1308,10 @@ func (d *Downloader) process() { d.process() } }() - // Release the lock upon exit (note, before checking for reentry!), and set + // Release the lock upon exit (note, before checking for reentry!) // the import statistics to zero. - defer func() { - d.importLock.Lock() - d.importQueue = nil - d.importDone = 0 - d.importLock.Unlock() + defer atomic.StoreInt32(&d.processing, 0) - atomic.StoreInt32(&d.processing, 0) - }() // Repeat the processing as long as there are blocks to import for { // Fetch the next batch of blocks @@ -1218,13 +1322,6 @@ func (d *Downloader) process() { if d.chainInsertHook != nil { d.chainInsertHook(blocks) } - // Reset the import statistics - d.importLock.Lock() - d.importStart = time.Now() - d.importQueue = blocks - d.importDone = 0 - d.importLock.Unlock() - // Actually import the blocks glog.V(logger.Debug).Infof("Inserting chain with %d blocks (#%v - #%v)\n", len(blocks), blocks[0].RawBlock.Number(), blocks[len(blocks)-1].RawBlock.Number()) for len(blocks) != 0 { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 8d009b671..885fab8bd 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "sync" "sync/atomic" "testing" "time" @@ -93,38 +94,45 @@ func makeChainFork(n, f int, parent *types.Block) (h1, h2 []common.Hash, b1, b2 type downloadTester struct { downloader *Downloader - ownHashes []common.Hash // Hash chain belonging to the tester - ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester - peerHashes map[string][]common.Hash // Hash chain belonging to different test peers - peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers + ownHashes []common.Hash // Hash chain belonging to the tester + ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester + ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain + peerHashes map[string][]common.Hash // Hash chain belonging to different test peers + peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers + peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains + + lock sync.RWMutex } // newTester creates a new downloader test mocker. func newTester() *downloadTester { tester := &downloadTester{ - ownHashes: []common.Hash{genesis.Hash()}, - ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - peerHashes: make(map[string][]common.Hash), - peerBlocks: make(map[string]map[common.Hash]*types.Block), + ownHashes: []common.Hash{genesis.Hash()}, + ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, + ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, + peerHashes: make(map[string][]common.Hash), + peerBlocks: make(map[string]map[common.Hash]*types.Block), + peerChainTds: make(map[string]map[common.Hash]*big.Int), } - tester.downloader = New(new(event.TypeMux), tester.hasBlock, tester.getBlock, tester.headBlock, tester.insertChain, tester.dropPeer) + tester.downloader = New(new(event.TypeMux), tester.hasBlock, tester.getBlock, tester.headBlock, tester.getTd, tester.insertChain, tester.dropPeer) return tester } // sync starts synchronizing with a remote peer, blocking until it completes. func (dl *downloadTester) sync(id string, td *big.Int) error { + dl.lock.RLock() hash := dl.peerHashes[id][0] - // If no particular TD was requested, load from the peer's blockchain if td == nil { td = big.NewInt(1) - if block, ok := dl.peerBlocks[id][hash]; ok { - td = block.Td + if diff, ok := dl.peerChainTds[id][hash]; ok { + td = diff } } - err := dl.downloader.synchronise(id, hash, td) + dl.lock.RUnlock() + err := dl.downloader.synchronise(id, hash, td) for { // If the queue is empty and processing stopped, break hashes, blocks := dl.downloader.queue.Size() @@ -139,27 +147,48 @@ func (dl *downloadTester) sync(id string, td *big.Int) error { // hasBlock checks if a block is pres ent in the testers canonical chain. func (dl *downloadTester) hasBlock(hash common.Hash) bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + return dl.getBlock(hash) != nil } // getBlock retrieves a block from the testers canonical chain. func (dl *downloadTester) getBlock(hash common.Hash) *types.Block { + dl.lock.RLock() + defer dl.lock.RUnlock() + return dl.ownBlocks[hash] } // headBlock retrieves the current head block from the canonical chain. func (dl *downloadTester) headBlock() *types.Block { + dl.lock.RLock() + defer dl.lock.RUnlock() + return dl.getBlock(dl.ownHashes[len(dl.ownHashes)-1]) } +// getTd retrieves the block's total difficulty from the canonical chain. +func (dl *downloadTester) getTd(hash common.Hash) *big.Int { + dl.lock.RLock() + defer dl.lock.RUnlock() + + return dl.ownChainTd[hash] +} + // insertChain injects a new batch of blocks into the simulated chain. func (dl *downloadTester) insertChain(blocks types.Blocks) (int, error) { + dl.lock.Lock() + defer dl.lock.Unlock() + for i, block := range blocks { if _, ok := dl.ownBlocks[block.ParentHash()]; !ok { return i, errors.New("unknown parent") } dl.ownHashes = append(dl.ownHashes, block.Hash()) dl.ownBlocks[block.Hash()] = block + dl.ownChainTd[block.Hash()] = dl.ownChainTd[block.ParentHash()] } return len(blocks), nil } @@ -173,16 +202,26 @@ func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, // specific delay time on processing the network packets sent to it, simulating // potentially slow network IO. func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, blocks map[common.Hash]*types.Block, delay time.Duration) error { + dl.lock.Lock() + defer dl.lock.Unlock() + err := dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay), - nil, dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay)) + dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay)) if err == nil { // Assign the owned hashes and blocks to the peer (deep copy) dl.peerHashes[id] = make([]common.Hash, len(hashes)) copy(dl.peerHashes[id], hashes) + dl.peerBlocks[id] = make(map[common.Hash]*types.Block) - for hash, block := range blocks { - dl.peerBlocks[id][hash] = block + dl.peerChainTds[id] = make(map[common.Hash]*big.Int) + for _, hash := range hashes { + if block, ok := blocks[hash]; ok { + dl.peerBlocks[id][hash] = block + if parent, ok := dl.peerBlocks[id][block.ParentHash()]; ok { + dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][parent.Hash()]) + } + } } } return err @@ -190,8 +229,12 @@ func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Ha // dropPeer simulates a hard peer removal from the connection pool. func (dl *downloadTester) dropPeer(id string) { + dl.lock.Lock() + defer dl.lock.Unlock() + delete(dl.peerHashes, id) delete(dl.peerBlocks, id) + delete(dl.peerChainTds, id) dl.downloader.UnregisterPeer(id) } @@ -203,6 +246,9 @@ func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) fun return func(head common.Hash) error { time.Sleep(delay) + dl.lock.RLock() + defer dl.lock.RUnlock() + // Gather the next batch of hashes hashes := dl.peerHashes[id] result := make([]common.Hash, 0, MaxHashFetch) @@ -232,6 +278,9 @@ func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) fun return func(head uint64, count int) error { time.Sleep(delay) + dl.lock.RLock() + defer dl.lock.RUnlock() + // Gather the next batch of hashes hashes := dl.peerHashes[id] result := make([]common.Hash, 0, count) @@ -253,6 +302,10 @@ func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) fun func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([]common.Hash) error { return func(hashes []common.Hash) error { time.Sleep(delay) + + dl.lock.RLock() + defer dl.lock.RUnlock() + blocks := dl.peerBlocks[id] result := make([]*types.Block, 0, len(hashes)) for _, hash := range hashes { @@ -266,6 +319,27 @@ func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([ } } +// peerGetRelHeadersFn constructs a GetBlockHeaders function based on a hashed +// origin; associated with a particular peer in the download tester. The returned +// function can be used to retrieve batches of headers from the particular peer. +func (dl *downloadTester) peerGetRelHeadersFn(id string, delay time.Duration) func(common.Hash, int, int, bool) error { + return func(origin common.Hash, amount int, skip int, reverse bool) error { + // Find the canonical number of the hash + dl.lock.RLock() + number := uint64(0) + for num, hash := range dl.peerHashes[id] { + if hash == origin { + number = uint64(len(dl.peerHashes[id]) - num - 1) + break + } + } + dl.lock.RUnlock() + + // Use the absolute header fetcher to satisfy the query + return dl.peerGetAbsHeadersFn(id, delay)(number, amount, skip, reverse) + } +} + // peerGetAbsHeadersFn constructs a GetBlockHeaders function based on a numbered // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. @@ -273,6 +347,9 @@ func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) fu return func(origin uint64, amount int, skip int, reverse bool) error { time.Sleep(delay) + dl.lock.RLock() + defer dl.lock.RUnlock() + // Gather the next batch of hashes hashes := dl.peerHashes[id] blocks := dl.peerBlocks[id] @@ -297,6 +374,10 @@ func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) fu func (dl *downloadTester) peerGetBodiesFn(id string, delay time.Duration) func([]common.Hash) error { return func(hashes []common.Hash) error { time.Sleep(delay) + + dl.lock.RLock() + defer dl.lock.RUnlock() + blocks := dl.peerBlocks[id] transactions := make([][]*types.Transaction, 0, len(hashes)) @@ -366,13 +447,23 @@ func testThrottling(t *testing.T, protocol int) { errc <- tester.sync("peer", nil) }() // Iteratively take some blocks, always checking the retrieval count - for len(tester.ownBlocks) < targetBlocks+1 { + for { + // Check the retrieval count synchronously (! reason for this ugly block) + tester.lock.RLock() + retrieved := len(tester.ownBlocks) + tester.lock.RUnlock() + if retrieved >= targetBlocks+1 { + break + } // Wait a bit for sync to throttle itself var cached int for start := time.Now(); time.Since(start) < time.Second; { time.Sleep(25 * time.Millisecond) + tester.downloader.queue.lock.RLock() cached = len(tester.downloader.queue.blockPool) + tester.downloader.queue.lock.RUnlock() + if cached == blockCacheLimit || len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) == targetBlocks+1 { break } @@ -565,6 +656,67 @@ func testEmptyBlockShortCircuit(t *testing.T, protocol int) { } } +// Tests that headers are enqueued continuously, preventing malicious nodes from +// stalling the downloader by feeding gapped header chains. +func TestMissingHeaderAttack62(t *testing.T) { testMissingHeaderAttack(t, 62) } +func TestMissingHeaderAttack63(t *testing.T) { testMissingHeaderAttack(t, 63) } +func TestMissingHeaderAttack64(t *testing.T) { testMissingHeaderAttack(t, 64) } + +func testMissingHeaderAttack(t *testing.T, protocol int) { + // Create a small enough block chain to download + targetBlocks := blockCacheLimit - 15 + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + tester := newTester() + + // Attempt a full sync with an attacker feeding gapped headers + tester.newPeer("attack", protocol, hashes, blocks) + missing := targetBlocks / 2 + delete(tester.peerBlocks["attack"], hashes[missing]) + + if err := tester.sync("attack", nil); err == nil { + t.Fatalf("succeeded attacker synchronisation") + } + // Synchronise with the valid peer and make sure sync succeeds + tester.newPeer("valid", protocol, hashes, blocks) + if err := tester.sync("valid", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + if imported := len(tester.ownBlocks); imported != len(hashes) { + t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes)) + } +} + +// Tests that if requested headers are shifted (i.e. first is missing), the queue +// detects the invalid numbering. +func TestShiftedHeaderAttack62(t *testing.T) { testShiftedHeaderAttack(t, 62) } +func TestShiftedHeaderAttack63(t *testing.T) { testShiftedHeaderAttack(t, 63) } +func TestShiftedHeaderAttack64(t *testing.T) { testShiftedHeaderAttack(t, 64) } + +func testShiftedHeaderAttack(t *testing.T, protocol int) { + // Create a small enough block chain to download + targetBlocks := blockCacheLimit - 15 + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + tester := newTester() + + // Attempt a full sync with an attacker feeding shifted headers + tester.newPeer("attack", protocol, hashes, blocks) + delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) + + if err := tester.sync("attack", nil); err == nil { + t.Fatalf("succeeded attacker synchronisation") + } + // Synchronise with the valid peer and make sure sync succeeds + tester.newPeer("valid", protocol, hashes, blocks) + if err := tester.sync("valid", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + if imported := len(tester.ownBlocks); imported != len(hashes) { + t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes)) + } +} + // Tests that if a peer sends an invalid body for a requested block, it gets // dropped immediately by the downloader. func TestInvalidBlockBodyAttack62(t *testing.T) { testInvalidBlockBodyAttack(t, 62) } @@ -709,3 +861,259 @@ func testBlockBodyAttackerDropping(t *testing.T, protocol int) { } } } + +// Tests that synchronisation boundaries (origin block number and highest block +// number) is tracked and updated correctly. +func TestSyncBoundaries61(t *testing.T) { testSyncBoundaries(t, 61) } +func TestSyncBoundaries62(t *testing.T) { testSyncBoundaries(t, 62) } +func TestSyncBoundaries63(t *testing.T) { testSyncBoundaries(t, 63) } +func TestSyncBoundaries64(t *testing.T) { testSyncBoundaries(t, 64) } + +func testSyncBoundaries(t *testing.T, protocol int) { + // Create a small enough block chain to download + targetBlocks := blockCacheLimit - 15 + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + // Set a sync init hook to catch boundary changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester := newTester() + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + // Retrieve the sync boundaries and ensure they are zero (pristine sync) + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 { + t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0) + } + // Synchronise half the blocks and check initial boundaries + tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], blocks) + pending := new(sync.WaitGroup) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("peer-half", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks/2+1) { + t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks/2+1) + } + progress <- struct{}{} + pending.Wait() + + // Synchronise all the blocks and check continuation boundaries + tester.newPeer("peer-full", protocol, hashes, blocks) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("peer-full", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) { + t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, targetBlocks/2+1, targetBlocks) + } + progress <- struct{}{} + pending.Wait() +} + +// Tests that synchronisation boundaries (origin block number and highest block +// number) is tracked and updated correctly in case of a fork (or manual head +// revertal). +func TestForkedSyncBoundaries61(t *testing.T) { testForkedSyncBoundaries(t, 61) } +func TestForkedSyncBoundaries62(t *testing.T) { testForkedSyncBoundaries(t, 62) } +func TestForkedSyncBoundaries63(t *testing.T) { testForkedSyncBoundaries(t, 63) } +func TestForkedSyncBoundaries64(t *testing.T) { testForkedSyncBoundaries(t, 64) } + +func testForkedSyncBoundaries(t *testing.T, protocol int) { + // Create a forked chain to simulate origin revertal + common, fork := MaxHashFetch, 2*MaxHashFetch + hashesA, hashesB, blocksA, blocksB := makeChainFork(common+fork, fork, genesis) + + // Set a sync init hook to catch boundary changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester := newTester() + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + // Retrieve the sync boundaries and ensure they are zero (pristine sync) + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 { + t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0) + } + // Synchronise with one of the forks and check boundaries + tester.newPeer("fork A", protocol, hashesA, blocksA) + pending := new(sync.WaitGroup) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("fork A", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(len(hashesA)-1) { + t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, len(hashesA)-1) + } + progress <- struct{}{} + pending.Wait() + + // Simulate a successful sync above the fork + tester.downloader.syncStatsOrigin = tester.downloader.syncStatsHeight + + // Synchronise with the second fork and check boundary resets + tester.newPeer("fork B", protocol, hashesB, blocksB) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("fork B", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != uint64(common) || latest != uint64(len(hashesB)-1) { + t.Fatalf("Forking boundary mismatch: have %v/%v, want %v/%v", origin, latest, common, len(hashesB)-1) + } + progress <- struct{}{} + pending.Wait() +} + +// Tests that if synchronisation is aborted due to some failure, then the boundary +// origin is not updated in the next sync cycle, as it should be considered the +// continuation of the previous sync and not a new instance. +func TestFailedSyncBoundaries61(t *testing.T) { testFailedSyncBoundaries(t, 61) } +func TestFailedSyncBoundaries62(t *testing.T) { testFailedSyncBoundaries(t, 62) } +func TestFailedSyncBoundaries63(t *testing.T) { testFailedSyncBoundaries(t, 63) } +func TestFailedSyncBoundaries64(t *testing.T) { testFailedSyncBoundaries(t, 64) } + +func testFailedSyncBoundaries(t *testing.T, protocol int) { + // Create a small enough block chain to download + targetBlocks := blockCacheLimit - 15 + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + // Set a sync init hook to catch boundary changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester := newTester() + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + // Retrieve the sync boundaries and ensure they are zero (pristine sync) + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 { + t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0) + } + // Attempt a full sync with a faulty peer + tester.newPeer("faulty", protocol, hashes, blocks) + missing := targetBlocks / 2 + delete(tester.peerBlocks["faulty"], hashes[missing]) + + pending := new(sync.WaitGroup) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("faulty", nil); err == nil { + t.Fatalf("succeeded faulty synchronisation") + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) { + t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks) + } + progress <- struct{}{} + pending.Wait() + + // Synchronise with a good peer and check that the boundary origin remind the same after a failure + tester.newPeer("valid", protocol, hashes, blocks) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("valid", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) { + t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks) + } + progress <- struct{}{} + pending.Wait() +} + +// Tests that if an attacker fakes a chain height, after the attack is detected, +// the boundary height is successfully reduced at the next sync invocation. +func TestFakedSyncBoundaries61(t *testing.T) { testFakedSyncBoundaries(t, 61) } +func TestFakedSyncBoundaries62(t *testing.T) { testFakedSyncBoundaries(t, 62) } +func TestFakedSyncBoundaries63(t *testing.T) { testFakedSyncBoundaries(t, 63) } +func TestFakedSyncBoundaries64(t *testing.T) { testFakedSyncBoundaries(t, 64) } + +func testFakedSyncBoundaries(t *testing.T, protocol int) { + // Create a small block chain + targetBlocks := blockCacheLimit - 15 + hashes, blocks := makeChain(targetBlocks+3, 0, genesis) + + // Set a sync init hook to catch boundary changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester := newTester() + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + // Retrieve the sync boundaries and ensure they are zero (pristine sync) + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 { + t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0) + } + // Create and sync with an attacker that promises a higher chain than available + tester.newPeer("attack", protocol, hashes, blocks) + for i := 1; i < 3; i++ { + delete(tester.peerBlocks["attack"], hashes[i]) + } + + pending := new(sync.WaitGroup) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("attack", nil); err == nil { + t.Fatalf("succeeded attacker synchronisation") + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks+3) { + t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks+3) + } + progress <- struct{}{} + pending.Wait() + + // Synchronise with a good peer and check that the boundary height has been reduced to the true value + tester.newPeer("valid", protocol, hashes[3:], blocks) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("valid", nil); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + }() + <-starting + if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) { + t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks) + } + progress <- struct{}{} + pending.Wait() +} diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 7db78327b..49d1046fb 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -57,6 +57,7 @@ type queue struct { headerPool map[common.Hash]*types.Header // [eth/62] Pending headers, mapping from their hashes headerQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the bodies for + headerHead common.Hash // [eth/62] Hash of the last queued header to verify order pendPool map[string]*fetchRequest // Currently pending block retrieval operations @@ -91,6 +92,7 @@ func (q *queue) Reset() { q.headerPool = make(map[common.Hash]*types.Header) q.headerQueue.Reset() + q.headerHead = common.Hash{} q.pendPool = make(map[string]*fetchRequest) @@ -186,7 +188,7 @@ func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash { // Insert adds a set of headers for the download queue for scheduling, returning // the new headers encountered. -func (q *queue) Insert(headers []*types.Header) []*types.Header { +func (q *queue) Insert(headers []*types.Header, from uint64) []*types.Header { q.lock.Lock() defer q.lock.Unlock() @@ -196,13 +198,24 @@ func (q *queue) Insert(headers []*types.Header) []*types.Header { // Make sure no duplicate requests are executed hash := header.Hash() if _, ok := q.headerPool[hash]; ok { - glog.V(logger.Warn).Infof("Header %x already scheduled", hash) + glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled", header.Number.Uint64(), hash[:4]) continue } + // Make sure chain order is honored and preserved throughout + if header.Number == nil || header.Number.Uint64() != from { + glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ordering, expected %d", header.Number, hash[:4], from) + break + } + if q.headerHead != (common.Hash{}) && q.headerHead != header.ParentHash { + glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ancestry", header.Number, hash[:4]) + break + } // Queue the header for body retrieval inserts = append(inserts, header) q.headerPool[hash] = header q.headerQueue.Push(header, -float32(header.Number.Uint64())) + q.headerHead = hash + from++ } return inserts } diff --git a/eth/handler.go b/eth/handler.go index f22afecb7..52c9c4151 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -36,8 +37,10 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// This is the target maximum size of returned blocks, headers or node data. -const softResponseLimit = 2 * 1024 * 1024 +const ( + softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. + estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header +) func errResp(code errCode, format string, v ...interface{}) error { return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) @@ -59,7 +62,7 @@ func (ep extProt) GetBlock(hashes []common.Hash) error { return ep.getBlocks(has type ProtocolManager struct { txpool txPool chainman *core.ChainManager - chaindb common.Database + chaindb ethdb.Database downloader *downloader.Downloader fetcher *fetcher.Fetcher @@ -84,7 +87,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, chainman *core.ChainManager, chaindb common.Database) *ProtocolManager { +func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, chainman *core.ChainManager, chaindb ethdb.Database) *ProtocolManager { // Create the protocol manager with the base fields manager := &ProtocolManager{ eventMux: mux, @@ -113,10 +116,10 @@ func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow po } } // Construct the different synchronisation mechanisms - manager.downloader = downloader.New(manager.eventMux, manager.chainman.HasBlock, manager.chainman.GetBlock, manager.chainman.CurrentBlock, manager.chainman.InsertChain, manager.removePeer) + manager.downloader = downloader.New(manager.eventMux, manager.chainman.HasBlock, manager.chainman.GetBlock, manager.chainman.CurrentBlock, manager.chainman.GetTd, manager.chainman.InsertChain, manager.removePeer) validator := func(block *types.Block, parent *types.Block) error { - return core.ValidateHeader(pow, block.Header(), parent, true, false) + return core.ValidateHeader(pow, block.Header(), parent.Header(), true, false) } heighter := func() uint64 { return manager.chainman.CurrentBlock().NumberU64() @@ -345,33 +348,33 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&query); err != nil { return errResp(ErrDecode, "%v: %v", msg, err) } - // Gather blocks until the fetch or network limits is reached + // Gather headers until the fetch or network limits is reached var ( bytes common.StorageSize headers []*types.Header unknown bool ) for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit && len(headers) < downloader.MaxHeaderFetch { - // Retrieve the next block satisfying the query - var origin *types.Block + // Retrieve the next header satisfying the query + var origin *types.Header if query.Origin.Hash != (common.Hash{}) { - origin = pm.chainman.GetBlock(query.Origin.Hash) + origin = pm.chainman.GetHeader(query.Origin.Hash) } else { - origin = pm.chainman.GetBlockByNumber(query.Origin.Number) + origin = pm.chainman.GetHeaderByNumber(query.Origin.Number) } if origin == nil { break } - headers = append(headers, origin.Header()) - bytes += origin.Size() + headers = append(headers, origin) + bytes += estHeaderRlpSize - // Advance to the next block of the query + // Advance to the next header of the query switch { case query.Origin.Hash != (common.Hash{}) && query.Reverse: // Hash based traversal towards the genesis block for i := 0; i < int(query.Skip)+1; i++ { - if block := pm.chainman.GetBlock(query.Origin.Hash); block != nil { - query.Origin.Hash = block.ParentHash() + if header := pm.chainman.GetHeader(query.Origin.Hash); header != nil { + query.Origin.Hash = header.ParentHash } else { unknown = true break @@ -379,9 +382,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } case query.Origin.Hash != (common.Hash{}) && !query.Reverse: // Hash based traversal towards the leaf block - if block := pm.chainman.GetBlockByNumber(origin.NumberU64() + query.Skip + 1); block != nil { - if pm.chainman.GetBlockHashesFromHash(block.Hash(), query.Skip+1)[query.Skip] == query.Origin.Hash { - query.Origin.Hash = block.Hash() + if header := pm.chainman.GetHeaderByNumber(origin.Number.Uint64() + query.Skip + 1); header != nil { + if pm.chainman.GetBlockHashesFromHash(header.Hash(), query.Skip+1)[query.Skip] == query.Origin.Hash { + query.Origin.Hash = header.Hash() } else { unknown = true } @@ -452,23 +455,23 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Gather blocks until the fetch or network limits is reached var ( hash common.Hash - bytes common.StorageSize - bodies []*blockBody + bytes int + bodies []rlp.RawValue ) for bytes < softResponseLimit && len(bodies) < downloader.MaxBlockFetch { - //Retrieve the hash of the next block + // Retrieve the hash of the next block if err := msgStream.Decode(&hash); err == rlp.EOL { break } else if err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - // Retrieve the requested block, stopping if enough was found - if block := pm.chainman.GetBlock(hash); block != nil { - bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) - bytes += block.Size() + // Retrieve the requested block body, stopping if enough was found + if data := pm.chainman.GetBodyRLP(hash); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) } } - return p.SendBlockBodies(bodies) + return p.SendBlockBodiesRLP(bodies) case p.version >= eth63 && msg.Code == GetNodeDataMsg: // Decode the retrieval message @@ -643,7 +646,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) var td *big.Int if parent := pm.chainman.GetBlock(block.ParentHash()); parent != nil { - td = new(big.Int).Add(parent.Td, block.Difficulty()) + td = new(big.Int).Add(block.Difficulty(), pm.chainman.GetTd(block.ParentHash())) } else { glog.V(logger.Error).Infof("propagating dangling block #%d [%x]", block.NumberU64(), hash[:4]) return diff --git a/eth/peer.go b/eth/peer.go index 8d7c48885..603b49b88 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" "gopkg.in/fatih/set.v0" ) @@ -184,6 +185,12 @@ func (p *peer) SendBlockBodies(bodies []*blockBody) error { return p2p.Send(p.rw, BlockBodiesMsg, blockBodiesData(bodies)) } +// SendBlockBodiesRLP sends a batch of block contents to the remote peer from +// an already RLP encoded format. +func (p *peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { + return p2p.Send(p.rw, BlockBodiesMsg, bodies) +} + // SendNodeData sends a batch of arbitrary internal data, corresponding to the // hashes requested. func (p *peer) SendNodeData(data [][]byte) error { diff --git a/ethdb/database.go b/ethdb/database.go index 9e80e5409..047821c30 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -61,9 +61,7 @@ type LDBDatabase struct { quitChan chan chan error // Quit channel to stop the metrics collection before closing the database } -// NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by -// it self but requires a background poller which syncs every X. `Flush` should be called -// when data needs to be stored and written to disk. +// NewLDBDatabase returns a LevelDB wrapped object. func NewLDBDatabase(file string, cache int) (*LDBDatabase, error) { // Calculate the cache allowance for this particular database cache = int(float64(cache) * cacheRatio[filepath.Base(file)]) @@ -142,11 +140,6 @@ func (self *LDBDatabase) NewIterator() iterator.Iterator { return self.db.NewIterator(nil, nil) } -// Flush flushes out the queue to leveldb -func (self *LDBDatabase) Flush() error { - return nil -} - func (self *LDBDatabase) Close() { // Stop the metrics collection to avoid internal database races self.quitLock.Lock() @@ -159,12 +152,14 @@ func (self *LDBDatabase) Close() { glog.V(logger.Error).Infof("metrics failure in '%s': %v\n", self.fn, err) } } - // Flush and close the database - if err := self.Flush(); err != nil { - glog.V(logger.Error).Infof("flushing '%s' failed: %v\n", self.fn, err) + err := self.db.Close() + if glog.V(logger.Error) { + if err == nil { + glog.Infoln("closed db:", self.fn) + } else { + glog.Errorf("error closing db %s: %v", self.fn, err) + } } - self.db.Close() - glog.V(logger.Error).Infoln("flushed and closed db:", self.fn) } func (self *LDBDatabase) LDB() *leveldb.DB { @@ -268,3 +263,23 @@ func (self *LDBDatabase) meter(refresh time.Duration) { } } } + +// TODO: remove this stuff and expose leveldb directly + +func (db *LDBDatabase) NewBatch() Batch { + return &ldbBatch{db: db.db, b: new(leveldb.Batch)} +} + +type ldbBatch struct { + db *leveldb.DB + b *leveldb.Batch +} + +func (b *ldbBatch) Put(key, value []byte) error { + b.b.Put(key, value) + return nil +} + +func (b *ldbBatch) Write() error { + return b.db.Write(b.b, nil) +} diff --git a/common/db.go b/ethdb/interface.go index 60c090cdc..f4b787a52 100644 --- a/common/db.go +++ b/ethdb/interface.go @@ -14,13 +14,17 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package common +package ethdb -// Database interface type Database interface { Put(key []byte, value []byte) error Get(key []byte) ([]byte, error) Delete(key []byte) error Close() - Flush() error + NewBatch() Batch +} + +type Batch interface { + Put(key, value []byte) error + Write() error } diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index d50f8f9d4..81911f23f 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -36,8 +36,7 @@ func NewMemDatabase() (*MemDatabase, error) { } func (db *MemDatabase) Put(key []byte, value []byte) error { - db.db[string(key)] = value - + db.db[string(key)] = common.CopyBytes(value) return nil } @@ -92,6 +91,25 @@ func (db *MemDatabase) LastKnownTD() []byte { return data } -func (db *MemDatabase) Flush() error { +func (db *MemDatabase) NewBatch() Batch { + return &memBatch{db: db} +} + +type kv struct{ k, v []byte } + +type memBatch struct { + db *MemDatabase + writes []kv +} + +func (w *memBatch) Put(key, value []byte) error { + w.writes = append(w.writes, kv{key, common.CopyBytes(value)}) + return nil +} + +func (w *memBatch) Write() error { + for _, kv := range w.writes { + w.db.db[string(kv.k)] = kv.v + } return nil } diff --git a/jsre/ethereum_js.go b/jsre/ethereum_js.go index f33bb7c25..2d7dbfec0 100644 --- a/jsre/ethereum_js.go +++ b/jsre/ethereum_js.go @@ -650,7 +650,7 @@ module.exports = SolidityTypeBytes; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file coder.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -680,7 +680,7 @@ var SolidityCoder = function (types) { * * @method _requireType * @param {String} type - * @returns {SolidityType} + * @returns {SolidityType} * @throws {Error} throws if no matching type is found */ SolidityCoder.prototype._requireType = function (type) { @@ -726,7 +726,7 @@ SolidityCoder.prototype.encodeParams = function (types, params) { return acc + solidityType.staticPartLength(types[index]); }, 0); - var result = this.encodeMultiWithOffset(types, solidityTypes, encodeds, dynamicOffset); + var result = this.encodeMultiWithOffset(types, solidityTypes, encodeds, dynamicOffset); return result; }; @@ -751,7 +751,7 @@ SolidityCoder.prototype.encodeMultiWithOffset = function (types, solidityTypes, // TODO: figure out nested arrays }); - + types.forEach(function (type, i) { if (isDynamic(i)) { var e = self.encodeWithOffset(types[i], solidityTypes[i], encodeds[i], dynamicOffset); @@ -771,7 +771,7 @@ SolidityCoder.prototype.encodeWithOffset = function (type, solidityType, encoded var nestedName = solidityType.nestedName(type); var nestedStaticPartLength = solidityType.staticPartLength(nestedName); var result = encoded[0]; - + (function () { var previousLength = 2; // in int if (solidityType.isDynamicArray(nestedName)) { @@ -781,7 +781,7 @@ SolidityCoder.prototype.encodeWithOffset = function (type, solidityType, encoded } } })(); - + // first element is length, skip it (function () { for (var i = 0; i < encoded.length - 1; i++) { @@ -792,7 +792,7 @@ SolidityCoder.prototype.encodeWithOffset = function (type, solidityType, encoded return result; })(); - + } else if (solidityType.isStaticArray(type)) { return (function () { var nestedName = solidityType.nestedName(type); @@ -805,7 +805,7 @@ SolidityCoder.prototype.encodeWithOffset = function (type, solidityType, encoded var previousLength = 0; // in int for (var i = 0; i < encoded.length; i++) { // calculate length of previous item - previousLength += +(encoded[i - 1] || [])[0] || 0; + previousLength += +(encoded[i - 1] || [])[0] || 0; result += f.formatInputInt(offset + i * nestedStaticPartLength + previousLength * 32).encode(); } })(); @@ -848,7 +848,7 @@ SolidityCoder.prototype.decodeParam = function (type, bytes) { SolidityCoder.prototype.decodeParams = function (types, bytes) { var solidityTypes = this.getSolidityTypes(types); var offsets = this.getOffsets(types, solidityTypes); - + return solidityTypes.map(function (solidityType, index) { return solidityType.decode(bytes, offsets[index], types[index], index); }); @@ -856,10 +856,10 @@ SolidityCoder.prototype.decodeParams = function (types, bytes) { SolidityCoder.prototype.getOffsets = function (types, solidityTypes) { var lengths = solidityTypes.map(function (solidityType, index) { - return solidityType.staticPartLength(types[index]); + return solidityType.staticPartLength(types[index]); // get length }); - + for (var i = 0; i < lengths.length; i++) { // sum with length of previous element var previous = (lengths[i - 1] || 0); @@ -938,7 +938,7 @@ module.exports = SolidityTypeDynamicBytes; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file formatters.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -1002,7 +1002,7 @@ var formatInputDynamicBytes = function (value) { * @returns {SolidityParam} */ var formatInputString = function (value) { - var result = utils.fromAscii(value).substr(2); + var result = utils.fromUtf8(value).substr(2); var length = result.length / 2; var l = Math.floor((result.length + 63) / 64); result = utils.padRight(result, l * 64); @@ -1082,7 +1082,7 @@ var formatOutputUInt = function (param) { * @returns {BigNumber} input bytes formatted to real */ var formatOutputReal = function (param) { - return formatOutputInt(param).dividedBy(new BigNumber(2).pow(128)); + return formatOutputInt(param).dividedBy(new BigNumber(2).pow(128)); }; /** @@ -1093,7 +1093,7 @@ var formatOutputReal = function (param) { * @returns {BigNumber} input bytes formatted to ureal */ var formatOutputUReal = function (param) { - return formatOutputUInt(param).dividedBy(new BigNumber(2).pow(128)); + return formatOutputUInt(param).dividedBy(new BigNumber(2).pow(128)); }; /** @@ -1139,7 +1139,7 @@ var formatOutputDynamicBytes = function (param) { */ var formatOutputString = function (param) { var length = (new BigNumber(param.dynamicPart().slice(0, 64), 16)).toNumber() * 2; - return utils.toAscii(param.dynamicPart().substr(64, length)); + return utils.toUtf8(param.dynamicPart().substr(64, length)); }; /** @@ -1228,7 +1228,7 @@ module.exports = SolidityTypeInt; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file param.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -1247,7 +1247,7 @@ var SolidityParam = function (value, offset) { /** * This method should be used to get length of params's dynamic part - * + * * @method dynamicPartLength * @returns {Number} length of dynamic part (in bytes) */ @@ -1275,7 +1275,7 @@ SolidityParam.prototype.withOffset = function (offset) { * @param {SolidityParam} result of combination */ SolidityParam.prototype.combine = function (param) { - return new SolidityParam(this.value + param.value); + return new SolidityParam(this.value + param.value); }; /** @@ -1307,8 +1307,8 @@ SolidityParam.prototype.offsetAsBytes = function () { */ SolidityParam.prototype.staticPart = function () { if (!this.isDynamic()) { - return this.value; - } + return this.value; + } return this.offsetAsBytes(); }; @@ -1340,7 +1340,7 @@ SolidityParam.prototype.encode = function () { * @returns {String} */ SolidityParam.encodeList = function (params) { - + // updating offsets var totalOffset = params.length * 32; var offsetParams = params.map(function (param) { @@ -1466,13 +1466,13 @@ SolidityType.prototype.staticPartLength = function (name) { /** * Should be used to determine if type is dynamic array - * eg: + * eg: * "type[]" => true * "type[4]" => false * * @method isDynamicArray * @param {String} name - * @return {Bool} true if the type is dynamic array + * @return {Bool} true if the type is dynamic array */ SolidityType.prototype.isDynamicArray = function (name) { var nestedTypes = this.nestedTypes(name); @@ -1481,13 +1481,13 @@ SolidityType.prototype.isDynamicArray = function (name) { /** * Should be used to determine if type is static array - * eg: + * eg: * "type[]" => false * "type[4]" => true * * @method isStaticArray * @param {String} name - * @return {Bool} true if the type is static array + * @return {Bool} true if the type is static array */ SolidityType.prototype.isStaticArray = function (name) { var nestedTypes = this.nestedTypes(name); @@ -1496,7 +1496,7 @@ SolidityType.prototype.isStaticArray = function (name) { /** * Should return length of static array - * eg. + * eg. * "int[32]" => 32 * "int256[14]" => 14 * "int[2][3]" => 3 @@ -1571,7 +1571,7 @@ SolidityType.prototype.nestedTypes = function (name) { * Should be used to encode the value * * @method encode - * @param {Object} value + * @param {Object} value * @param {String} name * @return {String} encoded value */ @@ -1585,7 +1585,7 @@ SolidityType.prototype.encode = function (value, name) { var result = []; result.push(f.formatInputInt(length).encode()); - + value.forEach(function (v) { result.push(self.encode(v, nestedName)); }); @@ -1659,12 +1659,12 @@ SolidityType.prototype.decode = function (bytes, offset, name) { return result; })(); } else if (this.isDynamicType(name)) { - + return (function () { var dynamicOffset = parseInt('0x' + bytes.substr(offset * 2, 64)); // in bytes var length = parseInt('0x' + bytes.substr(dynamicOffset * 2, 64)); // in bytes var roundedLength = Math.floor((length + 31) / 32); // in int - + return self._outputFormatter(new SolidityParam(bytes.substr(dynamicOffset * 2, ( 1 + roundedLength) * 64), 0)); })(); } @@ -1697,7 +1697,7 @@ var SolidityType = require('./type'); */ var SolidityTypeUInt = function () { this._inputFormatter = f.formatInputInt; - this._outputFormatter = f.formatOutputInt; + this._outputFormatter = f.formatOutputUInt; }; SolidityTypeUInt.prototype = new SolidityType({}); @@ -1787,13 +1787,13 @@ if (typeof XMLHttpRequest === 'undefined') { /** * Utils - * + * * @module utils */ /** * Utility functions - * + * * @class [utils] config * @constructor */ @@ -1860,7 +1860,7 @@ module.exports = { You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file sha3.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -1876,7 +1876,7 @@ module.exports = function (str, isNew) { console.warn('new usage: \'web3.sha3("hello")\''); console.warn('see https://github.com/ethereum/web3.js/pull/205'); console.warn('if you need to hash hex value, you can do \'sha3("0xfff", true)\''); - str = utils.toAscii(str); + str = utils.toUtf8(str); } return sha3(str, { @@ -1885,7 +1885,7 @@ module.exports = function (str, isNew) { }; -},{"./utils":20,"crypto-js/sha3":47}],20:[function(require,module,exports){ +},{"./utils":20,"crypto-js/sha3":48}],20:[function(require,module,exports){ /* This file is part of ethereum.js. @@ -1902,7 +1902,7 @@ module.exports = function (str, isNew) { You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file utils.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -1910,19 +1910,20 @@ module.exports = function (str, isNew) { /** * Utils - * + * * @module utils */ /** * Utility functions - * + * * @class [utils] utils * @constructor */ var BigNumber = require('bignumber.js'); +var utf8 = require('utf8'); var unitMap = { 'wei': '1', @@ -1977,9 +1978,30 @@ var padRight = function (string, chars, sign) { return string + (new Array(chars - string.length + 1).join(sign ? sign : "0")); }; -/** - * Should be called to get sting from it's hex representation - * TODO: it should be called toUTF8 +/** + * Should be called to get utf8 from it's hex representation + * + * @method toUtf8 + * @param {String} string in hex + * @returns {String} ascii string representation of hex value + */ +var toUtf8 = function(hex) { +// Find termination + var str = ""; + var i = 0, l = hex.length; + if (hex.substring(0, 2) === '0x') { + i = 2; + } + for (; i < l; i+=2) { + var code = parseInt(hex.substr(i, 2), 16); + str += String.fromCharCode(code); + } + + return utf8.decode(str); +}; + +/** + * Should be called to get ascii from it's hex representation * * @method toAscii * @param {String} string in hex @@ -1997,40 +2019,44 @@ var toAscii = function(hex) { str += String.fromCharCode(code); } - return decodeURIComponent(escape(str)); // jshint ignore:line + return str; }; - + /** - * Shold be called to get hex representation (prefixed by 0x) of ascii string + * Shold be called to get hex representation (prefixed by 0x) of utf8 string * - * @method toHexNative + * @method fromUtf8 * @param {String} string + * @param {Number} optional padding * @returns {String} hex representation of input string */ -var toHexNative = function(str) { - str = unescape(encodeURIComponent(str)); // jshint ignore:line +var fromUtf8 = function(str) { + str = utf8.encode(str); var hex = ""; for(var i = 0; i < str.length; i++) { var n = str.charCodeAt(i).toString(16); hex += n.length < 2 ? '0' + n : n; } - return hex; + return "0x" + hex; }; /** - * Shold be called to get hex representation (prefixed by 0x) of ascii string + * Shold be called to get hex representation (prefixed by 0x) of ascii string * * @method fromAscii * @param {String} string * @param {Number} optional padding * @returns {String} hex representation of input string */ -var fromAscii = function(str, pad) { - pad = pad === undefined ? 0 : pad; - var hex = toHexNative(str); - while (hex.length < pad*2) - hex += "00"; +var fromAscii = function(str) { + var hex = ""; + for(var i = 0; i < str.length; i++) { + var code = str.charCodeAt(i); + var n = code.toString(16); + hex += n.length < 2 ? '0' + n : n; + } + return "0x" + hex; }; @@ -2052,13 +2078,13 @@ var transformToFullName = function (json) { /** * Should be called to get display name of contract function - * + * * @method extractDisplayName * @param {String} name of function/event * @returns {String} display name for function/event eg. multiply(uint256) -> multiply */ var extractDisplayName = function (name) { - var length = name.indexOf('('); + var length = name.indexOf('('); return length !== -1 ? name.substr(0, length) : name; }; @@ -2113,7 +2139,7 @@ var toHex = function (val) { return fromDecimal(val); if (isObject(val)) - return fromAscii(JSON.stringify(val)); + return fromUtf8(JSON.stringify(val)); // if its a negative number, pass it through fromDecimal if (isString(val)) { @@ -2156,7 +2182,7 @@ var getValueOfUnit = function (unit) { * - -- microether szabo micro * - -- milliether finney milli * - ether -- -- - * - kether einstein grand + * - kether einstein grand * - mether * - gether * - tether @@ -2169,7 +2195,7 @@ var getValueOfUnit = function (unit) { var fromWei = function(number, unit) { var returnValue = toBigNumber(number).dividedBy(getValueOfUnit(unit)); - return isBigNumber(number) ? returnValue : returnValue.toString(10); + return isBigNumber(number) ? returnValue : returnValue.toString(10); }; /** @@ -2178,12 +2204,12 @@ var fromWei = function(number, unit) { * Possible units are: * SI Short SI Full Effigy Other * - kwei femtoether ada - * - mwei picoether babbage + * - mwei picoether babbage * - gwei nanoether shannon nano * - -- microether szabo micro * - -- milliether finney milli * - ether -- -- - * - kether einstein grand + * - kether einstein grand * - mether * - gether * - tether @@ -2196,7 +2222,7 @@ var fromWei = function(number, unit) { var toWei = function(number, unit) { var returnValue = toBigNumber(number).times(getValueOfUnit(unit)); - return isBigNumber(number) ? returnValue : returnValue.toString(10); + return isBigNumber(number) ? returnValue : returnValue.toString(10); }; /** @@ -2215,7 +2241,7 @@ var toBigNumber = function(number) { if (isString(number) && (number.indexOf('0x') === 0 || number.indexOf('-0x') === 0)) { return new BigNumber(number.replace('0x',''), 16); } - + return new BigNumber(number.toString(10), 10); }; @@ -2242,7 +2268,7 @@ var toTwosComplement = function (number) { * @return {Boolean} */ var isStrictAddress = function (address) { - return /^0x[0-9a-f]{40}$/.test(address); + return /^0x[0-9a-f]{40}$/i.test(address); }; /** @@ -2253,7 +2279,7 @@ var isStrictAddress = function (address) { * @return {Boolean} */ var isAddress = function (address) { - return /^(0x)?[0-9a-f]{40}$/.test(address); + return /^(0x)?[0-9a-f]{40}$/i.test(address); }; /** @@ -2267,7 +2293,7 @@ var toAddress = function (address) { if (isStrictAddress(address)) { return address; } - + if (/^[0-9a-f]{40}$/.test(address)) { return '0x' + address; } @@ -2281,7 +2307,7 @@ var toAddress = function (address) { * * @method isBigNumber * @param {Object} - * @return {Boolean} + * @return {Boolean} */ var isBigNumber = function (object) { return object instanceof BigNumber || @@ -2290,7 +2316,7 @@ var isBigNumber = function (object) { /** * Returns true if object is string, otherwise false - * + * * @method isString * @param {Object} * @return {Boolean} @@ -2341,12 +2367,12 @@ var isBoolean = function (object) { * @return {Boolean} */ var isArray = function (object) { - return object instanceof Array; + return object instanceof Array; }; /** * Returns true if given string is valid json object - * + * * @method isJson * @param {String} * @return {Boolean} @@ -2365,7 +2391,9 @@ module.exports = { toHex: toHex, toDecimal: toDecimal, fromDecimal: fromDecimal, + toUtf8: toUtf8, toAscii: toAscii, + fromUtf8: fromUtf8, fromAscii: fromAscii, transformToFullName: transformToFullName, extractDisplayName: extractDisplayName, @@ -2386,10 +2414,9 @@ module.exports = { isJson: isJson }; - -},{"bignumber.js":"bignumber.js"}],21:[function(require,module,exports){ +},{"bignumber.js":"bignumber.js","utf8":50}],21:[function(require,module,exports){ module.exports={ - "version": "0.12.1" + "version": "0.13.0" } },{}],22:[function(require,module,exports){ @@ -2426,6 +2453,7 @@ var db = require('./web3/methods/db'); var shh = require('./web3/methods/shh'); var watches = require('./web3/methods/watches'); var Filter = require('./web3/filter'); +var IsSyncing = require('./web3/syncing'); var utils = require('./utils/utils'); var formatters = require('./web3/formatters'); var RequestManager = require('./web3/requestmanager'); @@ -2480,6 +2508,10 @@ web3.version = {}; web3.version.api = version.version; web3.eth = {}; +web3.eth.isSyncing = function (callback) { + return new IsSyncing(callback); +}; + /*jshint maxparams:4 */ web3.eth.filter = function (fil, callback) { return new Filter(fil, watches.eth(), formatters.outputLogFormatter, callback); @@ -2499,14 +2531,16 @@ web3.setProvider = function (provider) { web3.isConnected = function(){ return (this.currentProvider && this.currentProvider.isConnected()); }; -web3.reset = function () { - RequestManager.getInstance().reset(); +web3.reset = function (keepIsSyncing) { + RequestManager.getInstance().reset(keepIsSyncing); c.defaultBlock = 'latest'; c.defaultAccount = undefined; }; web3.toHex = utils.toHex; web3.toAscii = utils.toAscii; +web3.toUtf8 = utils.toUtf8; web3.fromAscii = utils.fromAscii; +web3.fromUtf8 = utils.fromUtf8; web3.toDecimal = utils.toDecimal; web3.fromDecimal = utils.fromDecimal; web3.toBigNumber = utils.toBigNumber; @@ -2569,7 +2603,7 @@ setupMethods(web3.shh, shh.methods); module.exports = web3; -},{"./utils/config":18,"./utils/sha3":19,"./utils/utils":20,"./version.json":21,"./web3/batch":24,"./web3/filter":28,"./web3/formatters":29,"./web3/method":35,"./web3/methods/db":36,"./web3/methods/eth":37,"./web3/methods/net":38,"./web3/methods/shh":39,"./web3/methods/watches":40,"./web3/property":42,"./web3/requestmanager":43}],23:[function(require,module,exports){ +},{"./utils/config":18,"./utils/sha3":19,"./utils/utils":20,"./version.json":21,"./web3/batch":24,"./web3/filter":28,"./web3/formatters":29,"./web3/method":35,"./web3/methods/db":36,"./web3/methods/eth":37,"./web3/methods/net":38,"./web3/methods/shh":39,"./web3/methods/watches":40,"./web3/property":42,"./web3/requestmanager":43,"./web3/syncing":44}],23:[function(require,module,exports){ /* This file is part of ethereum.js. @@ -2586,7 +2620,7 @@ module.exports = web3; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file allevents.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2014 @@ -2675,7 +2709,7 @@ module.exports = AllSolidityEvents; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file batch.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -2720,7 +2754,7 @@ Batch.prototype.execute = function () { requests[index].callback(null, (requests[index].format ? requests[index].format(result.result) : result.result)); } }); - }); + }); }; module.exports = Batch; @@ -2743,13 +2777,13 @@ module.exports = Batch; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file contract.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2014 */ -var web3 = require('../web3'); +var web3 = require('../web3'); var utils = require('../utils/utils'); var coder = require('../solidity/coder'); var SolidityEvent = require('./event'); @@ -2806,7 +2840,7 @@ var addEventsToContract = function (contract, abi) { var All = new AllEvents(events, contract.address); All.attachToContract(contract); - + events.map(function (json) { return new SolidityEvent(json, contract.address); }).forEach(function (e) { @@ -2846,7 +2880,7 @@ var checkForContractAddress = function(contract, abi, callback){ // stop watching after 50 blocks (timeout) if(count > 50) { - + filter.stopWatching(); callbackFired = true; @@ -2866,7 +2900,7 @@ var checkForContractAddress = function(contract, abi, callback){ if(callbackFired) return; - + filter.stopWatching(); callbackFired = true; @@ -2910,7 +2944,7 @@ var ContractFactory = function (abi) { /** * Should be called to create new contract on a blockchain - * + * * @method new * @param {Any} contract constructor param1 (optional) * @param {Any} contract constructor param2 (optional) @@ -2984,10 +3018,10 @@ ContractFactory.prototype.at = function (address, callback) { // attach functions addFunctionsToContract(contract, this.abi); addEventsToContract(contract, this.abi); - + if (callback) { callback(null, contract); - } + } return contract; }; @@ -3022,7 +3056,7 @@ module.exports = contract; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file errors.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -3062,7 +3096,7 @@ module.exports = { You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file event.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2014 @@ -3132,7 +3166,7 @@ SolidityEvent.prototype.signature = function () { /** * Should be used to encode indexed params and options to one final object - * + * * @method encode * @param {Object} indexed * @param {Object} options @@ -3163,7 +3197,7 @@ SolidityEvent.prototype.encode = function (indexed, options) { if (value === undefined || value === null) { return null; } - + if (utils.isArray(value)) { return value.map(function (v) { return '0x' + coder.encodeParam(i.type, v); @@ -3185,17 +3219,17 @@ SolidityEvent.prototype.encode = function (indexed, options) { * @return {Object} result object with decoded indexed && not indexed params */ SolidityEvent.prototype.decode = function (data) { - + data.data = data.data || ''; data.topics = data.topics || []; var argTopics = this._anonymous ? data.topics : data.topics.slice(1); var indexedData = argTopics.map(function (topics) { return topics.slice(2); }).join(""); - var indexedParams = coder.decodeParams(this.types(true), indexedData); + var indexedParams = coder.decodeParams(this.types(true), indexedData); var notIndexedData = data.data.slice(2); var notIndexedParams = coder.decodeParams(this.types(false), notIndexedData); - + var result = formatters.outputLogFormatter(data); result.event = this.displayName(); result.address = data.address; @@ -3230,7 +3264,7 @@ SolidityEvent.prototype.execute = function (indexed, options, callback) { indexed = {}; } } - + var o = this.encode(indexed, options); var formatter = this.decode.bind(this); return new Filter(o, watches.eth(), formatter, callback); @@ -3301,7 +3335,7 @@ var toTopic = function(value){ if(value.indexOf('0x') === 0) return value; else - return utils.fromAscii(value); + return utils.fromUtf8(value); }; /// This method should be called on options object, to verify deprecated properties && lazy load dynamic ones @@ -3311,7 +3345,7 @@ var getOptions = function (options) { if (utils.isString(options)) { return options; - } + } options = options || {}; @@ -3327,8 +3361,8 @@ var getOptions = function (options) { to: options.to, address: options.address, fromBlock: formatters.inputBlockNumberFormatter(options.fromBlock), - toBlock: formatters.inputBlockNumberFormatter(options.toBlock) - }; + toBlock: formatters.inputBlockNumberFormatter(options.toBlock) + }; }; /** @@ -3336,7 +3370,7 @@ Adds the callback and sets up the methods, to iterate over the results. @method getLogsAtStart @param {Object} self -@param {funciton} +@param {funciton} */ var getLogsAtStart = function(self, callback){ // call getFilterLogs for the first watch callback start @@ -3371,12 +3405,14 @@ var pollFilter = function(self) { }); } - messages.forEach(function (message) { - message = self.formatter ? self.formatter(message) : message; - self.callbacks.forEach(function (callback) { - callback(null, message); + if(utils.isArray(messages)) { + messages.forEach(function (message) { + message = self.formatter ? self.formatter(message) : message; + self.callbacks.forEach(function (callback) { + callback(null, message); + }); }); - }); + } }; RequestManager.getInstance().startPolling({ @@ -3396,6 +3432,7 @@ var Filter = function (options, methods, formatter, callback) { this.implementation = implementation; this.filterId = null; this.callbacks = []; + this.getLogsCallbacks = []; this.pollFilters = []; this.formatter = formatter; this.implementation.newFilter(this.options, function(error, id){ @@ -3406,6 +3443,13 @@ var Filter = function (options, methods, formatter, callback) { } else { self.filterId = id; + // check if there are get pending callbacks as a consequence + // of calling get() with filterId unassigned. + self.getLogsCallbacks.forEach(function (cb){ + self.get(cb); + }); + self.getLogsCallbacks = []; + // get filter logs for the already existing watch calls self.callbacks.forEach(function(cb){ getLogsAtStart(self, cb); @@ -3444,16 +3488,25 @@ Filter.prototype.stopWatching = function () { Filter.prototype.get = function (callback) { var self = this; if (utils.isFunction(callback)) { - this.implementation.getLogs(this.filterId, function(err, res){ - if (err) { - callback(err); - } else { - callback(null, res.map(function (log) { - return self.formatter ? self.formatter(log) : log; - })); - } - }); + if (this.filterId === null) { + // If filterId is not set yet, call it back + // when newFilter() assigns it. + this.getLogsCallbacks.push(callback); + } else { + this.implementation.getLogs(this.filterId, function(err, res){ + if (err) { + callback(err); + } else { + callback(null, res.map(function (log) { + return self.formatter ? self.formatter(log) : log; + })); + } + }); + } } else { + if (this.filterId === null) { + throw new Error('Filter ID Error: filter().get() can\'t be chained synchronous, please provide a callback for the get() method.'); + } var logs = this.implementation.getLogs(this.filterId); return logs.map(function (log) { return self.formatter ? self.formatter(log) : log; @@ -3483,7 +3536,7 @@ module.exports = Filter; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file formatters.js * @author Marek Kotewicz <marek@ethdev.com> * @author Fabian Vogelsteller <fabian@ethdev.com> @@ -3550,7 +3603,7 @@ var inputCallFormatter = function (options){ options[key] = utils.fromDecimal(options[key]); }); - return options; + return options; }; /** @@ -3575,12 +3628,12 @@ var inputTransactionFormatter = function (options){ options[key] = utils.fromDecimal(options[key]); }); - return options; + return options; }; /** * Formats the output of a transaction to its proper values - * + * * @method outputTransactionFormatter * @param {Object} tx * @returns {Object} @@ -3599,7 +3652,7 @@ var outputTransactionFormatter = function (tx){ /** * Formats the output of a transaction receipt to its proper values - * + * * @method outputTransactionReceiptFormatter * @param {Object} receipt * @returns {Object} @@ -3625,7 +3678,7 @@ var outputTransactionReceiptFormatter = function (receipt){ * Formats the output of a block to its proper values * * @method outputBlockFormatter - * @param {Object} block + * @param {Object} block * @returns {Object} */ var outputBlockFormatter = function(block) { @@ -3653,7 +3706,7 @@ var outputBlockFormatter = function(block) { /** * Formats the output of a log - * + * * @method outputLogFormatter * @param {Object} log object * @returns {Object} log @@ -3690,10 +3743,10 @@ var inputPostFormatter = function(post) { // format the following options post.topics = post.topics.map(function(topic){ - return utils.fromAscii(topic); + return utils.fromUtf8(topic); }); - return post; + return post; }; /** @@ -3710,7 +3763,7 @@ var outputPostFormatter = function(post){ post.ttl = utils.toDecimal(post.ttl); post.workProved = utils.toDecimal(post.workProved); post.payloadRaw = post.payload; - post.payload = utils.toAscii(post.payload); + post.payload = utils.toUtf8(post.payload); if (utils.isJson(post.payload)) { post.payload = JSON.parse(post.payload); @@ -3721,7 +3774,7 @@ var outputPostFormatter = function(post){ post.topics = []; } post.topics = post.topics.map(function(topic){ - return utils.toAscii(topic); + return utils.toUtf8(topic); }); return post; @@ -3739,6 +3792,16 @@ var inputAddressFormatter = function (address) { throw 'invalid address'; }; + +var outputSyncingFormatter = function(result) { + + result.startingBlock = utils.toDecimal(result.startingBlock); + result.currentBlock = utils.toDecimal(result.currentBlock); + result.highestBlock = utils.toDecimal(result.highestBlock); + + return result; +}; + module.exports = { inputDefaultBlockNumberFormatter: inputDefaultBlockNumberFormatter, inputBlockNumberFormatter: inputBlockNumberFormatter, @@ -3751,7 +3814,8 @@ module.exports = { outputTransactionReceiptFormatter: outputTransactionReceiptFormatter, outputBlockFormatter: outputBlockFormatter, outputLogFormatter: outputLogFormatter, - outputPostFormatter: outputPostFormatter + outputPostFormatter: outputPostFormatter, + outputSyncingFormatter: outputSyncingFormatter }; @@ -3869,8 +3933,8 @@ SolidityFunction.prototype.call = function () { if (!callback) { var output = web3.eth.call(payload, defaultBlock); return this.unpackOutput(output); - } - + } + var self = this; web3.eth.call(payload, defaultBlock, function (error, output) { callback(error, self.unpackOutput(output)); @@ -3944,11 +4008,11 @@ SolidityFunction.prototype.request = function () { var callback = this.extractCallback(args); var payload = this.toPayload(args); var format = this.unpackOutput.bind(this); - + return { method: this._constant ? 'eth_call' : 'eth_sendTransaction', callback: callback, - params: [payload], + params: [payload], format: format }; }; @@ -4079,7 +4143,7 @@ HttpProvider.prototype.send = function (payload) { try { result = JSON.parse(result); } catch(e) { - throw errors.InvalidResponse(request.responseText); + throw errors.InvalidResponse(request.responseText); } return result; @@ -4093,7 +4157,7 @@ HttpProvider.prototype.send = function (payload) { * @param {Function} callback triggered on end with (err, result) */ HttpProvider.prototype.sendAsync = function (payload, callback) { - var request = this.prepareRequest(true); + var request = this.prepareRequest(true); request.onreadystatechange = function() { if (request.readyState === 4) { @@ -4103,13 +4167,13 @@ HttpProvider.prototype.sendAsync = function (payload, callback) { try { result = JSON.parse(result); } catch(e) { - error = errors.InvalidResponse(request.responseText); + error = errors.InvalidResponse(request.responseText); } callback(error, result); } }; - + try { request.send(JSON.stringify(payload)); } catch(error) { @@ -4157,7 +4221,7 @@ module.exports = HttpProvider; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file iban.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -4289,7 +4353,7 @@ Iban.isValid = function (iban) { * @returns {Boolean} true if it is, otherwise false */ Iban.prototype.isValid = function () { - return /^XE[0-9]{2}(ETH[0-9A-Z]{13}|[0-9A-Z]{30})$/.test(this._iban) && + return /^XE[0-9]{2}(ETH[0-9A-Z]{13}|[0-9A-Z]{30,31})$/.test(this._iban) && mod9710(iso13616Prepare(this._iban)) === 1; }; @@ -4357,7 +4421,7 @@ Iban.prototype.address = function () { var base36 = this._iban.substr(4); var asBn = new BigNumber(base36, 36); return padLeft(asBn.toString(16), 20); - } + } return ''; }; @@ -4401,9 +4465,9 @@ var errorTimeout = function (method, id) { var err = { "jsonrpc": "2.0", "error": { - "code": -32603, + "code": -32603, "message": "IPC Request timed out for method \'" + method + "\'" - }, + }, "id": id }; return JSON.stringify(err); @@ -4413,7 +4477,7 @@ var IpcProvider = function (path, net) { var _this = this; this.responseCallbacks = {}; this.path = path; - + this.connection = net.connect({path: this.path}); this.connection.on('error', function(e){ @@ -4423,7 +4487,7 @@ var IpcProvider = function (path, net) { this.connection.on('end', function(){ _this._timeout(); - }); + }); // LISTEN FOR CONNECTION RESPONSES @@ -4462,7 +4526,7 @@ Will parse the response and make an array out of it. IpcProvider.prototype._parseResponse = function(data) { var _this = this, returnValues = []; - + // DE-CHUNKER var dechunkedData = data .replace(/\}\{/g,'}|--|{') // }{ @@ -4566,7 +4630,7 @@ IpcProvider.prototype.send = function (payload) { try { result = JSON.parse(data); } catch(e) { - throw errors.InvalidResponse(data); + throw errors.InvalidResponse(data); } return result; @@ -4743,7 +4807,7 @@ Method.prototype.extractCallback = function (args) { /** * Should be called to check if the number of arguments is correct - * + * * @method validateArgs * @param {Array} arguments * @throws {Error} if it is not @@ -4756,7 +4820,7 @@ Method.prototype.validateArgs = function (args) { /** * Should be called to format input args of method - * + * * @method formatInput * @param {Array} * @return {Array} @@ -4784,7 +4848,7 @@ Method.prototype.formatOutput = function (result) { /** * Should attach function to method - * + * * @method attachToObject * @param {Object} * @param {Function} @@ -4798,7 +4862,7 @@ Method.prototype.attachToObject = function (obj) { obj[name[0]] = obj[name[0]] || {}; obj[name[0]][name[1]] = func; } else { - obj[name[0]] = func; + obj[name[0]] = func; } }; @@ -5186,6 +5250,11 @@ var properties = [ outputFormatter: utils.toDecimal }), new Property({ + name: 'syncing', + getter: 'eth_syncing', + outputFormatter: formatters.outputSyncingFormatter + }), + new Property({ name: 'gasPrice', getter: 'eth_gasPrice', outputFormatter: formatters.outputBigNumberFormatter @@ -5284,8 +5353,8 @@ var Method = require('../method'); var formatters = require('../formatters'); var post = new Method({ - name: 'post', - call: 'shh_post', + name: 'post', + call: 'shh_post', params: 1, inputFormatter: [formatters.inputPostFormatter] }); @@ -5460,7 +5529,7 @@ module.exports = { You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file namereg.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -5516,7 +5585,7 @@ var Property = function (options) { /** * Should be called to format input args of method - * + * * @method formatInput * @param {Array} * @return {Array} @@ -5551,7 +5620,7 @@ Property.prototype.extractCallback = function (args) { /** * Should attach function to method - * + * * @method attachToObject * @param {Object} * @param {Function} @@ -5568,7 +5637,7 @@ Property.prototype.attachToObject = function (obj) { obj = obj[names[0]]; name = names[1]; } - + Object.defineProperty(obj, name, proto); var toAsyncName = function (prefix, name) { @@ -5648,7 +5717,7 @@ module.exports = Property; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** * @file requestmanager.js * @author Jeffrey Wilcke <jeff@ethdev.com> * @author Marek Kotewicz <marek@ethdev.com> @@ -5730,7 +5799,7 @@ RequestManager.prototype.sendAsync = function (data, callback) { if (err) { return callback(err); } - + if (!Jsonrpc.getInstance().isValidResponse(result)) { return callback(errors.InvalidResponse(result)); } @@ -5763,7 +5832,7 @@ RequestManager.prototype.sendBatch = function (data, callback) { } callback(err, results); - }); + }); }; /** @@ -5811,11 +5880,15 @@ RequestManager.prototype.stopPolling = function (pollId) { * * @method reset */ -RequestManager.prototype.reset = function () { +RequestManager.prototype.reset = function (keepIsSyncing) { for (var key in this.polls) { - this.polls[key].uninstall(); + // remove all polls, except sync polls, + // they need to be removed manually by calling syncing.stopWatching() + if(!keepIsSyncing || key.indexOf('syncPoll_') === -1) { + this.polls[key].uninstall(); + delete this.polls[key]; + } } - this.polls = {}; if (this.timeout) { clearTimeout(this.timeout); @@ -5843,10 +5916,10 @@ RequestManager.prototype.poll = function () { } var pollsData = []; - var pollsKeys = []; + var pollsIds = []; for (var key in this.polls) { pollsData.push(this.polls[key].data); - pollsKeys.push(key); + pollsIds.push(key); } if (pollsData.length === 0) { @@ -5855,8 +5928,17 @@ RequestManager.prototype.poll = function () { var payload = Jsonrpc.getInstance().toBatchPayload(pollsData); + // map the request id to they poll id + var pollsIdMap = {}; + payload.forEach(function(load, index){ + pollsIdMap[load.id] = pollsIds[index]; + }); + + var self = this; this.provider.sendAsync(payload, function (error, results) { + + // TODO: console log? if (error) { return; @@ -5865,25 +5947,23 @@ RequestManager.prototype.poll = function () { if (!utils.isArray(results)) { throw errors.InvalidResponse(results); } + results.map(function (result) { + var id = pollsIdMap[result.id]; - results.map(function (result, index) { - var key = pollsKeys[index]; // make sure the filter is still installed after arrival of the request - if (self.polls[key]) { - result.callback = self.polls[key].callback; + if (self.polls[id]) { + result.callback = self.polls[id].callback; return result; } else return false; }).filter(function (result) { - return !!result; + return !!result; }).filter(function (result) { var valid = Jsonrpc.getInstance().isValidResponse(result); if (!valid) { result.callback(errors.InvalidResponse(result)); } return valid; - }).filter(function (result) { - return utils.isArray(result.result) && result.result.length > 0; }).forEach(function (result) { result.callback(null, result.result); }); @@ -5910,7 +5990,110 @@ module.exports = RequestManager; You should have received a copy of the GNU Lesser General Public License along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. */ -/** +/** @file syncing.js + * @authors: + * Fabian Vogelsteller <fabian@ethdev.com> + * @date 2015 + */ + +var RequestManager = require('./requestmanager'); +var Method = require('./method'); +var formatters = require('./formatters'); +var utils = require('../utils/utils'); + + + +/** +Adds the callback and sets up the methods, to iterate over the results. + +@method pollSyncing +@param {Object} self +*/ +var pollSyncing = function(self) { + var lastSyncState = false; + + var onMessage = function (error, sync) { + if (error) { + return self.callbacks.forEach(function (callback) { + callback(error); + }); + } + + if(utils.isObject(sync)) + sync = self.implementation.outputFormatter(sync); + + self.callbacks.forEach(function (callback) { + if(lastSyncState !== sync) { + + // call the callback with true first so the app can stop anything, before receiving the sync data + if(!lastSyncState && utils.isObject(sync)) + callback(null, true); + + // call on the next CPU cycle, so the actions of the sync stop can be processes first + setTimeout(function() { + callback(null, sync); + }, 1); + + lastSyncState = sync; + } + }); + }; + + RequestManager.getInstance().startPolling({ + method: self.implementation.call, + params: [], + }, self.pollId, onMessage, self.stopWatching.bind(self)); + +}; + +var IsSyncing = function (callback) { + this.pollId = 'syncPoll_'+ Math.floor(Math.random() * 1000); + this.callbacks = []; + this.implementation = new Method({ + name: 'isSyncing', + call: 'eth_syncing', + params: 0, + outputFormatter: formatters.outputSyncingFormatter + }); + + this.addCallback(callback); + pollSyncing(this); + + return this; +}; + +IsSyncing.prototype.addCallback = function (callback) { + if(callback) + this.callbacks.push(callback); + return this; +}; + +IsSyncing.prototype.stopWatching = function () { + RequestManager.getInstance().stopPolling(this.pollId); + this.callbacks = []; +}; + +module.exports = IsSyncing; + + +},{"../utils/utils":20,"./formatters":29,"./method":35,"./requestmanager":43}],45:[function(require,module,exports){ +/* + This file is part of ethereum.js. + + ethereum.js is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ethereum.js is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with ethereum.js. If not, see <http://www.gnu.org/licenses/>. +*/ +/** * @file transfer.js * @author Marek Kotewicz <marek@ethdev.com> * @date 2015 @@ -5932,7 +6115,7 @@ var exchangeAbi = require('../contracts/SmartExchange.json'); * @param {Function} callback, callback */ var transfer = function (from, to, value, callback) { - var iban = new Iban(to); + var iban = new Iban(to); if (!iban.isValid()) { throw new Error('invalid iban address'); } @@ -5940,7 +6123,7 @@ var transfer = function (from, to, value, callback) { if (iban.isDirect()) { return transferToAddress(from, iban.address(), value, callback); } - + if (!callback) { var address = namereg.addr(iban.institution()); return deposit(from, address, value, iban.client()); @@ -5949,7 +6132,7 @@ var transfer = function (from, to, value, callback) { namereg.addr(iban.institution(), function (err, address) { return deposit(from, address, value, iban.client(), callback); }); - + }; /** @@ -5990,9 +6173,9 @@ var deposit = function (from, to, value, client, callback) { module.exports = transfer; -},{"../contracts/SmartExchange.json":3,"../web3":22,"./contract":25,"./iban":32,"./namereg":41}],45:[function(require,module,exports){ +},{"../contracts/SmartExchange.json":3,"../web3":22,"./contract":25,"./iban":32,"./namereg":41}],46:[function(require,module,exports){ -},{}],46:[function(require,module,exports){ +},{}],47:[function(require,module,exports){ ;(function (root, factory) { if (typeof exports === "object") { // CommonJS @@ -6735,7 +6918,7 @@ module.exports = transfer; return CryptoJS; })); -},{}],47:[function(require,module,exports){ +},{}],48:[function(require,module,exports){ ;(function (root, factory, undef) { if (typeof exports === "object") { // CommonJS @@ -7059,7 +7242,7 @@ module.exports = transfer; return CryptoJS.SHA3; })); -},{"./core":46,"./x64-core":48}],48:[function(require,module,exports){ +},{"./core":47,"./x64-core":49}],49:[function(require,module,exports){ ;(function (root, factory) { if (typeof exports === "object") { // CommonJS @@ -7364,7 +7547,253 @@ module.exports = transfer; return CryptoJS; })); -},{"./core":46}],"bignumber.js":[function(require,module,exports){ +},{"./core":47}],50:[function(require,module,exports){ +/*! https://mths.be/utf8js v2.0.0 by @mathias */ +;(function(root) { + + // Detect free variables 'exports' + var freeExports = typeof exports == 'object' && exports; + + // Detect free variable 'module' + var freeModule = typeof module == 'object' && module && + module.exports == freeExports && module; + + // Detect free variable 'global', from Node.js or Browserified code, + // and use it as 'root' + var freeGlobal = typeof global == 'object' && global; + if (freeGlobal.global === freeGlobal || freeGlobal.window === freeGlobal) { + root = freeGlobal; + } + + /*--------------------------------------------------------------------------*/ + + var stringFromCharCode = String.fromCharCode; + + // Taken from https://mths.be/punycode + function ucs2decode(string) { + var output = []; + var counter = 0; + var length = string.length; + var value; + var extra; + while (counter < length) { + value = string.charCodeAt(counter++); + if (value >= 0xD800 && value <= 0xDBFF && counter < length) { + // high surrogate, and there is a next character + extra = string.charCodeAt(counter++); + if ((extra & 0xFC00) == 0xDC00) { // low surrogate + output.push(((value & 0x3FF) << 10) + (extra & 0x3FF) + 0x10000); + } else { + // unmatched surrogate; only append this code unit, in case the next + // code unit is the high surrogate of a surrogate pair + output.push(value); + counter--; + } + } else { + output.push(value); + } + } + return output; + } + + // Taken from https://mths.be/punycode + function ucs2encode(array) { + var length = array.length; + var index = -1; + var value; + var output = ''; + while (++index < length) { + value = array[index]; + if (value > 0xFFFF) { + value -= 0x10000; + output += stringFromCharCode(value >>> 10 & 0x3FF | 0xD800); + value = 0xDC00 | value & 0x3FF; + } + output += stringFromCharCode(value); + } + return output; + } + + function checkScalarValue(codePoint) { + if (codePoint >= 0xD800 && codePoint <= 0xDFFF) { + throw Error( + 'Lone surrogate U+' + codePoint.toString(16).toUpperCase() + + ' is not a scalar value' + ); + } + } + /*--------------------------------------------------------------------------*/ + + function createByte(codePoint, shift) { + return stringFromCharCode(((codePoint >> shift) & 0x3F) | 0x80); + } + + function encodeCodePoint(codePoint) { + if ((codePoint & 0xFFFFFF80) == 0) { // 1-byte sequence + return stringFromCharCode(codePoint); + } + var symbol = ''; + if ((codePoint & 0xFFFFF800) == 0) { // 2-byte sequence + symbol = stringFromCharCode(((codePoint >> 6) & 0x1F) | 0xC0); + } + else if ((codePoint & 0xFFFF0000) == 0) { // 3-byte sequence + checkScalarValue(codePoint); + symbol = stringFromCharCode(((codePoint >> 12) & 0x0F) | 0xE0); + symbol += createByte(codePoint, 6); + } + else if ((codePoint & 0xFFE00000) == 0) { // 4-byte sequence + symbol = stringFromCharCode(((codePoint >> 18) & 0x07) | 0xF0); + symbol += createByte(codePoint, 12); + symbol += createByte(codePoint, 6); + } + symbol += stringFromCharCode((codePoint & 0x3F) | 0x80); + return symbol; + } + + function utf8encode(string) { + var codePoints = ucs2decode(string); + var length = codePoints.length; + var index = -1; + var codePoint; + var byteString = ''; + while (++index < length) { + codePoint = codePoints[index]; + byteString += encodeCodePoint(codePoint); + } + return byteString; + } + + /*--------------------------------------------------------------------------*/ + + function readContinuationByte() { + if (byteIndex >= byteCount) { + throw Error('Invalid byte index'); + } + + var continuationByte = byteArray[byteIndex] & 0xFF; + byteIndex++; + + if ((continuationByte & 0xC0) == 0x80) { + return continuationByte & 0x3F; + } + + // If we end up here, it’s not a continuation byte + throw Error('Invalid continuation byte'); + } + + function decodeSymbol() { + var byte1; + var byte2; + var byte3; + var byte4; + var codePoint; + + if (byteIndex > byteCount) { + throw Error('Invalid byte index'); + } + + if (byteIndex == byteCount) { + return false; + } + + // Read first byte + byte1 = byteArray[byteIndex] & 0xFF; + byteIndex++; + + // 1-byte sequence (no continuation bytes) + if ((byte1 & 0x80) == 0) { + return byte1; + } + + // 2-byte sequence + if ((byte1 & 0xE0) == 0xC0) { + var byte2 = readContinuationByte(); + codePoint = ((byte1 & 0x1F) << 6) | byte2; + if (codePoint >= 0x80) { + return codePoint; + } else { + throw Error('Invalid continuation byte'); + } + } + + // 3-byte sequence (may include unpaired surrogates) + if ((byte1 & 0xF0) == 0xE0) { + byte2 = readContinuationByte(); + byte3 = readContinuationByte(); + codePoint = ((byte1 & 0x0F) << 12) | (byte2 << 6) | byte3; + if (codePoint >= 0x0800) { + checkScalarValue(codePoint); + return codePoint; + } else { + throw Error('Invalid continuation byte'); + } + } + + // 4-byte sequence + if ((byte1 & 0xF8) == 0xF0) { + byte2 = readContinuationByte(); + byte3 = readContinuationByte(); + byte4 = readContinuationByte(); + codePoint = ((byte1 & 0x0F) << 0x12) | (byte2 << 0x0C) | + (byte3 << 0x06) | byte4; + if (codePoint >= 0x010000 && codePoint <= 0x10FFFF) { + return codePoint; + } + } + + throw Error('Invalid UTF-8 detected'); + } + + var byteArray; + var byteCount; + var byteIndex; + function utf8decode(byteString) { + byteArray = ucs2decode(byteString); + byteCount = byteArray.length; + byteIndex = 0; + var codePoints = []; + var tmp; + while ((tmp = decodeSymbol()) !== false) { + codePoints.push(tmp); + } + return ucs2encode(codePoints); + } + + /*--------------------------------------------------------------------------*/ + + var utf8 = { + 'version': '2.0.0', + 'encode': utf8encode, + 'decode': utf8decode + }; + + // Some AMD build optimizers, like r.js, check for specific condition patterns + // like the following: + if ( + typeof define == 'function' && + typeof define.amd == 'object' && + define.amd + ) { + define(function() { + return utf8; + }); + } else if (freeExports && !freeExports.nodeType) { + if (freeModule) { // in Node.js or RingoJS v0.8.0+ + freeModule.exports = utf8; + } else { // in Narwhal or RingoJS v0.7.0- + var object = {}; + var hasOwnProperty = object.hasOwnProperty; + for (var key in utf8) { + hasOwnProperty.call(utf8, key) && (freeExports[key] = utf8[key]); + } + } + } else { // in Rhino or a web browser + root.utf8 = utf8; + } + +}(this)); + +},{}],"bignumber.js":[function(require,module,exports){ 'use strict'; module.exports = BigNumber; // jshint ignore:line @@ -7391,6 +7820,6 @@ if (typeof window !== 'undefined' && typeof window.web3 === 'undefined') { module.exports = web3; -},{"./lib/web3":22,"./lib/web3/contract":25,"./lib/web3/httpprovider":31,"./lib/web3/iban":32,"./lib/web3/ipcprovider":33,"./lib/web3/namereg":41,"./lib/web3/transfer":44}]},{},["web3"]) +},{"./lib/web3":22,"./lib/web3/contract":25,"./lib/web3/httpprovider":31,"./lib/web3/iban":32,"./lib/web3/ipcprovider":33,"./lib/web3/namereg":41,"./lib/web3/transfer":45}]},{},["web3"]) //# sourceMappingURL=web3-light.js.map ` diff --git a/jsre/jsre.go b/jsre/jsre.go index 0db9e33fc..af7d507c6 100644 --- a/jsre/jsre.go +++ b/jsre/jsre.go @@ -154,7 +154,9 @@ loop: if err != nil { fmt.Println("js error:", err, arguments) } - if timer.interval { + + _, inreg := registry[timer] // when clearInterval is called from within the callback don't reset it + if timer.interval && inreg { timer.timer.Reset(timer.duration) } else { delete(registry, timer) diff --git a/miner/agent.go b/miner/agent.go index 7ccf8d2e0..e80b222c8 100644 --- a/miner/agent.go +++ b/miner/agent.go @@ -19,11 +19,12 @@ package miner import ( "sync" + "sync/atomic" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/pow" - "sync/atomic" ) type CpuAgent struct { @@ -63,7 +64,7 @@ func (self *CpuAgent) Stop() { func (self *CpuAgent) Start() { self.mu.Lock() defer self.mu.Unlock() - + if !atomic.CompareAndSwapInt32(&self.isMining, 0, 1) { return // agent already started } diff --git a/miner/worker.go b/miner/worker.go index 16a16931d..22d0b9b6e 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -100,7 +101,7 @@ type worker struct { eth core.Backend chain *core.ChainManager proc *core.BlockProcessor - chainDb common.Database + chainDb ethdb.Database coinbase common.Address gasPrice *big.Int @@ -278,12 +279,12 @@ func (self *worker) wait() { glog.V(logger.Error).Infoln("Invalid block found during mining") continue } - if err := core.ValidateHeader(self.eth.BlockProcessor().Pow, block.Header(), parent, true, false); err != nil && err != core.BlockFutureErr { + if err := core.ValidateHeader(self.eth.BlockProcessor().Pow, block.Header(), parent.Header(), true, false); err != nil && err != core.BlockFutureErr { glog.V(logger.Error).Infoln("Invalid header on mined block:", err) continue } - stat, err := self.chain.WriteBlock(block, false) + stat, err := self.chain.WriteBlock(block) if err != nil { glog.V(logger.Error).Infoln("error writing block to chain", err) continue @@ -533,14 +534,12 @@ func (self *worker) commitNewWork() { // create the new block whose nonce will be mined. work.Block = types.NewBlock(header, work.txs, uncles, work.receipts) - work.Block.Td = new(big.Int).Set(core.CalcTD(work.Block, self.chain.GetBlock(work.Block.ParentHash()))) // We only care about logging if we're actually mining. if atomic.LoadInt32(&self.mining) == 1 { glog.V(logger.Info).Infof("commit new work on block %v with %d txs & %d uncles. Took %v\n", work.Block.Number(), work.tcount, len(uncles), time.Since(tstart)) self.logLocalMinedBlocks(work, previous) } - self.push(work) } diff --git a/rlp/decode.go b/rlp/decode.go index 1381f5274..c4d42c6fc 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -173,6 +173,8 @@ var ( func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { kind := typ.Kind() switch { + case typ == rawValueType: + return decodeRawValue, nil case typ.Implements(decoderInterface): return decodeDecoder, nil case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(decoderInterface): @@ -203,6 +205,15 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { } } +func decodeRawValue(s *Stream, val reflect.Value) error { + r, err := s.Raw() + if err != nil { + return err + } + val.SetBytes(r) + return nil +} + func decodeUint(s *Stream, val reflect.Value) error { typ := val.Type() num, err := s.uint(typ.Bits()) diff --git a/rlp/decode_test.go b/rlp/decode_test.go index d6b0611dd..408f1a5a9 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -24,6 +24,7 @@ import ( "io" "math/big" "reflect" + "strings" "testing" ) @@ -437,6 +438,11 @@ var decodeTests = []decodeTest{ error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I", }, + // RawValue + {input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))}, + {input: "82FFFF", ptr: new(RawValue), value: RawValue(unhex("82FFFF"))}, + {input: "C20102", ptr: new([]RawValue), value: []RawValue{unhex("01"), unhex("02")}}, + // pointers {input: "00", ptr: new(*[]byte), value: &[]byte{0}}, {input: "80", ptr: new(*uint), value: uintp(0)}, @@ -725,7 +731,7 @@ func encodeTestSlice(n uint) []byte { } func unhex(str string) []byte { - b, err := hex.DecodeString(str) + b, err := hex.DecodeString(strings.Replace(str, " ", "", -1)) if err != nil { panic(fmt.Sprintf("invalid hex string: %q", str)) } diff --git a/rlp/encode.go b/rlp/encode.go index b525ae4e7..d73b17c28 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -45,12 +45,6 @@ type Encoder interface { EncodeRLP(io.Writer) error } -// ListSize returns the encoded size of an RLP list with the given -// content size. -func ListSize(contentSize uint64) uint64 { - return uint64(headsize(contentSize)) + contentSize -} - // Encode writes the RLP encoding of val to w. Note that Encode may // perform many small writes in some cases. Consider making w // buffered. @@ -90,8 +84,8 @@ func Encode(w io.Writer, val interface{}) error { return outer.encode(val) } eb := encbufPool.Get().(*encbuf) - eb.reset() defer encbufPool.Put(eb) + eb.reset() if err := eb.encode(val); err != nil { return err } @@ -102,8 +96,8 @@ func Encode(w io.Writer, val interface{}) error { // Please see the documentation of Encode for the encoding rules. func EncodeToBytes(val interface{}) ([]byte, error) { eb := encbufPool.Get().(*encbuf) - eb.reset() defer encbufPool.Put(eb) + eb.reset() if err := eb.encode(val); err != nil { return nil, err } @@ -288,8 +282,13 @@ type encReader struct { func (r *encReader) Read(b []byte) (n int, err error) { for { if r.piece = r.next(); r.piece == nil { - encbufPool.Put(r.buf) - r.buf = nil + // Put the encode buffer back into the pool at EOF when it + // is first encountered. Subsequent calls still return EOF + // as the error but the buffer is no longer valid. + if r.buf != nil { + encbufPool.Put(r.buf) + r.buf = nil + } return n, io.EOF } nn := copy(b[n:], r.piece) @@ -349,6 +348,8 @@ var ( func makeWriter(typ reflect.Type) (writer, error) { kind := typ.Kind() switch { + case typ == rawValueType: + return writeRawValue, nil case typ.Implements(encoderInterface): return writeEncoder, nil case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(encoderInterface): @@ -384,6 +385,11 @@ func isByte(typ reflect.Type) bool { return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) } +func writeRawValue(val reflect.Value, w *encbuf) error { + w.str = append(w.str, val.Bytes()...) + return nil +} + func writeUint(val reflect.Value, w *encbuf) error { i := val.Uint() if i == 0 { diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 60bd95692..a3f30d804 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -23,6 +23,7 @@ import ( "io" "io/ioutil" "math/big" + "sync" "testing" ) @@ -203,6 +204,11 @@ var encTests = []encTest{ output: "F90200CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376", }, + // RawValue + {val: RawValue(unhex("01")), output: "01"}, + {val: RawValue(unhex("82FFFF")), output: "82FFFF"}, + {val: []RawValue{unhex("01"), unhex("02")}, output: "C20102"}, + // structs {val: simplestruct{}, output: "C28080"}, {val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"}, @@ -306,3 +312,25 @@ func TestEncodeToReaderPiecewise(t *testing.T) { return output, nil }) } + +// This is a regression test verifying that encReader +// returns its encbuf to the pool only once. +func TestEncodeToReaderReturnToPool(t *testing.T) { + buf := make([]byte, 50) + wg := new(sync.WaitGroup) + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + for i := 0; i < 1000; i++ { + _, r, _ := EncodeToReader("foo") + ioutil.ReadAll(r) + r.Read(buf) + r.Read(buf) + r.Read(buf) + r.Read(buf) + } + wg.Done() + }() + } + wg.Wait() +} diff --git a/rlp/raw.go b/rlp/raw.go new file mode 100644 index 000000000..33aae6ee5 --- /dev/null +++ b/rlp/raw.go @@ -0,0 +1,156 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rlp + +import ( + "io" + "reflect" +) + +// RawValue represents an encoded RLP value and can be used to delay +// RLP decoding or precompute an encoding. Note that the decoder does +// not verify whether the content of RawValues is valid RLP. +type RawValue []byte + +var rawValueType = reflect.TypeOf(RawValue{}) + +// ListSize returns the encoded size of an RLP list with the given +// content size. +func ListSize(contentSize uint64) uint64 { + return uint64(headsize(contentSize)) + contentSize +} + +// Split returns the content of first RLP value and any +// bytes after the value as subslices of b. +func Split(b []byte) (k Kind, content, rest []byte, err error) { + k, ts, cs, err := readKind(b) + if err != nil { + return 0, nil, b, err + } + return k, b[ts : ts+cs], b[ts+cs:], nil +} + +// SplitString splits b into the content of an RLP string +// and any remaining bytes after the string. +func SplitString(b []byte) (content, rest []byte, err error) { + k, content, rest, err := Split(b) + if err != nil { + return nil, b, err + } + if k == List { + return nil, b, ErrExpectedString + } + return content, rest, nil +} + +// SplitList splits b into the content of a list and any remaining +// bytes after the list. +func SplitList(b []byte) (content, rest []byte, err error) { + k, content, rest, err := Split(b) + if err != nil { + return nil, b, err + } + if k != List { + return nil, b, ErrExpectedList + } + return content, rest, nil +} + +// CountValues counts the number of encoded values in b. +func CountValues(b []byte) (int, error) { + i := 0 + for ; len(b) > 0; i++ { + _, tagsize, size, err := readKind(b) + if err != nil { + return 0, err + } + b = b[tagsize+size:] + } + return i, nil +} + +func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { + if len(buf) == 0 { + return 0, 0, 0, io.ErrUnexpectedEOF + } + b := buf[0] + switch { + case b < 0x80: + k = Byte + tagsize = 0 + contentsize = 1 + case b < 0xB8: + k = String + tagsize = 1 + contentsize = uint64(b - 0x80) + // Reject strings that should've been single bytes. + if contentsize == 1 && buf[1] < 128 { + return 0, 0, 0, ErrCanonSize + } + case b < 0xC0: + k = String + tagsize = uint64(b-0xB7) + 1 + contentsize, err = readSize(buf[1:], b-0xB7) + case b < 0xF8: + k = List + tagsize = 1 + contentsize = uint64(b - 0xC0) + default: + k = List + tagsize = uint64(b-0xF7) + 1 + contentsize, err = readSize(buf[1:], b-0xF7) + } + if err != nil { + return 0, 0, 0, err + } + // Reject values larger than the input slice. + if contentsize > uint64(len(buf))-tagsize { + return 0, 0, 0, ErrValueTooLarge + } + return k, tagsize, contentsize, err +} + +func readSize(b []byte, slen byte) (uint64, error) { + if int(slen) > len(b) { + return 0, io.ErrUnexpectedEOF + } + var s uint64 + switch slen { + case 1: + s = uint64(b[0]) + case 2: + s = uint64(b[0])<<8 | uint64(b[1]) + case 3: + s = uint64(b[0])<<16 | uint64(b[1])<<8 | uint64(b[2]) + case 4: + s = uint64(b[0])<<24 | uint64(b[1])<<16 | uint64(b[2])<<8 | uint64(b[3]) + case 5: + s = uint64(b[0])<<32 | uint64(b[1])<<24 | uint64(b[2])<<16 | uint64(b[3])<<8 | uint64(b[4]) + case 6: + s = uint64(b[0])<<40 | uint64(b[1])<<32 | uint64(b[2])<<24 | uint64(b[3])<<16 | uint64(b[4])<<8 | uint64(b[5]) + case 7: + s = uint64(b[0])<<48 | uint64(b[1])<<40 | uint64(b[2])<<32 | uint64(b[3])<<24 | uint64(b[4])<<16 | uint64(b[5])<<8 | uint64(b[6]) + case 8: + s = uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 | uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7]) + } + // Reject sizes < 56 (shouldn't have separate size) and sizes with + // leading zero bytes. + if s < 56 || b[0] == 0 { + return 0, ErrCanonSize + } + return s, nil +} diff --git a/rlp/raw_test.go b/rlp/raw_test.go new file mode 100644 index 000000000..7d3ca13af --- /dev/null +++ b/rlp/raw_test.go @@ -0,0 +1,195 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rlp + +import ( + "bytes" + "io" + "reflect" + "testing" +) + +func TestCountValues(t *testing.T) { + tests := []struct { + input string // note: spaces in input are stripped by unhex + count int + err error + }{ + // simple cases + {"", 0, nil}, + {"00", 1, nil}, + {"80", 1, nil}, + {"C0", 1, nil}, + {"01 02 03", 3, nil}, + {"01 C406070809 02", 3, nil}, + {"820101 820202 8403030303 04", 4, nil}, + + // size errors + {"8142", 0, ErrCanonSize}, + {"01 01 8142", 0, ErrCanonSize}, + {"02 84020202", 0, ErrValueTooLarge}, + + { + input: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", + count: 2, + }, + } + for i, test := range tests { + count, err := CountValues(unhex(test.input)) + if count != test.count { + t.Errorf("test %d: count mismatch, got %d want %d\ninput: %s", i, count, test.count, test.input) + } + if !reflect.DeepEqual(err, test.err) { + t.Errorf("test %d: err mismatch, got %q want %q\ninput: %s", i, err, test.err, test.input) + } + } +} + +func TestSplitTypes(t *testing.T) { + if _, _, err := SplitString(unhex("C100")); err != ErrExpectedString { + t.Error("SplitString returned %q, want %q", err, ErrExpectedString) + } + if _, _, err := SplitList(unhex("01")); err != ErrExpectedList { + t.Error("SplitString returned %q, want %q", err, ErrExpectedList) + } + if _, _, err := SplitList(unhex("81FF")); err != ErrExpectedList { + t.Error("SplitString returned %q, want %q", err, ErrExpectedList) + } +} + +func TestSplit(t *testing.T) { + tests := []struct { + input string + kind Kind + val, rest string + err error + }{ + {input: "01FFFF", kind: Byte, val: "01", rest: "FFFF"}, + {input: "80FFFF", kind: String, val: "", rest: "FFFF"}, + {input: "C3010203", kind: List, val: "010203"}, + + // errors + {input: "", err: io.ErrUnexpectedEOF}, + + {input: "8141", err: ErrCanonSize, rest: "8141"}, + {input: "B800", err: ErrCanonSize, rest: "B800"}, + {input: "B802FFFF", err: ErrCanonSize, rest: "B802FFFF"}, + {input: "B90000", err: ErrCanonSize, rest: "B90000"}, + {input: "B90055", err: ErrCanonSize, rest: "B90055"}, + {input: "BA0002FFFF", err: ErrCanonSize, rest: "BA0002FFFF"}, + {input: "F800", err: ErrCanonSize, rest: "F800"}, + {input: "F90000", err: ErrCanonSize, rest: "F90000"}, + {input: "F90055", err: ErrCanonSize, rest: "F90055"}, + {input: "FA0002FFFF", err: ErrCanonSize, rest: "FA0002FFFF"}, + + {input: "8501010101", err: ErrValueTooLarge, rest: "8501010101"}, + {input: "C60607080902", err: ErrValueTooLarge, rest: "C60607080902"}, + + // size check overflow + {input: "BFFFFFFFFFFFFFFFFF", err: ErrValueTooLarge, rest: "BFFFFFFFFFFFFFFFFF"}, + {input: "FFFFFFFFFFFFFFFFFF", err: ErrValueTooLarge, rest: "FFFFFFFFFFFFFFFFFF"}, + + { + input: "B838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + err: ErrValueTooLarge, + rest: "B838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + }, + { + input: "F838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + err: ErrValueTooLarge, + rest: "F838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + }, + + // a few bigger values, just for kicks + { + input: "F839FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + kind: List, + val: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + rest: "", + }, + { + input: "F90211A060EF29F20CC1007AE6E9530AEE16F4B31F8F1769A2D1264EC995C6D1241868D6A07C62AB8AC9838F5F5877B20BB37B387BC2106E97A3D52172CBEDB5EE17C36008A00EAB6B7324AADC0F6047C6AFC8229F09F7CF451B51D67C8DFB08D49BA8C3C626A04453343B2F3A6E42FCF87948F88AF7C8FC16D0C2735CBA7F026836239AB2C15FA024635C7291C882CE4C0763760C1A362DFC3FFCD802A55722236DE058D74202ACA0A220C808DE10F55E40AB25255201CFF009EA181D3906638E944EE2BF34049984A08D325AB26796F1CCB470F69C0F842501DC35D368A0C2575B2D243CFD1E8AB0FDA0B5298FF60DA5069463D610513C9F04F24051348391A143AFFAB7197DFACDEA72A02D2A7058A4463F8FB69378369E11EF33AE3252E2DB86CB545B36D3C26DDECE5AA0888F97BCA8E0BD83DC5B3B91CFF5FAF2F66F9501010682D67EF4A3B4E66115FBA0E8175A60C93BE9ED02921958F0EA55DA0FB5E4802AF5846147BAD92BC2D8AF26A08B3376FF433F3A4250FA64B7F804004CAC5807877D91C4427BD1CD05CF912ED8A09B32EF0F03BD13C37FF950C0CCCEFCCDD6669F2E7F2AA5CB859928E84E29763EA09BBA5E46610C8C8B1F8E921E5691BF8C7E40D75825D5EA3217AA9C3A8A355F39A0EEB95BC78251CCCEC54A97F19755C4A59A293544EEE6119AFA50531211E53C4FA00B6E86FE150BF4A9E0FEEE9C90F5465E617A861BB5E357F942881EE762212E2580", + kind: List, + val: "A060EF29F20CC1007AE6E9530AEE16F4B31F8F1769A2D1264EC995C6D1241868D6A07C62AB8AC9838F5F5877B20BB37B387BC2106E97A3D52172CBEDB5EE17C36008A00EAB6B7324AADC0F6047C6AFC8229F09F7CF451B51D67C8DFB08D49BA8C3C626A04453343B2F3A6E42FCF87948F88AF7C8FC16D0C2735CBA7F026836239AB2C15FA024635C7291C882CE4C0763760C1A362DFC3FFCD802A55722236DE058D74202ACA0A220C808DE10F55E40AB25255201CFF009EA181D3906638E944EE2BF34049984A08D325AB26796F1CCB470F69C0F842501DC35D368A0C2575B2D243CFD1E8AB0FDA0B5298FF60DA5069463D610513C9F04F24051348391A143AFFAB7197DFACDEA72A02D2A7058A4463F8FB69378369E11EF33AE3252E2DB86CB545B36D3C26DDECE5AA0888F97BCA8E0BD83DC5B3B91CFF5FAF2F66F9501010682D67EF4A3B4E66115FBA0E8175A60C93BE9ED02921958F0EA55DA0FB5E4802AF5846147BAD92BC2D8AF26A08B3376FF433F3A4250FA64B7F804004CAC5807877D91C4427BD1CD05CF912ED8A09B32EF0F03BD13C37FF950C0CCCEFCCDD6669F2E7F2AA5CB859928E84E29763EA09BBA5E46610C8C8B1F8E921E5691BF8C7E40D75825D5EA3217AA9C3A8A355F39A0EEB95BC78251CCCEC54A97F19755C4A59A293544EEE6119AFA50531211E53C4FA00B6E86FE150BF4A9E0FEEE9C90F5465E617A861BB5E357F942881EE762212E2580", + rest: "", + }, + { + input: "F877A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", + kind: List, + val: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", + rest: "", + }, + } + + for i, test := range tests { + kind, val, rest, err := Split(unhex(test.input)) + if kind != test.kind { + t.Errorf("test %d: kind mismatch: got %v, want %v", i, kind, test.kind) + } + if !bytes.Equal(val, unhex(test.val)) { + t.Errorf("test %d: val mismatch: got %x, want %s", i, val, test.val) + } + if !bytes.Equal(rest, unhex(test.rest)) { + t.Errorf("test %d: rest mismatch: got %x, want %s", i, rest, test.rest) + } + if err != test.err { + t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) + } + } +} + +func TestReadSize(t *testing.T) { + tests := []struct { + input string + slen byte + size uint64 + err error + }{ + {input: "", slen: 1, err: io.ErrUnexpectedEOF}, + {input: "FF", slen: 2, err: io.ErrUnexpectedEOF}, + {input: "00", slen: 1, err: ErrCanonSize}, + {input: "36", slen: 1, err: ErrCanonSize}, + {input: "37", slen: 1, err: ErrCanonSize}, + {input: "38", slen: 1, size: 0x38}, + {input: "FF", slen: 1, size: 0xFF}, + {input: "FFFF", slen: 2, size: 0xFFFF}, + {input: "FFFFFF", slen: 3, size: 0xFFFFFF}, + {input: "FFFFFFFF", slen: 4, size: 0xFFFFFFFF}, + {input: "FFFFFFFFFF", slen: 5, size: 0xFFFFFFFFFF}, + {input: "FFFFFFFFFFFF", slen: 6, size: 0xFFFFFFFFFFFF}, + {input: "FFFFFFFFFFFFFF", slen: 7, size: 0xFFFFFFFFFFFFFF}, + {input: "FFFFFFFFFFFFFFFF", slen: 8, size: 0xFFFFFFFFFFFFFFFF}, + {input: "0102", slen: 2, size: 0x0102}, + {input: "010203", slen: 3, size: 0x010203}, + {input: "01020304", slen: 4, size: 0x01020304}, + {input: "0102030405", slen: 5, size: 0x0102030405}, + {input: "010203040506", slen: 6, size: 0x010203040506}, + {input: "01020304050607", slen: 7, size: 0x01020304050607}, + {input: "0102030405060708", slen: 8, size: 0x0102030405060708}, + } + + for _, test := range tests { + size, err := readSize(unhex(test.input), test.slen) + if err != test.err { + t.Errorf("readSize(%s, %d): error mismatch: got %q, want %q", test.input, test.slen, err, test.err) + continue + } + if size != test.size { + t.Errorf("readSize(%s, %d): size mismatch: got %#x, want %#x", test.input, test.slen, size, test.size) + } + } +} diff --git a/rpc/api/admin.go b/rpc/api/admin.go index 5e392ae32..8af69b189 100644 --- a/rpc/api/admin.go +++ b/rpc/api/admin.go @@ -55,7 +55,6 @@ var ( "admin_exportChain": (*adminApi).ExportChain, "admin_importChain": (*adminApi).ImportChain, "admin_verbosity": (*adminApi).Verbosity, - "admin_chainSyncStatus": (*adminApi).ChainSyncStatus, "admin_setSolc": (*adminApi).SetSolc, "admin_datadir": (*adminApi).DataDir, "admin_startRPC": (*adminApi).StartRPC, @@ -232,17 +231,6 @@ func (self *adminApi) Verbosity(req *shared.Request) (interface{}, error) { return true, nil } -func (self *adminApi) ChainSyncStatus(req *shared.Request) (interface{}, error) { - pending, cached, importing, estimate := self.ethereum.Downloader().Stats() - - return map[string]interface{}{ - "blocksAvailable": pending, - "blocksWaitingForImport": cached, - "importing": importing, - "estimate": estimate.String(), - }, nil -} - func (self *adminApi) SetSolc(req *shared.Request) (interface{}, error) { args := new(SetSolcArgs) if err := self.coder.Decode(req.Params, &args); err != nil { diff --git a/rpc/api/admin_js.go b/rpc/api/admin_js.go index 25dbb4a8d..413ea8d47 100644 --- a/rpc/api/admin_js.go +++ b/rpc/api/admin_js.go @@ -143,10 +143,6 @@ web3._extend({ new web3._extend.Property({ name: 'datadir', getter: 'admin_datadir' - }), - new web3._extend.Property({ - name: 'chainSyncStatus', - getter: 'admin_chainSyncStatus' }) ] }); diff --git a/rpc/api/eth.go b/rpc/api/eth.go index ba87e86c6..30366a951 100644 --- a/rpc/api/eth.go +++ b/rpc/api/eth.go @@ -55,6 +55,7 @@ var ( "eth_protocolVersion": (*ethApi).ProtocolVersion, "eth_coinbase": (*ethApi).Coinbase, "eth_mining": (*ethApi).IsMining, + "eth_syncing": (*ethApi).IsSyncing, "eth_gasPrice": (*ethApi).GasPrice, "eth_getStorage": (*ethApi).GetStorage, "eth_storageAt": (*ethApi).GetStorage, @@ -166,6 +167,20 @@ func (self *ethApi) IsMining(req *shared.Request) (interface{}, error) { return self.xeth.IsMining(), nil } +func (self *ethApi) IsSyncing(req *shared.Request) (interface{}, error) { + current := self.ethereum.ChainManager().CurrentBlock().NumberU64() + origin, height := self.ethereum.Downloader().Boundaries() + + if current < height { + return map[string]interface{}{ + "startingBlock": newHexNum(big.NewInt(int64(origin)).Bytes()), + "currentBlock": newHexNum(big.NewInt(int64(current)).Bytes()), + "highestBlock": newHexNum(big.NewInt(int64(height)).Bytes()), + }, nil + } + return false, nil +} + func (self *ethApi) GasPrice(req *shared.Request) (interface{}, error) { return newHexNum(self.xeth.DefaultGasPrice().Bytes()), nil } @@ -204,7 +219,8 @@ func (self *ethApi) GetBlockTransactionCountByHash(req *shared.Request) (interfa return nil, shared.NewDecodeParamError(err.Error()) } - block := NewBlockRes(self.xeth.EthBlockByHash(args.Hash), false) + raw := self.xeth.EthBlockByHash(args.Hash) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), false) if block == nil { return nil, nil } else { @@ -218,7 +234,8 @@ func (self *ethApi) GetBlockTransactionCountByNumber(req *shared.Request) (inter return nil, shared.NewDecodeParamError(err.Error()) } - block := NewBlockRes(self.xeth.EthBlockByNumber(args.BlockNumber), false) + raw := self.xeth.EthBlockByNumber(args.BlockNumber) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), false) if block == nil { return nil, nil } else { @@ -232,12 +249,12 @@ func (self *ethApi) GetUncleCountByBlockHash(req *shared.Request) (interface{}, return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByHash(args.Hash) - br := NewBlockRes(block, false) - if br == nil { + raw := self.xeth.EthBlockByHash(args.Hash) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), false) + if block == nil { return nil, nil } - return newHexNum(big.NewInt(int64(len(br.Uncles))).Bytes()), nil + return newHexNum(big.NewInt(int64(len(block.Uncles))).Bytes()), nil } func (self *ethApi) GetUncleCountByBlockNumber(req *shared.Request) (interface{}, error) { @@ -246,12 +263,12 @@ func (self *ethApi) GetUncleCountByBlockNumber(req *shared.Request) (interface{} return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByNumber(args.BlockNumber) - br := NewBlockRes(block, false) - if br == nil { + raw := self.xeth.EthBlockByNumber(args.BlockNumber) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), false) + if block == nil { return nil, nil } - return newHexNum(big.NewInt(int64(len(br.Uncles))).Bytes()), nil + return newHexNum(big.NewInt(int64(len(block.Uncles))).Bytes()), nil } func (self *ethApi) GetData(req *shared.Request) (interface{}, error) { @@ -362,7 +379,7 @@ func (self *ethApi) GetBlockByHash(req *shared.Request) (interface{}, error) { } block := self.xeth.EthBlockByHash(args.BlockHash) - return NewBlockRes(block, args.IncludeTxs), nil + return NewBlockRes(block, self.xeth.Td(block.Hash()), args.IncludeTxs), nil } func (self *ethApi) GetBlockByNumber(req *shared.Request) (interface{}, error) { @@ -372,8 +389,7 @@ func (self *ethApi) GetBlockByNumber(req *shared.Request) (interface{}, error) { } block := self.xeth.EthBlockByNumber(args.BlockNumber) - br := NewBlockRes(block, args.IncludeTxs) - return br, nil + return NewBlockRes(block, self.xeth.Td(block.Hash()), args.IncludeTxs), nil } func (self *ethApi) GetTransactionByHash(req *shared.Request) (interface{}, error) { @@ -402,16 +418,15 @@ func (self *ethApi) GetTransactionByBlockHashAndIndex(req *shared.Request) (inte return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByHash(args.Hash) - br := NewBlockRes(block, true) - if br == nil { + raw := self.xeth.EthBlockByHash(args.Hash) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), true) + if block == nil { return nil, nil } - - if args.Index >= int64(len(br.Transactions)) || args.Index < 0 { + if args.Index >= int64(len(block.Transactions)) || args.Index < 0 { return nil, nil } else { - return br.Transactions[args.Index], nil + return block.Transactions[args.Index], nil } } @@ -421,17 +436,16 @@ func (self *ethApi) GetTransactionByBlockNumberAndIndex(req *shared.Request) (in return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByNumber(args.BlockNumber) - v := NewBlockRes(block, true) - if v == nil { + raw := self.xeth.EthBlockByNumber(args.BlockNumber) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), true) + if block == nil { return nil, nil } - - if args.Index >= int64(len(v.Transactions)) || args.Index < 0 { + if args.Index >= int64(len(block.Transactions)) || args.Index < 0 { // return NewValidationError("Index", "does not exist") return nil, nil } - return v.Transactions[args.Index], nil + return block.Transactions[args.Index], nil } func (self *ethApi) GetUncleByBlockHashAndIndex(req *shared.Request) (interface{}, error) { @@ -440,17 +454,16 @@ func (self *ethApi) GetUncleByBlockHashAndIndex(req *shared.Request) (interface{ return nil, shared.NewDecodeParamError(err.Error()) } - br := NewBlockRes(self.xeth.EthBlockByHash(args.Hash), false) - if br == nil { + raw := self.xeth.EthBlockByHash(args.Hash) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), false) + if block == nil { return nil, nil } - - if args.Index >= int64(len(br.Uncles)) || args.Index < 0 { + if args.Index >= int64(len(block.Uncles)) || args.Index < 0 { // return NewValidationError("Index", "does not exist") return nil, nil } - - return br.Uncles[args.Index], nil + return block.Uncles[args.Index], nil } func (self *ethApi) GetUncleByBlockNumberAndIndex(req *shared.Request) (interface{}, error) { @@ -459,17 +472,15 @@ func (self *ethApi) GetUncleByBlockNumberAndIndex(req *shared.Request) (interfac return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByNumber(args.BlockNumber) - v := NewBlockRes(block, true) - - if v == nil { + raw := self.xeth.EthBlockByNumber(args.BlockNumber) + block := NewBlockRes(raw, self.xeth.Td(raw.Hash()), true) + if block == nil { return nil, nil } - - if args.Index >= int64(len(v.Uncles)) || args.Index < 0 { + if args.Index >= int64(len(block.Uncles)) || args.Index < 0 { return nil, nil } else { - return v.Uncles[args.Index], nil + return block.Uncles[args.Index], nil } } diff --git a/rpc/api/parsing.go b/rpc/api/parsing.go index 5858bc136..cdfaa0ed1 100644 --- a/rpc/api/parsing.go +++ b/rpc/api/parsing.go @@ -281,7 +281,7 @@ func (b *BlockRes) MarshalJSON() ([]byte, error) { } } -func NewBlockRes(block *types.Block, fullTx bool) *BlockRes { +func NewBlockRes(block *types.Block, td *big.Int, fullTx bool) *BlockRes { if block == nil { return nil } @@ -299,7 +299,7 @@ func NewBlockRes(block *types.Block, fullTx bool) *BlockRes { res.ReceiptRoot = newHexData(block.ReceiptHash()) res.Miner = newHexData(block.Coinbase()) res.Difficulty = newHexNum(block.Difficulty()) - res.TotalDifficulty = newHexNum(block.Td) + res.TotalDifficulty = newHexNum(td) res.Size = newHexNum(block.Size().Int64()) res.ExtraData = newHexData(block.Extra()) res.GasLimit = newHexNum(block.GasLimit()) diff --git a/rpc/api/utils.go b/rpc/api/utils.go index 5072dc2cd..76b2c531d 100644 --- a/rpc/api/utils.go +++ b/rpc/api/utils.go @@ -32,7 +32,6 @@ var ( AutoCompletion = map[string][]string{ "admin": []string{ "addPeer", - "chainSyncStatus", "datadir", "exportChain", "getContractInfo", @@ -99,6 +98,7 @@ var ( "sendRawTransaction", "sendTransaction", "sign", + "syncing", }, "miner": []string{ "hashrate", diff --git a/rpc/comms/ipc_unix.go b/rpc/comms/ipc_unix.go index 9d90da071..d68363a45 100644 --- a/rpc/comms/ipc_unix.go +++ b/rpc/comms/ipc_unix.go @@ -21,6 +21,7 @@ package comms import ( "net" "os" + "path/filepath" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -69,7 +70,11 @@ func (self *ipcClient) reconnect() error { } func startIpc(cfg IpcConfig, codec codec.Codec, initializer func(conn net.Conn) (shared.EthereumApi, error)) error { - os.Remove(cfg.Endpoint) // in case it still exists from a previous run + // Ensure the IPC path exists and remove any previous leftover + if err := os.MkdirAll(filepath.Dir(cfg.Endpoint), 0751); err != nil { + return err + } + os.Remove(cfg.Endpoint) l, err := net.ListenUnix("unix", &net.UnixAddr{Name: cfg.Endpoint, Net: "unix"}) if err != nil { diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 2090afce7..e31ca6344 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -197,7 +197,7 @@ func (test *BlockTest) makeEthConfig() *eth.Config { Verbosity: 5, Etherbase: common.Address{}, AccountManager: accounts.NewManager(ks), - NewDB: func(path string) (common.Database, error) { return ethdb.NewMemDatabase() }, + NewDB: func(path string) (ethdb.Database, error) { return ethdb.NewMemDatabase() }, } } @@ -440,9 +440,8 @@ func convertBlockTest(in *btJSON) (out *BlockTest, err error) { func mustConvertGenesis(testGenesis btHeader) *types.Block { hdr := mustConvertHeader(testGenesis) hdr.Number = big.NewInt(0) - b := types.NewBlockWithHeader(hdr) - b.Td = new(big.Int) - return b + + return types.NewBlockWithHeader(hdr) } func mustConvertHeader(in btHeader) *types.Header { diff --git a/tests/util.go b/tests/util.go index a9b5011a9..72d927ada 100644 --- a/tests/util.go +++ b/tests/util.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" ) func checkLogs(tlog []Log, logs state.Logs) error { @@ -87,7 +88,7 @@ func (self Log) Topics() [][]byte { return t } -func StateObjectFromAccount(db common.Database, addr string, account Account) *state.StateObject { +func StateObjectFromAccount(db ethdb.Database, addr string, account Account) *state.StateObject { obj := state.NewStateObject(common.HexToAddress(addr), db) obj.SetBalance(common.Big(account.Balance)) diff --git a/xeth/xeth.go b/xeth/xeth.go index 8bd45998f..00b70da6c 100644 --- a/xeth/xeth.go +++ b/xeth/xeth.go @@ -355,6 +355,10 @@ func (self *XEth) EthBlockByNumber(num int64) *types.Block { return self.getBlockByHeight(num) } +func (self *XEth) Td(hash common.Hash) *big.Int { + return self.backend.ChainManager().GetTd(hash) +} + func (self *XEth) CurrentBlock() *types.Block { return self.backend.ChainManager().CurrentBlock() } |