tests: update tests and implement general state tests (#14734)

Tests are now included as a submodule. This should make updating easier
and removes ~60MB of JSON data from the working copy.

State tests are replaced by General State Tests, which run the same test
with multiple fork configurations.

With the new test runner, consensus tests are run as subtests by walking
json files. Many hex issues have been fixed upstream since the last
update and most custom parsing code is replaced by existing JSON hex
types. Tests can now be marked as 'expected failures', ensuring that
fixes for those tests will trigger an update to test configuration. The
new test runner also supports parallel execution and the -short flag.

3 files changed, 105 insertions, 24 deletions

diff --git a/core/gen_genesis.go b/core/gen_genesis.go
index 3f83905c9..1f3b4a8aa 100644
--- a/core/gen_genesis.go
+++ b/core/gen_genesis.go
@@ -13,24 +13,27 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
+var _ = (*genesisSpecMarshaling)(nil)
+
 func (g Genesis) MarshalJSON() ([]byte, error) {
 	type Genesis struct {
 		Config     *params.ChainConfig                         `json:"config"`
 		Nonce      math.HexOrDecimal64                         `json:"nonce"`
 		Timestamp  math.HexOrDecimal64                         `json:"timestamp"`
-		ParentHash common.Hash                                 `json:"parentHash"`
 		ExtraData  hexutil.Bytes                               `json:"extraData"`
 		GasLimit   math.HexOrDecimal64                         `json:"gasLimit"   gencodec:"required"`
 		Difficulty *math.HexOrDecimal256                       `json:"difficulty" gencodec:"required"`
 		Mixhash    common.Hash                                 `json:"mixHash"`
 		Coinbase   common.Address                              `json:"coinbase"`
 		Alloc      map[common.UnprefixedAddress]GenesisAccount `json:"alloc"      gencodec:"required"`
+		Number     uint64                                      `json:"number"`
+		GasUsed    math.HexOrDecimal64                         `json:"gasUsed"`
+		ParentHash common.Hash                                 `json:"parentHash"`
 	}
 	var enc Genesis
 	enc.Config = g.Config
 	enc.Nonce = math.HexOrDecimal64(g.Nonce)
 	enc.Timestamp = math.HexOrDecimal64(g.Timestamp)
-	enc.ParentHash = g.ParentHash
 	enc.ExtraData = g.ExtraData
 	enc.GasLimit = math.HexOrDecimal64(g.GasLimit)
 	enc.Difficulty = (*math.HexOrDecimal256)(g.Difficulty)
@@ -42,6 +45,9 @@ func (g Genesis) MarshalJSON() ([]byte, error) {
 			enc.Alloc[common.UnprefixedAddress(k)] = v
 		}
 	}
+	enc.Number = g.Number
+	enc.GasUsed = math.HexOrDecimal64(g.GasUsed)
+	enc.ParentHash = g.ParentHash
 	return json.Marshal(&enc)
 }
 
@@ -50,13 +56,15 @@ func (g *Genesis) UnmarshalJSON(input []byte) error {
 		Config     *params.ChainConfig                         `json:"config"`
 		Nonce      *math.HexOrDecimal64                        `json:"nonce"`
 		Timestamp  *math.HexOrDecimal64                        `json:"timestamp"`
-		ParentHash *common.Hash                                `json:"parentHash"`
 		ExtraData  hexutil.Bytes                               `json:"extraData"`
 		GasLimit   *math.HexOrDecimal64                        `json:"gasLimit"   gencodec:"required"`
 		Difficulty *math.HexOrDecimal256                       `json:"difficulty" gencodec:"required"`
 		Mixhash    *common.Hash                                `json:"mixHash"`
 		Coinbase   *common.Address                             `json:"coinbase"`
 		Alloc      map[common.UnprefixedAddress]GenesisAccount `json:"alloc"      gencodec:"required"`
+		Number     *uint64                                     `json:"number"`
+		GasUsed    *math.HexOrDecimal64                        `json:"gasUsed"`
+		ParentHash *common.Hash                                `json:"parentHash"`
 	}
 	var dec Genesis
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -71,9 +79,6 @@ func (g *Genesis) UnmarshalJSON(input []byte) error {
 	if dec.Timestamp != nil {
 		g.Timestamp = uint64(*dec.Timestamp)
 	}
-	if dec.ParentHash != nil {
-		g.ParentHash = *dec.ParentHash
-	}
 	if dec.ExtraData != nil {
 		g.ExtraData = dec.ExtraData
 	}
@@ -98,5 +103,14 @@ func (g *Genesis) UnmarshalJSON(input []byte) error {
 	for k, v := range dec.Alloc {
 		g.Alloc[common.Address(k)] = v
 	}
+	if dec.Number != nil {
+		g.Number = *dec.Number
+	}
+	if dec.GasUsed != nil {
+		g.GasUsed = uint64(*dec.GasUsed)
+	}
+	if dec.ParentHash != nil {
+		g.ParentHash = *dec.ParentHash
+	}
 	return nil
 }
diff --git a/core/gen_genesis_account.go b/core/gen_genesis_account.go
index bc5fc936b..15c9565a2 100644
--- a/core/gen_genesis_account.go
+++ b/core/gen_genesis_account.go
@@ -12,27 +12,37 @@ import (
 	"github.com/ethereum/go-ethereum/common/math"
 )
 
+var _ = (*genesisAccountMarshaling)(nil)
+
 func (g GenesisAccount) MarshalJSON() ([]byte, error) {
 	type GenesisAccount struct {
-		Code    hexutil.Bytes               `json:"code,omitempty"`
-		Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
-		Balance *math.HexOrDecimal256       `json:"balance" gencodec:"required"`
-		Nonce   math.HexOrDecimal64         `json:"nonce,omitempty"`
+		Code       hexutil.Bytes               `json:"code,omitempty"`
+		Storage    map[storageJSON]storageJSON `json:"storage,omitempty"`
+		Balance    *math.HexOrDecimal256       `json:"balance" gencodec:"required"`
+		Nonce      math.HexOrDecimal64         `json:"nonce,omitempty"`
+		PrivateKey hexutil.Bytes               `json:"secretKey,omitempty"`
 	}
 	var enc GenesisAccount
 	enc.Code = g.Code
-	enc.Storage = g.Storage
+	if g.Storage != nil {
+		enc.Storage = make(map[storageJSON]storageJSON, len(g.Storage))
+		for k, v := range g.Storage {
+			enc.Storage[storageJSON(k)] = storageJSON(v)
+		}
+	}
 	enc.Balance = (*math.HexOrDecimal256)(g.Balance)
 	enc.Nonce = math.HexOrDecimal64(g.Nonce)
+	enc.PrivateKey = g.PrivateKey
 	return json.Marshal(&enc)
 }
 
 func (g *GenesisAccount) UnmarshalJSON(input []byte) error {
 	type GenesisAccount struct {
-		Code    hexutil.Bytes               `json:"code,omitempty"`
-		Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
-		Balance *math.HexOrDecimal256       `json:"balance" gencodec:"required"`
-		Nonce   *math.HexOrDecimal64        `json:"nonce,omitempty"`
+		Code       hexutil.Bytes               `json:"code,omitempty"`
+		Storage    map[storageJSON]storageJSON `json:"storage,omitempty"`
+		Balance    *math.HexOrDecimal256       `json:"balance" gencodec:"required"`
+		Nonce      *math.HexOrDecimal64        `json:"nonce,omitempty"`
+		PrivateKey hexutil.Bytes               `json:"secretKey,omitempty"`
 	}
 	var dec GenesisAccount
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -42,7 +52,10 @@ func (g *GenesisAccount) UnmarshalJSON(input []byte) error {
 		g.Code = dec.Code
 	}
 	if dec.Storage != nil {
-		g.Storage = dec.Storage
+		g.Storage = make(map[common.Hash]common.Hash, len(dec.Storage))
+		for k, v := range dec.Storage {
+			g.Storage[common.Hash(k)] = common.Hash(v)
+		}
 	}
 	if dec.Balance == nil {
 		return errors.New("missing required field 'balance' for GenesisAccount")
@@ -51,5 +64,8 @@ func (g *GenesisAccount) UnmarshalJSON(input []byte) error {
 	if dec.Nonce != nil {
 		g.Nonce = uint64(*dec.Nonce)
 	}
+	if dec.PrivateKey != nil {
+		g.PrivateKey = dec.PrivateKey
+	}
 	return nil
 }
diff --git a/core/genesis.go b/core/genesis.go
index d587011f0..a507d522b 100644
--- a/core/genesis.go
+++ b/core/genesis.go
@@ -17,6 +17,9 @@
 package core
 
 import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
@@ -44,24 +47,42 @@ type Genesis struct {
 	Config     *params.ChainConfig `json:"config"`
 	Nonce      uint64              `json:"nonce"`
 	Timestamp  uint64              `json:"timestamp"`
-	ParentHash common.Hash         `json:"parentHash"`
 	ExtraData  []byte              `json:"extraData"`
 	GasLimit   uint64              `json:"gasLimit"   gencodec:"required"`
 	Difficulty *big.Int            `json:"difficulty" gencodec:"required"`
 	Mixhash    common.Hash         `json:"mixHash"`
 	Coinbase   common.Address      `json:"coinbase"`
 	Alloc      GenesisAlloc        `json:"alloc"      gencodec:"required"`
+
+	// These fields are used for consensus tests. Please don't use them
+	// in actual genesis blocks.
+	Number     uint64      `json:"number"`
+	GasUsed    uint64      `json:"gasUsed"`
+	ParentHash common.Hash `json:"parentHash"`
 }
 
 // GenesisAlloc specifies the initial state that is part of the genesis block.
 type GenesisAlloc map[common.Address]GenesisAccount
 
+func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error {
+	m := make(map[common.UnprefixedAddress]GenesisAccount)
+	if err := json.Unmarshal(data, &m); err != nil {
+		return err
+	}
+	*ga = make(GenesisAlloc)
+	for addr, a := range m {
+		(*ga)[common.Address(addr)] = a
+	}
+	return nil
+}
+
 // GenesisAccount is an account in the state of the genesis block.
 type GenesisAccount struct {
-	Code    []byte                      `json:"code,omitempty"`
-	Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
-	Balance *big.Int                    `json:"balance" gencodec:"required"`
-	Nonce   uint64                      `json:"nonce,omitempty"`
+	Code       []byte                      `json:"code,omitempty"`
+	Storage    map[common.Hash]common.Hash `json:"storage,omitempty"`
+	Balance    *big.Int                    `json:"balance" gencodec:"required"`
+	Nonce      uint64                      `json:"nonce,omitempty"`
+	PrivateKey []byte                      `json:"secretKey,omitempty"` // for tests
 }
 
 // field type overrides for gencodec
@@ -70,13 +91,38 @@ type genesisSpecMarshaling struct {
 	Timestamp  math.HexOrDecimal64
 	ExtraData  hexutil.Bytes
 	GasLimit   math.HexOrDecimal64
+	GasUsed    math.HexOrDecimal64
 	Difficulty *math.HexOrDecimal256
 	Alloc      map[common.UnprefixedAddress]GenesisAccount
 }
+
 type genesisAccountMarshaling struct {
-	Code    hexutil.Bytes
-	Balance *math.HexOrDecimal256
-	Nonce   math.HexOrDecimal64
+	Code       hexutil.Bytes
+	Balance    *math.HexOrDecimal256
+	Nonce      math.HexOrDecimal64
+	Storage    map[storageJSON]storageJSON
+	PrivateKey hexutil.Bytes
+}
+
+// storageJSON represents a 256 bit byte array, but allows less than 256 bits when
+// unmarshaling from hex.
+type storageJSON common.Hash
+
+func (h *storageJSON) UnmarshalText(text []byte) error {
+	text = bytes.TrimPrefix(text, []byte("0x"))
+	if len(text) > 64 {
+		return fmt.Errorf("too many hex characters in storage key/value %q", text)
+	}
+	offset := len(h) - len(text)/2 // pad on the left
+	if _, err := hex.Decode(h[offset:], text); err != nil {
+		fmt.Println(err)
+		return fmt.Errorf("invalid hex storage key/value %q", text)
+	}
+	return nil
+}
+
+func (h storageJSON) MarshalText() ([]byte, error) {
+	return hexutil.Bytes(h[:]).MarshalText()
 }
 
 // GenesisMismatchError is raised when trying to overwrite an existing
@@ -187,11 +233,13 @@ func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
 	}
 	root := statedb.IntermediateRoot(false)
 	head := &types.Header{
+		Number:     new(big.Int).SetUint64(g.Number),
 		Nonce:      types.EncodeNonce(g.Nonce),
 		Time:       new(big.Int).SetUint64(g.Timestamp),
 		ParentHash: g.ParentHash,
 		Extra:      g.ExtraData,
 		GasLimit:   new(big.Int).SetUint64(g.GasLimit),
+		GasUsed:    new(big.Int).SetUint64(g.GasUsed),
 		Difficulty: g.Difficulty,
 		MixDigest:  g.Mixhash,
 		Coinbase:   g.Coinbase,
@@ -210,6 +258,9 @@ func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
 // The block is committed as the canonical head block.
 func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
 	block, statedb := g.ToBlock()
+	if block.Number().Sign() != 0 {
+		return nil, fmt.Errorf("can't commit genesis block with number > 0")
+	}
 	if _, err := statedb.CommitTo(db, false); err != nil {
 		return nil, fmt.Errorf("cannot write state: %v", err)
 	}