tests: update tests and implement general state tests (#14734)

Tests are now included as a submodule. This should make updating easier
and removes ~60MB of JSON data from the working copy.

State tests are replaced by General State Tests, which run the same test
with multiple fork configurations.

With the new test runner, consensus tests are run as subtests by walking
json files. Many hex issues have been fixed upstream since the last
update and most custom parsing code is replaced by existing JSON hex
types. Tests can now be marked as 'expected failures', ensuring that
fixes for those tests will trigger an update to test configuration. The
new test runner also supports parallel execution and the -short flag.

1 files changed, 94 insertions, 180 deletions

diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go
index e7fe74f49..afdd896c3 100644
--- a/tests/vm_test_util.go
+++ b/tests/vm_test_util.go
@@ -18,218 +18,132 @@ package tests
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
-	"io"
 	"math/big"
-	"strconv"
-	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 )
 
-func RunVmTestWithReader(r io.Reader, skipTests []string) error {
-	tests := make(map[string]VmTest)
-	err := readJson(r, &tests)
-	if err != nil {
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	if err := runVmTests(tests, skipTests); err != nil {
-		return err
-	}
-
-	return nil
+// VMTest checks EVM execution without block or transaction context.
+// See https://github.com/ethereum/tests/wiki/VM-Tests for the test format specification.
+type VMTest struct {
+	json vmJSON
 }
 
-type bconf struct {
-	name    string
-	precomp bool
-	jit     bool
+func (t *VMTest) UnmarshalJSON(data []byte) error {
+	return json.Unmarshal(data, &t.json)
 }
 
-func BenchVmTest(p string, conf bconf, b *testing.B) error {
-	tests := make(map[string]VmTest)
-	err := readJsonFile(p, &tests)
-	if err != nil {
-		return err
-	}
-
-	test, ok := tests[conf.name]
-	if !ok {
-		return fmt.Errorf("test not found: %s", conf.name)
-	}
-
-	env := make(map[string]string)
-	env["currentCoinbase"] = test.Env.CurrentCoinbase
-	env["currentDifficulty"] = test.Env.CurrentDifficulty
-	env["currentGasLimit"] = test.Env.CurrentGasLimit
-	env["currentNumber"] = test.Env.CurrentNumber
-	env["previousHash"] = test.Env.PreviousHash
-	if n, ok := test.Env.CurrentTimestamp.(float64); ok {
-		env["currentTimestamp"] = strconv.Itoa(int(n))
-	} else {
-		env["currentTimestamp"] = test.Env.CurrentTimestamp.(string)
-	}
-
-	/*
-		if conf.precomp {
-			program := vm.NewProgram(test.code)
-			err := vm.AttachProgram(program)
-			if err != nil {
-				return err
-			}
-		}
-	*/
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		benchVmTest(test, env, b)
-	}
-
-	return nil
+type vmJSON struct {
+	Env           stEnv                `json:"env"`
+	Exec          vmExec               `json:"exec"`
+	Logs          []stLog              `json:"logs"`
+	GasRemaining  *math.HexOrDecimal64 `json:"gas"`
+	Out           hexutil.Bytes        `json:"out"`
+	Pre           core.GenesisAlloc    `json:"pre"`
+	Post          core.GenesisAlloc    `json:"post"`
+	PostStateRoot common.Hash          `json:"postStateRoot"`
 }
 
-func benchVmTest(test VmTest, env map[string]string, b *testing.B) {
-	b.StopTimer()
-	db, _ := ethdb.NewMemDatabase()
-	statedb := makePreState(db, test.Pre)
-	b.StartTimer()
-
-	RunVm(statedb, env, test.Exec)
+//go:generate gencodec -type vmExec -field-override vmExecMarshaling -out gen_vmexec.go
+
+type vmExec struct {
+	Address  common.Address `json:"address"  gencodec:"required"`
+	Caller   common.Address `json:"caller"   gencodec:"required"`
+	Origin   common.Address `json:"origin"   gencodec:"required"`
+	Code     []byte         `json:"code"     gencodec:"required"`
+	Data     []byte         `json:"data"     gencodec:"required"`
+	Value    *big.Int       `json:"value"    gencodec:"required"`
+	GasLimit uint64         `json:"gas"      gencodec:"required"`
+	GasPrice *big.Int       `json:"gasPrice" gencodec:"required"`
 }
 
-func RunVmTest(p string, skipTests []string) error {
-	tests := make(map[string]VmTest)
-	err := readJsonFile(p, &tests)
-	if err != nil {
-		return err
-	}
-
-	if err := runVmTests(tests, skipTests); err != nil {
-		return err
-	}
-
-	return nil
+type vmExecMarshaling struct {
+	Address  common.UnprefixedAddress
+	Caller   common.UnprefixedAddress
+	Origin   common.UnprefixedAddress
+	Code     hexutil.Bytes
+	Data     hexutil.Bytes
+	Value    *math.HexOrDecimal256
+	GasLimit math.HexOrDecimal64
+	GasPrice *math.HexOrDecimal256
 }
 
-func runVmTests(tests map[string]VmTest, skipTests []string) error {
-	skipTest := make(map[string]bool, len(skipTests))
-	for _, name := range skipTests {
-		skipTest[name] = true
-	}
+func (t *VMTest) Run(vmconfig vm.Config) error {
+	db, _ := ethdb.NewMemDatabase()
+	statedb := makePreState(db, t.json.Pre)
+	ret, gasRemaining, err := t.exec(statedb, vmconfig)
 
-	for name, test := range tests {
-		if skipTest[name] /*|| name != "exp0"*/ {
-			log.Info(fmt.Sprint("Skipping VM test", name))
-			continue
+	if t.json.GasRemaining == nil {
+		if err == nil {
+			return fmt.Errorf("gas unspecified (indicating an error), but VM returned no error")
 		}
-
-		if err := runVmTest(test); err != nil {
-			return fmt.Errorf("%s %s", name, err.Error())
+		if gasRemaining > 0 {
+			return fmt.Errorf("gas unspecified (indicating an error), but VM returned gas remaining > 0")
 		}
-
-		log.Info(fmt.Sprint("VM test passed: ", name))
-		//fmt.Println(string(statedb.Dump()))
+		return nil
 	}
-	return nil
-}
-
-func runVmTest(test VmTest) error {
-	db, _ := ethdb.NewMemDatabase()
-	statedb := makePreState(db, test.Pre)
-
-	// XXX Yeah, yeah...
-	env := make(map[string]string)
-	env["currentCoinbase"] = test.Env.CurrentCoinbase
-	env["currentDifficulty"] = test.Env.CurrentDifficulty
-	env["currentGasLimit"] = test.Env.CurrentGasLimit
-	env["currentNumber"] = test.Env.CurrentNumber
-	env["previousHash"] = test.Env.PreviousHash
-	if n, ok := test.Env.CurrentTimestamp.(float64); ok {
-		env["currentTimestamp"] = strconv.Itoa(int(n))
-	} else {
-		env["currentTimestamp"] = test.Env.CurrentTimestamp.(string)
+	// Test declares gas, expecting outputs to match.
+	if !bytes.Equal(ret, t.json.Out) {
+		return fmt.Errorf("return data mismatch: got %x, want %x", ret, t.json.Out)
 	}
-
-	var (
-		ret  []byte
-		gas  *big.Int
-		err  error
-		logs []*types.Log
-	)
-
-	ret, logs, gas, err = RunVm(statedb, env, test.Exec)
-
-	// Compare expected and actual return
-	rexp := common.FromHex(test.Out)
-	if !bytes.Equal(rexp, ret) {
-		return fmt.Errorf("return failed. Expected %x, got %x\n", rexp, ret)
+	if gasRemaining != uint64(*t.json.GasRemaining) {
+		return fmt.Errorf("remaining gas %v, want %v", gasRemaining, *t.json.GasRemaining)
 	}
-
-	// Check gas usage
-	if len(test.Gas) == 0 && err == nil {
-		return fmt.Errorf("gas unspecified, indicating an error. VM returned (incorrectly) successful")
-	} else {
-		gexp := math.MustParseBig256(test.Gas)
-		if gexp.Cmp(gas) != 0 {
-			return fmt.Errorf("gas failed. Expected %v, got %v\n", gexp, gas)
-		}
-	}
-
-	// check post state
-	for address, account := range test.Post {
-		accountAddr := common.HexToAddress(address)
-		if !statedb.Exist(accountAddr) {
-			continue
-		}
-		for addr, value := range account.Storage {
-			v := statedb.GetState(accountAddr, common.HexToHash(addr))
-			vexp := common.HexToHash(value)
-			if v != vexp {
-				return fmt.Errorf("(%x: %s) storage failed. Expected %x, got %x (%v %v)\n", addr[:4], addr, vexp, v, vexp.Big(), v.Big())
+	for addr, account := range t.json.Post {
+		for k, wantV := range account.Storage {
+			if haveV := statedb.GetState(addr, k); haveV != wantV {
+				return fmt.Errorf("wrong storage value at %x:\n  got  %x\n  want %x", k, haveV, wantV)
 			}
 		}
 	}
+	// if root := statedb.IntermediateRoot(false); root != t.json.PostStateRoot {
+	// 	return fmt.Errorf("post state root mismatch, got %x, want %x", root, t.json.PostStateRoot)
+	// }
+	return checkLogs(statedb.Logs(), t.json.Logs)
+}
 
-	// check logs
-	if len(test.Logs) > 0 {
-		lerr := checkLogs(test.Logs, logs)
-		if lerr != nil {
-			return lerr
-		}
-	}
-
-	return nil
+func (t *VMTest) exec(statedb *state.StateDB, vmconfig vm.Config) ([]byte, uint64, error) {
+	evm := t.newEVM(statedb, vmconfig)
+	e := t.json.Exec
+	return evm.Call(vm.AccountRef(e.Caller), e.Address, e.Data, e.GasLimit, e.Value)
 }
 
-func RunVm(statedb *state.StateDB, env, exec map[string]string) ([]byte, []*types.Log, *big.Int, error) {
-	chainConfig := &params.ChainConfig{
-		HomesteadBlock: params.MainnetChainConfig.HomesteadBlock,
-		DAOForkBlock:   params.MainnetChainConfig.DAOForkBlock,
-		DAOForkSupport: true,
-	}
-	var (
-		to    = common.HexToAddress(exec["address"])
-		from  = common.HexToAddress(exec["caller"])
-		data  = common.FromHex(exec["data"])
-		gas   = math.MustParseBig256(exec["gas"])
-		value = math.MustParseBig256(exec["value"])
-	)
-	caller := statedb.GetOrNewStateObject(from)
-	vm.PrecompiledContracts = make(map[common.Address]vm.PrecompiledContract)
+func (t *VMTest) newEVM(statedb *state.StateDB, vmconfig vm.Config) *vm.EVM {
+	initialCall := true
+	canTransfer := func(db vm.StateDB, address common.Address, amount *big.Int) bool {
+		if initialCall {
+			initialCall = false
+			return true
+		}
+		return core.CanTransfer(db, address, amount)
+	}
+	transfer := func(db vm.StateDB, sender, recipient common.Address, amount *big.Int) {}
+	context := vm.Context{
+		CanTransfer: canTransfer,
+		Transfer:    transfer,
+		GetHash:     vmTestBlockHash,
+		Origin:      t.json.Exec.Origin,
+		Coinbase:    t.json.Env.Coinbase,
+		BlockNumber: new(big.Int).SetUint64(t.json.Env.Number),
+		Time:        new(big.Int).SetUint64(t.json.Env.Timestamp),
+		GasLimit:    t.json.Env.GasLimit,
+		Difficulty:  t.json.Env.Difficulty,
+		GasPrice:    t.json.Exec.GasPrice,
+	}
+	vmconfig.NoRecursion = true
+	return vm.NewEVM(context, statedb, params.MainnetChainConfig, vmconfig)
+}
 
-	environment, _ := NewEVMEnvironment(true, chainConfig, statedb, env, exec)
-	ret, g, err := environment.Call(caller, to, data, gas.Uint64(), value)
-	return ret, statedb.Logs(), new(big.Int).SetUint64(g), err
+func vmTestBlockHash(n uint64) common.Hash {
+	return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String())))
 }