cmd/evm: adds ability to run individual state test file (#14998)

* cmd/evm: adds ability to run individual state test file

* cmd/evm: Fix statetest runner to be more json friendly

* cmd/evm, tests: minor polishes, dump state on fail

2 files changed, 120 insertions, 0 deletions

diff --git a/cmd/evm/main.go b/cmd/evm/main.go
index a2e3b048e..6c39cf8b8 100644
--- a/cmd/evm/main.go
+++ b/cmd/evm/main.go
@@ -143,6 +143,7 @@ func init() {
 		compileCommand,
 		disasmCommand,
 		runCommand,
+		stateTestCommand,
 	}
 }
 
diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go
new file mode 100644
index 000000000..3a4cc51c0
--- /dev/null
+++ b/cmd/evm/staterunner.go
@@ -0,0 +1,119 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/tests"
+
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var stateTestCommand = cli.Command{
+	Action:    stateTestCmd,
+	Name:      "statetest",
+	Usage:     "executes the given state tests",
+	ArgsUsage: "<file>",
+}
+
+type StatetestResult struct {
+	Name  string      `json:"name"`
+	Pass  bool        `json:"pass"`
+	Fork  string      `json:"fork"`
+	Error string      `json:"error,omitempty"`
+	State *state.Dump `json:"state,omitempty"`
+}
+
+func stateTestCmd(ctx *cli.Context) error {
+	if len(ctx.Args().First()) == 0 {
+		return errors.New("path-to-test argument required")
+	}
+	// Configure the go-ethereum logger
+	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
+	glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name)))
+	log.Root().SetHandler(glogger)
+
+	// Configure the EVM logger
+	config := &vm.LogConfig{
+		DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name),
+		DisableStack:  ctx.GlobalBool(DisableStackFlag.Name),
+	}
+	var (
+		tracer   vm.Tracer
+		debugger *vm.StructLogger
+	)
+	switch {
+	case ctx.GlobalBool(MachineFlag.Name):
+		tracer = NewJSONLogger(config, os.Stderr)
+
+	case ctx.GlobalBool(DebugFlag.Name):
+		debugger = vm.NewStructLogger(config)
+		tracer = debugger
+
+	default:
+		debugger = vm.NewStructLogger(config)
+	}
+	// Load the test content from the input file
+	src, err := ioutil.ReadFile(ctx.Args().First())
+	if err != nil {
+		return err
+	}
+	var tests map[string]tests.StateTest
+	if err = json.Unmarshal(src, &tests); err != nil {
+		return err
+	}
+	// Iterate over all the tests, run them and aggregate the results
+	cfg := vm.Config{
+		Tracer: tracer,
+		Debug:  ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
+	}
+	results := make([]StatetestResult, 0, len(tests))
+	for key, test := range tests {
+		for _, st := range test.Subtests() {
+			// Run the test and aggregate the result
+			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
+			if state, err := test.Run(st, cfg); err != nil {
+				// Test failed, mark as so and dump any state to aid debugging
+				result.Pass, result.Error = false, err.Error()
+				if ctx.GlobalBool(DumpFlag.Name) && state != nil {
+					dump := state.RawDump()
+					result.State = &dump
+				}
+			}
+			results = append(results, *result)
+
+			// Print any structured logs collected
+			if ctx.GlobalBool(DebugFlag.Name) {
+				if debugger != nil {
+					fmt.Fprintln(os.Stderr, "#### TRACE ####")
+					vm.WriteTrace(os.Stderr, debugger.StructLogs())
+				}
+			}
+		}
+	}
+	out, _ := json.MarshalIndent(results, "", "  ")
+	fmt.Println(string(out))
+	return nil
+}