From 5071d1dfba2f1effd3ef5e745a2ce16dcf9e4003 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Mon, 28 May 2018 12:48:58 -0600 Subject: test.py: initial requirements for test translation script --- test.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 test.py (limited to 'test.py') diff --git a/test.py b/test.py new file mode 100755 index 000000000..703dcf327 --- /dev/null +++ b/test.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +# Eventual goals: +# +# - Check validity of tests fillers. +# - Filter test fillers based on properties. +# - Convert between various test filler formats. + +# Non-goals: +# +# - Test filling. +# - Test post-state checking. + +# Current goals: +# +# - Generate GeneralStateTests from VMTests. + +# Input: +# +# - VMTest filler directory/name, without suffix Filler.json +# eg. vmArithmeticTest/add0 + +# Output: +# +# - GeneralStateTest filler +# eg. stVMTests/vmArithmeticTest/add0Filler.json + -- cgit v1.2.3 From bf6fea96d47bc64c2e0d538fce3d4d177cfbc0b1 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Mon, 28 May 2018 13:09:17 -0600 Subject: test.py: add generic infrastructure, usage messages, JSON reading, format command --- test.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'test.py') diff --git a/test.py b/test.py index 703dcf327..04c100542 100755 --- a/test.py +++ b/test.py @@ -25,3 +25,49 @@ # - GeneralStateTest filler # eg. stVMTests/vmArithmeticTest/add0Filler.json +import sys +import os +import json + +def _report(*msg): + print("== " + sys.argv[0] + ":", *msg, file=sys.stderr) + +def _die(*msg, exit_code=1): + _report(*msg) + _report("exiting...") + sys.exit(exit_code) + +def readJSONFile(fname): + if not os.path.isfile(fname): + _die("Not a file:", fname) + with open(fname, "r") as f: + fcontents = f.read() + return json.loads(fcontents) + +def writeJSONFile(fname, fcontents): + if not os.path.exists(os.path.dirname(fname)): + os.makedirs(os.path.dirname(fname)) + with open(fname, "w") as f: + f.write(json.dumps(fcontents, indent=4, sort_keys=True)) + +def _usage(): + usage_lines = [ "" + , " usage: " + sys.argv[0] + " format " + , " where:" + , " format: command to format/sort the JSON file." + , " : JSON test file/filler to read and write with sorted keys/standard formatting." + ] + _die("\n".join(usage_lines)) + +def main(): + if len(sys.argv) < 2: + _usage() + test_command = sys.argv[1] + if test_command == "format": + file_name = sys.argv[2] + writeJSONFile(file_name, readJSONFile(file_name)) + else: + _usage() + +if __name__ == "__main__": + main() -- cgit v1.2.3 From ea227d5d888542ebd7c6e7d504a1ddf1a534761a Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 09:46:00 -0600 Subject: test.py: add schema validator, individual files or all files --- test.py | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) (limited to 'test.py') diff --git a/test.py b/test.py index 04c100542..bb9e345d4 100755 --- a/test.py +++ b/test.py @@ -14,6 +14,12 @@ # Current goals: # # - Generate GeneralStateTests from VMTests. +# - Validate test inputs with JSON Schemas. + +# Dependencies: +# +# - python-json +# - python-jsonschema # Input: # @@ -28,6 +34,7 @@ import sys import os import json +import jsonschema def _report(*msg): print("== " + sys.argv[0] + ":", *msg, file=sys.stderr) @@ -50,11 +57,42 @@ def writeJSONFile(fname, fcontents): with open(fname, "w") as f: f.write(json.dumps(fcontents, indent=4, sort_keys=True)) +def findTests(testDir="."): + return [ os.path.join(root, file) for root, _, files in os.walk(testDir) + for file in files + if file.endswith(".json") + ] + +def validateSchema(jsonFile, schemaFile): + _report("validating", jsonFile, "with", schemaFile) + testSchema = readJSONFile(schemaFile) + jsonInput = readJSONFile(jsonFile) + jsonschema.validate(jsonInput, testSchema) + +def validateTestFile(jsonFile): + if jsonFile.startswith("src/GeneralStateTestsFiller/"): + validateSchema(jsonFile, "JSONSchema/st-filler-schema.json") + elif jsonFile.startswith("GeneralStateTests/"): + validateSchema(jsonFile, "JSONSchema/st-schema.json") + elif jsonFile.startswith("BlockchainTests/"): + validateSchema(jsonFile, "JSONSchema/bc-schema.json") + else: + _die("Do not know how to validate file:", jsonFile) + +def validateAllTests(): + for jsonFile in ( findTests(testDir="src/GeneralStateTestsFiller/") + + findTests(testDir="GeneralStateTests/") + + findTests(testDir="BlockchainTests/") + ): + validateTestFile(jsonFile) + def _usage(): usage_lines = [ "" - , " usage: " + sys.argv[0] + " format " + , " usage: " + sys.argv[0] + " format " + , " usage: " + sys.argv[0] + " validate [*]" , " where:" , " format: command to format/sort the JSON file." + , " validate: command to check a file against the associated JSON schema (defaults to all files)." , " : JSON test file/filler to read and write with sorted keys/standard formatting." ] _die("\n".join(usage_lines)) @@ -66,6 +104,12 @@ def main(): if test_command == "format": file_name = sys.argv[2] writeJSONFile(file_name, readJSONFile(file_name)) + elif test_command == "validate": + if len(sys.argv) > 2: + for testFile in sys.argv[2:]: + validateTestFile(testFile) + else: + validateAllTests() else: _usage() -- cgit v1.2.3 From 9bdd043874f02d7148319d1591a94094f72f135f Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 07:34:02 -0600 Subject: test.py, JSONSchema/*: factor out common schema definitions --- test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'test.py') diff --git a/test.py b/test.py index bb9e345d4..0db989cb1 100755 --- a/test.py +++ b/test.py @@ -65,9 +65,15 @@ def findTests(testDir="."): def validateSchema(jsonFile, schemaFile): _report("validating", jsonFile, "with", schemaFile) + testSchema = readJSONFile(schemaFile) - jsonInput = readJSONFile(jsonFile) - jsonschema.validate(jsonInput, testSchema) + defSchema = readJSONFile("JSONSchema/definitions.json") + schema = { "definitions" : dict(defSchema["definitions"], **testSchema["definitions"]) + , "patternProperties" : testSchema["patternProperties"] + } + + jsonInput = readJSONFile(jsonFile) + jsonschema.validate(jsonInput, schema) def validateTestFile(jsonFile): if jsonFile.startswith("src/GeneralStateTestsFiller/"): -- cgit v1.2.3 From 6ea80ef71b8d2975cdc840c3fe2c10979de8e4d4 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 13:03:41 -0600 Subject: test.py: more generic/common test listing infrastructure --- test.py | 63 ++++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 29 deletions(-) (limited to 'test.py') diff --git a/test.py b/test.py index 0db989cb1..146537a5a 100755 --- a/test.py +++ b/test.py @@ -57,15 +57,20 @@ def writeJSONFile(fname, fcontents): with open(fname, "w") as f: f.write(json.dumps(fcontents, indent=4, sort_keys=True)) -def findTests(testDir="."): - return [ os.path.join(root, file) for root, _, files in os.walk(testDir) - for file in files - if file.endswith(".json") +def findTests(filePrefix=""): + return [ fullTest for fullTest in [ os.path.join(root, file) for root, _, files in os.walk(".") + for file in files + if file.endswith(".json") + ] + if fullTest.startswith(filePrefix) ] -def validateSchema(jsonFile, schemaFile): - _report("validating", jsonFile, "with", schemaFile) +def listTests(filePrefixes=[""]): + return [ test for fPrefix in filePrefixes + for test in findTests(filePrefix=fPrefix) + ] +def validateSchema(jsonFile, schemaFile): testSchema = readJSONFile(schemaFile) defSchema = readJSONFile("JSONSchema/definitions.json") schema = { "definitions" : dict(defSchema["definitions"], **testSchema["definitions"]) @@ -76,30 +81,24 @@ def validateSchema(jsonFile, schemaFile): jsonschema.validate(jsonInput, schema) def validateTestFile(jsonFile): - if jsonFile.startswith("src/GeneralStateTestsFiller/"): + elif jsonFile.startswith("./src/GeneralStateTestsFiller/"): validateSchema(jsonFile, "JSONSchema/st-filler-schema.json") - elif jsonFile.startswith("GeneralStateTests/"): + elif jsonFile.startswith("./GeneralStateTests/"): validateSchema(jsonFile, "JSONSchema/st-schema.json") - elif jsonFile.startswith("BlockchainTests/"): + elif jsonFile.startswith("./BlockchainTests/"): validateSchema(jsonFile, "JSONSchema/bc-schema.json") else: _die("Do not know how to validate file:", jsonFile) -def validateAllTests(): - for jsonFile in ( findTests(testDir="src/GeneralStateTestsFiller/") - + findTests(testDir="GeneralStateTests/") - + findTests(testDir="BlockchainTests/") - ): - validateTestFile(jsonFile) - def _usage(): usage_lines = [ "" - , " usage: " + sys.argv[0] + " format " - , " usage: " + sys.argv[0] + " validate [*]" + , " usage: " + sys.argv[0] + " [list|format|validate] [*]" , " where:" - , " format: command to format/sort the JSON file." - , " validate: command to check a file against the associated JSON schema (defaults to all files)." - , " : JSON test file/filler to read and write with sorted keys/standard formatting." + , " list: command to list the matching tests." + , " format: command to format/sort the JSON file." + , " validate: command to check a file against the associated JSON schema (defaults to all files)." + , " : file path prefix to search for tests with." + , " eg. './src/VMTestsFiller' './VMTests' for all VMTests and their fillers." ] _die("\n".join(usage_lines)) @@ -107,17 +106,23 @@ def main(): if len(sys.argv) < 2: _usage() test_command = sys.argv[1] - if test_command == "format": - file_name = sys.argv[2] - writeJSONFile(file_name, readJSONFile(file_name)) + if len(sys.argv) == 2: + testList = listTests() + else: + testList = listTests(filePrefixes=sys.argv[2:]) + + if test_command == "list": + testDo = lambda t: print(t) + elif test_command == "format": + testDo = lambda t: writeJSONFile(t, readJSONFile(t)) elif test_command == "validate": - if len(sys.argv) > 2: - for testFile in sys.argv[2:]: - validateTestFile(testFile) - else: - validateAllTests() + testDo = validateTestFile else: _usage() + for test in testList: + _report(test_command + ":", test) + testDo(test) + if __name__ == "__main__": main() -- cgit v1.2.3 From e5c12114d6417b44137171588043679bfd6550b0 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 10:45:50 -0600 Subject: test.py, JSONSchema/{vm,vm-filler}-schema: add schemas for vm tests --- test.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'test.py') diff --git a/test.py b/test.py index 146537a5a..79fab1ec0 100755 --- a/test.py +++ b/test.py @@ -81,8 +81,12 @@ def validateSchema(jsonFile, schemaFile): jsonschema.validate(jsonInput, schema) def validateTestFile(jsonFile): + if jsonFile.startswith("./src/VMTestsFiller/"): + validateSchema(jsonFile, "JSONSchema/vm-filler-schema.json") elif jsonFile.startswith("./src/GeneralStateTestsFiller/"): validateSchema(jsonFile, "JSONSchema/st-filler-schema.json") + elif jsonFile.startswith("./VMTests/"): + validateSchema(jsonFile, "JSONSchema/vm-schema.json") elif jsonFile.startswith("./GeneralStateTests/"): validateSchema(jsonFile, "JSONSchema/st-schema.json") elif jsonFile.startswith("./BlockchainTests/"): -- cgit v1.2.3 From db66cd83f14c09dcc4934e5b46871450a42abb66 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 15:19:11 -0600 Subject: test.py, JSONSchema/bc-filler-schema: add BlockchainTests filler validation schema --- test.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test.py') diff --git a/test.py b/test.py index 79fab1ec0..b78123aee 100755 --- a/test.py +++ b/test.py @@ -85,6 +85,8 @@ def validateTestFile(jsonFile): validateSchema(jsonFile, "JSONSchema/vm-filler-schema.json") elif jsonFile.startswith("./src/GeneralStateTestsFiller/"): validateSchema(jsonFile, "JSONSchema/st-filler-schema.json") + elif jsonFile.startswith("./src/BlockchainTestsFiller/"): + validateSchema(jsonFile, "JSONSchema/bc-filler-schema.json") elif jsonFile.startswith("./VMTests/"): validateSchema(jsonFile, "JSONSchema/vm-schema.json") elif jsonFile.startswith("./GeneralStateTests/"): -- cgit v1.2.3 From 0c26bbe6b034710cb4a20903b111e3dadb2ba539 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 17:20:17 -0600 Subject: test.py: fail if no tests lists/run --- test.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'test.py') diff --git a/test.py b/test.py index b78123aee..45c0fb321 100755 --- a/test.py +++ b/test.py @@ -117,6 +117,9 @@ def main(): else: testList = listTests(filePrefixes=sys.argv[2:]) + if len(testList) == 0: + _die("No tests listed!!!") + if test_command == "list": testDo = lambda t: print(t) elif test_command == "format": -- cgit v1.2.3 From 6946875d23ea86d42e3cf1bdc06df536c88c0f4e Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Wed, 30 May 2018 12:37:29 -0600 Subject: test.py: execute through errors, reprint errors together at end --- test.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) (limited to 'test.py') diff --git a/test.py b/test.py index 45c0fb321..28a0670b8 100755 --- a/test.py +++ b/test.py @@ -36,9 +36,18 @@ import os import json import jsonschema +exit_status = 0 +error_log = [] + def _report(*msg): print("== " + sys.argv[0] + ":", *msg, file=sys.stderr) +def _logerror(*msg): + global exit_status + _report("ERROR:", *msg) + error_log.append(" ".join(msg)) + exit_status = 1 + def _die(*msg, exit_code=1): _report(*msg) _report("exiting...") @@ -78,23 +87,28 @@ def validateSchema(jsonFile, schemaFile): } jsonInput = readJSONFile(jsonFile) - jsonschema.validate(jsonInput, schema) + try: + jsonschema.validate(jsonInput, schema) + except: + _logerror("Validation failed:", "schema", schemaFile, "on", jsonFile) def validateTestFile(jsonFile): if jsonFile.startswith("./src/VMTestsFiller/"): - validateSchema(jsonFile, "JSONSchema/vm-filler-schema.json") + schemaFile = "JSONSchema/vm-filler-schema.json" elif jsonFile.startswith("./src/GeneralStateTestsFiller/"): - validateSchema(jsonFile, "JSONSchema/st-filler-schema.json") + schemaFile = "JSONSchema/st-filler-schema.json" elif jsonFile.startswith("./src/BlockchainTestsFiller/"): - validateSchema(jsonFile, "JSONSchema/bc-filler-schema.json") + schemaFile = "JSONSchema/bc-filler-schema.json" elif jsonFile.startswith("./VMTests/"): - validateSchema(jsonFile, "JSONSchema/vm-schema.json") + schemaFile = "JSONSchema/vm-schema.json" elif jsonFile.startswith("./GeneralStateTests/"): - validateSchema(jsonFile, "JSONSchema/st-schema.json") + schemaFile = "JSONSchema/st-schema.json" elif jsonFile.startswith("./BlockchainTests/"): - validateSchema(jsonFile, "JSONSchema/bc-schema.json") + schemaFile = "JSONSchema/bc-schema.json" else: - _die("Do not know how to validate file:", jsonFile) + _logerror("Do not know how to validate file:", jsonFile) + return + validateSchema(jsonFile, schemaFile) def _usage(): usage_lines = [ "" @@ -133,5 +147,8 @@ def main(): _report(test_command + ":", test) testDo(test) + if exit_status != 0: + _die("Errors reported!\n[ERROR] " + "\n[ERROR] ".join(error_log)) + if __name__ == "__main__": main() -- cgit v1.2.3 From 9b213d1e29353bc2f123177f33a412e6dbbc5c09 Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Tue, 29 May 2018 15:57:01 -0600 Subject: test.py: update documentation/organization --- test.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) (limited to 'test.py') diff --git a/test.py b/test.py index 28a0670b8..33d397711 100755 --- a/test.py +++ b/test.py @@ -1,9 +1,15 @@ #!/usr/bin/env python3 -# Eventual goals: +# For help: # -# - Check validity of tests fillers. -# - Filter test fillers based on properties. +# - Run with no arguments. +# - Ask Everett Hildenbrandt (@ehildenb). + +# Goals: +# +# - Validate test inputs with JSON Schemas. +# - Check that tests have been filled. +# - Filter tests based on properties. # - Convert between various test filler formats. # Non-goals: @@ -11,31 +17,21 @@ # - Test filling. # - Test post-state checking. -# Current goals: -# -# - Generate GeneralStateTests from VMTests. -# - Validate test inputs with JSON Schemas. - # Dependencies: # # - python-json # - python-jsonschema -# Input: -# -# - VMTest filler directory/name, without suffix Filler.json -# eg. vmArithmeticTest/add0 - -# Output: -# -# - GeneralStateTest filler -# eg. stVMTests/vmArithmeticTest/add0Filler.json - import sys import os import json import jsonschema +# Utilities +# ========= + +# Errors/Reporting + exit_status = 0 error_log = [] @@ -53,6 +49,8 @@ def _die(*msg, exit_code=1): _report("exiting...") sys.exit(exit_code) +# Filesystem/parsing + def readJSONFile(fname): if not os.path.isfile(fname): _die("Not a file:", fname) @@ -66,6 +64,11 @@ def writeJSONFile(fname, fcontents): with open(fname, "w") as f: f.write(json.dumps(fcontents, indent=4, sort_keys=True)) +# Functionality +# ============= + +# Listing tests + def findTests(filePrefix=""): return [ fullTest for fullTest in [ os.path.join(root, file) for root, _, files in os.walk(".") for file in files @@ -79,6 +82,8 @@ def listTests(filePrefixes=[""]): for test in findTests(filePrefix=fPrefix) ] +# Schema Validation + def validateSchema(jsonFile, schemaFile): testSchema = readJSONFile(schemaFile) defSchema = readJSONFile("JSONSchema/definitions.json") @@ -110,6 +115,9 @@ def validateTestFile(jsonFile): return validateSchema(jsonFile, schemaFile) +# Main +# ==== + def _usage(): usage_lines = [ "" , " usage: " + sys.argv[0] + " [list|format|validate] [*]" -- cgit v1.2.3 From 7bfd7998cfc9d4f59f0bc29d9718709d3d15a81e Mon Sep 17 00:00:00 2001 From: Everett Hildenbrandt Date: Wed, 30 May 2018 12:41:43 -0600 Subject: test.py: add checkFilled command for ensuring that tests are filled --- test.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'test.py') diff --git a/test.py b/test.py index 33d397711..e7957989d 100755 --- a/test.py +++ b/test.py @@ -21,11 +21,13 @@ # # - python-json # - python-jsonschema +# - python-pysha3 import sys import os import json import jsonschema +import sha3 # Utilities # ========= @@ -115,6 +117,32 @@ def validateTestFile(jsonFile): return validateSchema(jsonFile, schemaFile) +# Check tests filled + +def hashFile(fname): + with open(fname ,"rb") as f: + k = sha3.keccak_256() + k.update(f.read()) + return k.hexdigest() + +def checkFilled(jsonFile): + jsonTest = readJSONFile(jsonFile) + if not ( jsonFile.startswith("./src/BlockchainTestsFiller/GeneralStateTests/") + # or jsonFile.startswith("./src/BlockchainTestsFiller/VMTests/") + or jsonFile.startswith("./VMTests/") + or jsonFile.startswith("./GeneralStateTests/") + or jsonFile.startswith("./TransactionTests/") + or jsonFile.startswith("./BlockchainTests/") + ): + _report("Not a file that is filled:", jsonFile) + return + for test in jsonTest: + if "_info" in jsonTest[test]: + fillerSource = jsonTest[test]["_info"]["source"] + fillerHash = jsonTest[test]["_info"]["sourceHash"] + if fillerHash != hashFile(fillerSource): + _logerror("Test must be filled:", jsonFile) + # Main # ==== @@ -148,6 +176,8 @@ def main(): testDo = lambda t: writeJSONFile(t, readJSONFile(t)) elif test_command == "validate": testDo = validateTestFile + elif test_command == "checkFilled": + testDo = checkFilled else: _usage() -- cgit v1.2.3