Makefile, .travis.yml: test sanitation of VMTests via python file

-   disallow the sol and md files in src/VMTestsFiller
-   only handling VMTests currently, because of massive formatting changes needed

1 files changed, 28 insertions, 14 deletions

diff --git a/Makefile b/Makefile
index 0e0d4df84..e8e5e29a4 100644
--- a/Makefile
+++ b/Makefile
@@ -5,26 +5,40 @@ tx_tests:=$(wildcard TransactionTests/*)
 gs_tests:=$(wildcard GeneralStateTests/*)
 bc_tests:=$(wildcard BlockchainTests/*)
 vm_tests:=$(wildcard VMTests/*)
-all_tests:=$(tx_tests) $(gs_tests) $(bc_tests) $(vm_tests)
+all_tests:=$(gs_tests) $(bc_tests) $(vm_tests)
 
-fill-tx: $(tx_tests:=.fill)
-fill-gs: $(gs_tests:=.fill)
-fill-bc: $(bc_tests:=.fill)
-fill-vm: $(vm_tests:=.fill)
-fill-all: fill-tx fill-gs fill-bc fill-vm
+tx_fillers:=$(wildcard src/TransactionTestsFiller/*)
+gs_fillers:=$(wildcard src/GeneralStateTestsFiller/*)
+bc_fillers:=$(wildcard src/BlockchainTestsFiller/*)
+vm_fillers:=$(filter-out %.sol %.md, $(wildcard src/VMTestsFiller/*))
+all_fillers:=$(gs_fillers) $(bc_fillers) $(vm_fillers)
 
-test-tx: $(tx_tests:=.test)
-test-gs: $(gs_tests:=.test)
-test-bc: $(bc_tests:=.test)
-test-vm: $(vm_tests:=.test)
-test-all: test-tx test-gs test-bc test-vm
+all_schemas:=$(wildcard JSONSchema/*.json)
 
-# Testing command
+# Testset sanitation
 
-%.test:
+sani: $(all_schemas:=.format) $(vm_fillers:=.format) $(vm_tests:=.format)
+
+%.format:
+	python3 test.py format ./$*
+	git diff --quiet --exit-code &>/dev/null
+
+%.sani:
+	python3 test.py validate    ./$*
+	python3 test.py checkFilled ./$*
+
+# Test running command
+
+run-tests:=$(all-tests:=.test)
+run: $(run-tests)
+
+%.run:
 	testeth -t $* -- --verbosity 2
 
-# Actual filling command
+# Test filling command
+
+fill-tests:=$(all-tests:=.fill)
+fill: $(fill-tests)
 
 %.fill:
 	testeth -t $* -- --filltests --verbosity 2