Refactor syntax test infrastructure to prepare introducing semantics tests.

7 files changed, 307 insertions, 200 deletions

diff --git a/test/boostTest.cpp b/test/boostTest.cpp
index f16973b5..145be6e4 100644
--- a/test/boostTest.cpp
+++ b/test/boostTest.cpp
@@ -38,7 +38,26 @@
 #include <test/Options.h>
 #include <test/libsolidity/SyntaxTest.h>
 
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/filesystem.hpp>
+
 using namespace boost::unit_test;
+using namespace dev::solidity::test;
+namespace fs = boost::filesystem;
+using namespace std;
+
+#if BOOST_VERSION < 105900
+test_case *make_test_case(
+	function<void()> const& _fn,
+	string const& _name,
+	string const& /* _filename */,
+	size_t /* _line */
+)
+{
+	return make_test_case(_fn, _name);
+}
+#endif
 
 namespace
 {
@@ -49,6 +68,56 @@ void removeTestSuite(std::string const& _name)
 	assert(id != INV_TEST_UNIT_ID);
 	master.remove(id);
 }
+
+int registerTests(
+	boost::unit_test::test_suite& _suite,
+	boost::filesystem::path const& _basepath,
+	boost::filesystem::path const& _path,
+	TestCase::TestCaseCreator _testCaseCreator
+)
+{
+	int numTestsAdded = 0;
+	fs::path fullpath = _basepath / _path;
+	if (fs::is_directory(fullpath))
+	{
+		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
+		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
+			fs::directory_iterator(fullpath),
+			fs::directory_iterator()
+		))
+			if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
+				numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _testCaseCreator);
+		_suite.add(sub_suite);
+	}
+	else
+	{
+		static vector<unique_ptr<string>> filenames;
+
+		filenames.emplace_back(new string(_path.string()));
+		_suite.add(make_test_case(
+			[fullpath, _testCaseCreator]
+			{
+				BOOST_REQUIRE_NO_THROW({
+					try
+					{
+						stringstream errorStream;
+						if (!_testCaseCreator(fullpath.string())->run(errorStream))
+							BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
+					}
+					catch (boost::exception const& _e)
+					{
+						BOOST_ERROR("Exception during extracted test: " << boost::diagnostic_information(_e));
+					}
+			   });
+			},
+			_path.stem().string(),
+			*filenames.back(),
+			0
+		));
+		numTestsAdded = 1;
+	}
+	return numTestsAdded;
+}
 }
 
 test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
@@ -56,10 +125,11 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 	master_test_suite_t& master = framework::master_test_suite();
 	master.p_name.value = "SolidityTests";
 	dev::test::Options::get().validate();
-	solAssert(dev::solidity::test::SyntaxTest::registerTests(
+	solAssert(registerTests(
 		master,
 		dev::test::Options::get().testPath / "libsolidity",
-		"syntaxTests"
+		"syntaxTests",
+		SyntaxTest::create
 	) > 0, "no syntax tests found");
 	if (dev::test::Options::get().disableIPC)
 	{
diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp
index 430073a0..1de42300 100644
--- a/test/libsolidity/SyntaxTest.cpp
+++ b/test/libsolidity/SyntaxTest.cpp
@@ -33,28 +33,10 @@ using namespace std;
 namespace fs = boost::filesystem;
 using namespace boost::unit_test;
 
-template<typename IteratorType>
-void skipWhitespace(IteratorType& _it, IteratorType _end)
+namespace
 {
-	while (_it != _end && isspace(*_it))
-		++_it;
-}
-
-template<typename IteratorType>
-void skipSlashes(IteratorType& _it, IteratorType _end)
-{
-	while (_it != _end && *_it == '/')
-		++_it;
-}
-
-void expect(string::iterator& _it, string::iterator _end, string::value_type _c)
-{
-	if (_it == _end || *_it != _c)
-		throw runtime_error(string("Invalid test expectation. Expected: \"") + _c + "\".");
-	++_it;
-}
 
-int parseUnsignedInteger(string::iterator &_it, string::iterator _end)
+int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
 {
 	if (_it == _end || !isdigit(*_it))
 		throw runtime_error("Invalid test expectation. Source location expected.");
@@ -68,6 +50,8 @@ int parseUnsignedInteger(string::iterator &_it, string::iterator _end)
 	return result;
 }
 
+}
+
 SyntaxTest::SyntaxTest(string const& _filename)
 {
 	ifstream file(_filename);
@@ -120,6 +104,55 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _fo
 	return true;
 }
 
+void SyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool const _formatted) const
+{
+	if (_formatted)
+	{
+		if (m_source.empty())
+			return;
+
+		vector<char const*> sourceFormatting(m_source.length(), formatting::RESET);
+		for (auto const& error: m_errorList)
+			if (error.locationStart >= 0 && error.locationEnd >= 0)
+			{
+				assert(static_cast<size_t>(error.locationStart) <= m_source.length());
+				assert(static_cast<size_t>(error.locationEnd) <= m_source.length());
+				bool isWarning = error.type == "Warning";
+				for (int i = error.locationStart; i < error.locationEnd; i++)
+					if (isWarning)
+					{
+						if (sourceFormatting[i] == formatting::RESET)
+							sourceFormatting[i] = formatting::ORANGE_BACKGROUND;
+					}
+					else
+						sourceFormatting[i] = formatting::RED_BACKGROUND;
+			}
+
+		_stream << _linePrefix << sourceFormatting.front() << m_source.front();
+		for (size_t i = 1; i < m_source.length(); i++)
+		{
+			if (sourceFormatting[i] != sourceFormatting[i - 1])
+				_stream << sourceFormatting[i];
+			if (m_source[i] != '\n')
+				_stream << m_source[i];
+			else
+			{
+				_stream << formatting::RESET << endl;
+				if (i + 1 < m_source.length())
+					_stream << _linePrefix << sourceFormatting[i];
+			}
+		}
+		_stream << formatting::RESET;
+	}
+	else
+	{
+		stringstream stream(m_source);
+		string line;
+		while (getline(stream, line))
+			_stream << _linePrefix << line << endl;
+	}
+}
+
 void SyntaxTest::printErrorList(
 	ostream& _stream,
 	vector<SyntaxTestError> const& _errorList,
@@ -159,19 +192,6 @@ string SyntaxTest::errorMessage(Exception const& _e)
 		return "NONE";
 }
 
-string SyntaxTest::parseSource(istream& _stream)
-{
-	string source;
-	string line;
-	string const delimiter("// ----");
-	while (getline(_stream, line))
-		if (boost::algorithm::starts_with(line, delimiter))
-			break;
-		else
-			source += line + "\n";
-	return source;
-}
-
 vector<SyntaxTestError> SyntaxTest::parseExpectations(istream& _stream)
 {
 	vector<SyntaxTestError> expectations;
@@ -220,71 +240,3 @@ vector<SyntaxTestError> SyntaxTest::parseExpectations(istream& _stream)
 	}
 	return expectations;
 }
-
-#if BOOST_VERSION < 105900
-test_case *make_test_case(
-	function<void()> const& _fn,
-	string const& _name,
-	string const& /* _filename */,
-	size_t /* _line */
-)
-{
-	return make_test_case(_fn, _name);
-}
-#endif
-
-bool SyntaxTest::isTestFilename(boost::filesystem::path const& _filename)
-{
-	return _filename.extension().string() == ".sol" &&
-		!boost::starts_with(_filename.string(), "~") &&
-		!boost::starts_with(_filename.string(), ".");
-}
-
-int SyntaxTest::registerTests(
-	boost::unit_test::test_suite& _suite,
-	boost::filesystem::path const& _basepath,
-	boost::filesystem::path const& _path
-)
-{
-	int numTestsAdded = 0;
-	fs::path fullpath = _basepath / _path;
-	if (fs::is_directory(fullpath))
-	{
-		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
-		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
-			fs::directory_iterator(fullpath),
-			fs::directory_iterator()
-		))
-			if (fs::is_directory(entry.path()) || isTestFilename(entry.path().filename()))
-				numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
-		_suite.add(sub_suite);
-	}
-	else
-	{
-		static vector<unique_ptr<string>> filenames;
-
-		filenames.emplace_back(new string(_path.string()));
-		_suite.add(make_test_case(
-			[fullpath]
-			{
-				BOOST_REQUIRE_NO_THROW({
-					try
-					{
-						stringstream errorStream;
-						if (!SyntaxTest(fullpath.string()).run(errorStream))
-							BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
-					}
-					catch (boost::exception const& _e)
-					{
-						BOOST_ERROR("Exception during syntax test: " << boost::diagnostic_information(_e));
-					}
-				});
-			},
-			_path.stem().string(),
-			*filenames.back(),
-			0
-		));
-		numTestsAdded = 1;
-	}
-	return numTestsAdded;
-}
diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h
index 6159e789..e9e36aa6 100644
--- a/test/libsolidity/SyntaxTest.h
+++ b/test/libsolidity/SyntaxTest.h
@@ -19,11 +19,9 @@
 
 #include <test/libsolidity/AnalysisFramework.h>
 #include <test/libsolidity/FormattedScope.h>
+#include <test/libsolidity/TestCase.h>
 #include <libsolidity/interface/Exceptions.h>
 
-#include <boost/noncopyable.hpp>
-#include <boost/test/unit_test.hpp>
-
 #include <iosfwd>
 #include <string>
 #include <vector>
@@ -52,17 +50,24 @@ struct SyntaxTestError
 };
 
 
-class SyntaxTest: AnalysisFramework
+class SyntaxTest: AnalysisFramework, public TestCase
 {
 public:
+	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	{ return std::unique_ptr<TestCase>(new SyntaxTest(_filename)); }
 	SyntaxTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false);
+	virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
-	std::vector<SyntaxTestError> const& expectations() const { return m_expectations; }
-	std::string const& source() const { return m_source; }
-	std::vector<SyntaxTestError> const& errorList() const { return m_errorList; }
+	virtual void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
+	virtual void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override
+	{
+		if (!m_errorList.empty())
+			printErrorList(_stream, m_errorList, _linePrefix, false);
+	}
 
+	static std::string errorMessage(Exception const& _e);
+private:
 	static void printErrorList(
 		std::ostream& _stream,
 		std::vector<SyntaxTestError> const& _errors,
@@ -70,15 +75,6 @@ public:
 		bool const _formatted = false
 	);
 
-	static int registerTests(
-		boost::unit_test::test_suite& _suite,
-		boost::filesystem::path const& _basepath,
-		boost::filesystem::path const& _path
-	);
-	static bool isTestFilename(boost::filesystem::path const& _filename);
-	static std::string errorMessage(Exception const& _e);
-private:
-	static std::string parseSource(std::istream& _stream);
 	static std::vector<SyntaxTestError> parseExpectations(std::istream& _stream);
 
 	std::string m_source;
diff --git a/test/libsolidity/TestCase.cpp b/test/libsolidity/TestCase.cpp
new file mode 100644
index 00000000..6c4e0aea
--- /dev/null
+++ b/test/libsolidity/TestCase.cpp
@@ -0,0 +1,55 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/TestCase.h>
+
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+
+bool TestCase::isTestFilename(boost::filesystem::path const& _filename)
+{
+	return _filename.extension().string() == ".sol" &&
+		   !boost::starts_with(_filename.string(), "~") &&
+		   !boost::starts_with(_filename.string(), ".");
+}
+
+string TestCase::parseSource(istream& _stream)
+{
+	string source;
+	string line;
+	string const delimiter("// ----");
+	while (getline(_stream, line))
+		if (boost::algorithm::starts_with(line, delimiter))
+			break;
+		else
+			source += line + "\n";
+	return source;
+}
+
+void TestCase::expect(string::iterator& _it, string::iterator _end, string::value_type _c)
+{
+	if (_it == _end || *_it != _c)
+		throw runtime_error(string("Invalid test expectation. Expected: \"") + _c + "\".");
+	++_it;
+}
diff --git a/test/libsolidity/TestCase.h b/test/libsolidity/TestCase.h
new file mode 100644
index 00000000..3c05ae4e
--- /dev/null
+++ b/test/libsolidity/TestCase.h
@@ -0,0 +1,80 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <boost/filesystem.hpp>
+
+#include <iosfwd>
+#include <memory>
+#include <string>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+/** Common superclass of SyntaxTest and SemanticsTest. */
+class TestCase
+{
+public:
+	using TestCaseCreator = std::unique_ptr<TestCase>(*)(std::string const&);
+
+	virtual ~TestCase() {}
+
+	/// Runs the test case.
+	/// Outputs error messages to @arg _stream. Each line of output is prefixed with @arg _linePrefix.
+	/// Optionally, color-coding can be enabled (if @arg _formatted is set to true).
+	/// @returns true, if the test case succeeds, false otherwise
+	virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
+
+	/// Outputs the test contract to @arg _stream.
+	/// Each line of output is prefixed with @arg _linePrefix.
+	/// If @arg _formatted is true, color-coding may be used to indicate
+	/// error locations in the contract, if applicable.
+	virtual void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const = 0;
+	/// Outputs test expectations to @arg _stream that match the actual results of the test.
+	/// Each line of output is prefixed with @arg _linePrefix.
+	virtual void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const = 0;
+
+	static bool isTestFilename(boost::filesystem::path const& _filename);
+
+protected:
+	static std::string parseSource(std::istream& _file);
+	static void expect(std::string::iterator& _it, std::string::iterator _end, std::string::value_type _c);
+
+	template<typename IteratorType>
+	static void skipWhitespace(IteratorType& _it, IteratorType _end)
+	{
+		while (_it != _end && isspace(*_it))
+			++_it;
+	}
+
+	template<typename IteratorType>
+	static void skipSlashes(IteratorType& _it, IteratorType _end)
+	{
+		while (_it != _end && *_it == '/')
+			++_it;
+	}
+
+};
+
+}
+}
+}
diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt
index 11714017..257b4f24 100644
--- a/test/tools/CMakeLists.txt
+++ b/test/tools/CMakeLists.txt
@@ -1,5 +1,7 @@
 add_executable(solfuzzer fuzzer.cpp)
 target_link_libraries(solfuzzer PRIVATE libsolc evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES} ${Boost_SYSTEM_LIBRARIES})
 
-add_executable(isoltest isoltest.cpp ../Options.cpp ../libsolidity/SyntaxTest.cpp ../libsolidity/AnalysisFramework.cpp)
+add_executable(isoltest isoltest.cpp ../Options.cpp ../libsolidity/TestCase.cpp ../libsolidity/SyntaxTest.cpp
+        ../libsolidity/AnalysisFramework.cpp ../libsolidity/SolidityExecutionFramework.cpp ../ExecutionFramework.cpp
+        ../RPCSession.cpp)
 target_link_libraries(isoltest PRIVATE libsolc solidity evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp
index d4b99e9d..41dff148 100644
--- a/test/tools/isoltest.cpp
+++ b/test/tools/isoltest.cpp
@@ -37,18 +37,22 @@ using namespace std;
 namespace po = boost::program_options;
 namespace fs = boost::filesystem;
 
-struct SyntaxTestStats
+struct TestStats
 {
 	int successCount;
 	int runCount;
 	operator bool() const { return successCount == runCount; }
 };
 
-class SyntaxTestTool
+class TestTool
 {
 public:
-	SyntaxTestTool(string const& _name, fs::path const& _path, bool _formatted):
-		m_formatted(_formatted), m_name(_name), m_path(_path)
+	TestTool(
+		TestCase::TestCaseCreator _testCaseCreator,
+		string const& _name,
+		fs::path const& _path,
+		bool _formatted
+	): m_testCaseCreator(_testCaseCreator), m_formatted(_formatted), m_name(_name), m_path(_path)
 	{}
 
 	enum class Result
@@ -60,7 +64,8 @@ public:
 
 	Result process();
 
-	static SyntaxTestStats processPath(
+	static TestStats processPath(
+		TestCase::TestCaseCreator _testCaseCreator,
 		fs::path const& _basepath,
 		fs::path const& _path,
 		bool const _formatted
@@ -77,68 +82,16 @@ private:
 
 	Request handleResponse(bool const _exception);
 
-	void printContract() const;
-
+	TestCase::TestCaseCreator m_testCaseCreator;
 	bool const m_formatted;
 	string const m_name;
 	fs::path const m_path;
-	unique_ptr<SyntaxTest> m_test;
+	unique_ptr<TestCase> m_test;
 };
 
-string SyntaxTestTool::editor;
-
-void SyntaxTestTool::printContract() const
-{
-	if (m_formatted)
-	{
-		string const& source = m_test->source();
-		if (source.empty())
-			return;
-
-		std::vector<char const*> sourceFormatting(source.length(), formatting::RESET);
-		for (auto const& error: m_test->errorList())
-			if (error.locationStart >= 0 && error.locationEnd >= 0)
-			{
-				assert(static_cast<size_t>(error.locationStart) <= source.length());
-				assert(static_cast<size_t>(error.locationEnd) <= source.length());
-				bool isWarning = error.type == "Warning";
-				for (int i = error.locationStart; i < error.locationEnd; i++)
-					if (isWarning)
-					{
-						if (sourceFormatting[i] == formatting::RESET)
-							sourceFormatting[i] = formatting::ORANGE_BACKGROUND;
-					}
-					else
-						sourceFormatting[i] = formatting::RED_BACKGROUND;
-			}
-
-		cout << "    " << sourceFormatting.front() << source.front();
-		for (size_t i = 1; i < source.length(); i++)
-		{
-			if (sourceFormatting[i] != sourceFormatting[i - 1])
-				cout << sourceFormatting[i];
-			if (source[i] != '\n')
-				cout << source[i];
-			else
-			{
-				cout << formatting::RESET << endl;
-				if (i + 1 < source.length())
-					cout << "    " << sourceFormatting[i];
-			}
-		}
-		cout << formatting::RESET << endl;
-	}
-	else
-	{
-		stringstream stream(m_test->source());
-		string line;
-		while (getline(stream, line))
-			cout << "    " << line << endl;
-		cout << endl;
-	}
-}
+string TestTool::editor;
 
-SyntaxTestTool::Result SyntaxTestTool::process()
+TestTool::Result TestTool::process()
 {
 	bool success;
 	std::stringstream outputMessages;
@@ -147,7 +100,7 @@ SyntaxTestTool::Result SyntaxTestTool::process()
 
 	try
 	{
-		m_test = unique_ptr<SyntaxTest>(new SyntaxTest(m_path.string()));
+		m_test = m_testCaseCreator(m_path.string());
 		success = m_test->run(outputMessages, "  ", m_formatted);
 	}
 	catch(boost::exception const& _e)
@@ -179,14 +132,14 @@ SyntaxTestTool::Result SyntaxTestTool::process()
 		FormattedScope(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl;
 
 		FormattedScope(cout, m_formatted, {BOLD, CYAN}) << "  Contract:" << endl;
-		printContract();
+		m_test->printSource(cout, "    ", m_formatted);
 
-		cout << outputMessages.str() << endl;
+		cout << endl << outputMessages.str() << endl;
 		return Result::Failure;
 	}
 }
 
-SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception)
+TestTool::Request TestTool::handleResponse(bool const _exception)
 {
 	if (_exception)
 		cout << "(e)dit/(s)kip/(q)uit? ";
@@ -208,15 +161,14 @@ SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception)
 			{
 				cout << endl;
 				ofstream file(m_path.string(), ios::trunc);
-				file << m_test->source();
+				m_test->printSource(file);
 				file << "// ----" << endl;
-				if (!m_test->errorList().empty())
-					m_test->printErrorList(file, m_test->errorList(), "// ", false);
+				m_test->printUpdatedExpectations(file, "// ");
 				return Request::Rerun;
 			}
 		case 'e':
 			cout << endl << endl;
-			if (system((editor + " \"" + m_path.string() + "\"").c_str()))
+			if (system((TestTool::editor + " \"" + m_path.string() + "\"").c_str()))
 				cerr << "Error running editor command." << endl << endl;
 			return Request::Rerun;
 		case 'q':
@@ -228,8 +180,8 @@ SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception)
 	}
 }
 
-
-SyntaxTestStats SyntaxTestTool::processPath(
+TestStats TestTool::processPath(
+	TestCase::TestCaseCreator _testCaseCreator,
 	fs::path const& _basepath,
 	fs::path const& _path,
 	bool const _formatted
@@ -252,12 +204,12 @@ SyntaxTestStats SyntaxTestTool::processPath(
 				fs::directory_iterator(fullpath),
 				fs::directory_iterator()
 			))
-				if (fs::is_directory(entry.path()) || SyntaxTest::isTestFilename(entry.path().filename()))
+				if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
 					paths.push(currentPath / entry.path().filename());
 		}
 		else
 		{
-			SyntaxTestTool testTool(currentPath.string(), fullpath, _formatted);
+			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _formatted);
 			++runCount;
 			auto result = testTool.process();
 
@@ -293,9 +245,9 @@ SyntaxTestStats SyntaxTestTool::processPath(
 int main(int argc, char *argv[])
 {
 	if (getenv("EDITOR"))
-		SyntaxTestTool::editor = getenv("EDITOR");
+		TestTool::editor = getenv("EDITOR");
 	else if (fs::exists("/usr/bin/editor"))
-		SyntaxTestTool::editor = "/usr/bin/editor";
+		TestTool::editor = "/usr/bin/editor";
 
 	fs::path testPath;
 	bool formatted = true;
@@ -311,7 +263,7 @@ Allowed options)",
 		("help", "Show this help screen.")
 		("testpath", po::value<fs::path>(&testPath), "path to test files")
 		("no-color", "don't use colors")
-		("editor", po::value<string>(&SyntaxTestTool::editor), "editor for opening contracts");
+		("editor", po::value<string>(&TestTool::editor), "editor for opening contracts");
 
 	po::variables_map arguments;
 	try
@@ -331,7 +283,7 @@ Allowed options)",
 
 		po::notify(arguments);
 	}
-	catch (po::error const& _exception)
+	catch (std::exception const& _exception)
 	{
 		cerr << _exception.what() << endl;
 		return 1;
@@ -362,7 +314,7 @@ Allowed options)",
 
 	if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath))
 	{
-		auto stats = SyntaxTestTool::processPath(testPath / "libsolidity", "syntaxTests", formatted);
+		auto stats = TestTool::processPath(SyntaxTest::create, testPath / "libsolidity", "syntaxTests", formatted);
 
 		cout << endl << "Summary: ";
 		FormattedScope(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
@@ -373,7 +325,7 @@ Allowed options)",
 	}
 	else
 	{
-		cerr << "Test path not found. Use the --testpath argument." << endl;
+		cerr << "Syntax tests not found. Use the --testpath argument." << endl;
 		return 1;
 	}
 }