Merge pull request #3709 from ethereum/syntaxTestsInteractiveTool

Interactive syntax test tool.

5 files changed, 478 insertions, 16 deletions

diff --git a/test/libsolidity/FormattedScope.h b/test/libsolidity/FormattedScope.h
new file mode 100644
index 00000000..78560848
--- /dev/null
+++ b/test/libsolidity/FormattedScope.h
@@ -0,0 +1,66 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <boost/noncopyable.hpp>
+
+#include <ostream>
+#include <vector>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+namespace formatting
+{
+
+static constexpr char const* RESET = "\033[0m";
+static constexpr char const* RED  = "\033[1;31m";
+static constexpr char const* GREEN = "\033[1;32m";
+static constexpr char const* YELLOW = "\033[1;33m";
+static constexpr char const* CYAN = "\033[1;36m";
+static constexpr char const* BOLD = "\033[1m";
+static constexpr char const* INVERSE = "\033[7m";
+
+}
+
+class FormattedScope: boost::noncopyable
+{
+public:
+	/// @arg _formatting List of formatting strings (e.g. colors) defined in the formatting namespace.
+	FormattedScope(std::ostream& _stream, bool const _enabled, std::vector<char const*> const& _formatting):
+		m_stream(_stream), m_enabled(_enabled)
+	{
+		if (m_enabled)
+			for (auto const& format: _formatting)
+				m_stream << format;
+	}
+	~FormattedScope() { if (m_enabled) m_stream << formatting::RESET; }
+	template<typename T>
+	std::ostream& operator<<(T&& _t) { return m_stream << std::forward<T>(_t); }
+private:
+	std::ostream& m_stream;
+	bool m_enabled;
+};
+
+}
+}
+}
diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp
index 45a851b6..acfdff29 100644
--- a/test/libsolidity/SyntaxTest.cpp
+++ b/test/libsolidity/SyntaxTest.cpp
@@ -27,6 +27,7 @@
 using namespace dev;
 using namespace solidity;
 using namespace dev::solidity::test;
+using namespace dev::solidity::test::formatting;
 using namespace std;
 namespace fs = boost::filesystem;
 using namespace boost::unit_test;
@@ -56,41 +57,76 @@ SyntaxTest::SyntaxTest(string const& _filename)
 	m_expectations = parseExpectations(file);
 }
 
-bool SyntaxTest::run(ostream& _stream, string const& _indent)
+bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	m_errorList = parseAnalyseAndReturnError(m_source, true, true, true).second;
 	if (!matchesExpectations(m_errorList))
 	{
-		std::string nextIndentLevel = _indent + "\t";
-		_stream << _indent << "Expected result:" << endl;
-		printExpected(_stream, nextIndentLevel);
-		_stream << _indent << "Obtained result:\n";
-		printErrorList(_stream, m_errorList, nextIndentLevel);
+		std::string nextIndentLevel = _linePrefix + "  ";
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
+		printExpected(_stream, nextIndentLevel, _formatted);
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:\n";
+		printErrorList(_stream, m_errorList, nextIndentLevel, false, false, _formatted);
 		return false;
 	}
 	return true;
 }
 
-void SyntaxTest::printExpected(ostream& _stream, string const& _indent) const
+void SyntaxTest::printExpected(ostream& _stream, string const& _linePrefix, bool const _formatted) const
 {
 	if (m_expectations.empty())
-		_stream << _indent << "Success" << endl;
+		FormattedScope(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl;
 	else
 		for (auto const& expectation: m_expectations)
-			_stream << _indent << expectation.type << ": " << expectation.message << endl;
+		{
+			FormattedScope(_stream, _formatted, {BOLD, expectation.type == "Warning" ? YELLOW : RED}) <<
+				_linePrefix << expectation.type << ": ";
+			_stream << expectation.message << endl;
+		}
 }
 
 void SyntaxTest::printErrorList(
 	ostream& _stream,
 	ErrorList const& _errorList,
-	string const& _indent
+	string const& _linePrefix,
+	bool const _ignoreWarnings,
+	bool const _lineNumbers,
+	bool const _formatted
 ) const
 {
 	if (_errorList.empty())
-		_stream << _indent << "Success" << endl;
+		FormattedScope(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl;
 	else
 		for (auto const& error: _errorList)
-			_stream << _indent << error->typeName() << ": " << errorMessage(*error) << endl;
+		{
+			bool isWarning = (error->type() == Error::Type::Warning);
+			if (isWarning && _ignoreWarnings) continue;
+
+			{
+				FormattedScope scope(_stream, _formatted, {BOLD, isWarning ? YELLOW : RED});
+				_stream << _linePrefix;
+				if (_lineNumbers)
+				{
+					int line = offsetToLineNumber(
+						boost::get_error_info<errinfo_sourceLocation>(*error)->start
+					);
+					if (line >= 0)
+						_stream << "(" << line << "): ";
+				}
+				_stream << error->typeName() << ": ";
+			}
+			_stream << errorMessage(*error) << endl;
+		}
+}
+
+int SyntaxTest::offsetToLineNumber(int _location) const
+{
+	// parseAnalyseAndReturnError(...) prepends a version pragma
+	_location -= strlen("pragma solidity >=0.0;\n");
+	if (_location < 0 || static_cast<size_t>(_location) >= m_source.size())
+		return -1;
+	else
+		return 1 + std::count(m_source.begin(), m_source.begin() + _location, '\n');
 }
 
 bool SyntaxTest::matchesExpectations(ErrorList const& _errorList) const
@@ -196,7 +232,7 @@ int SyntaxTest::registerTests(
 			[fullpath]
 			{
 				std::stringstream errorStream;
-				if (!SyntaxTest(fullpath.string()).run(errorStream, ""))
+				if (!SyntaxTest(fullpath.string()).run(errorStream))
 					BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
 			},
 			_path.stem().string(),
diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h
index 4379c77b..441cc4f8 100644
--- a/test/libsolidity/SyntaxTest.h
+++ b/test/libsolidity/SyntaxTest.h
@@ -18,6 +18,7 @@
 #pragma once
 
 #include <test/libsolidity/AnalysisFramework.h>
+#include <test/libsolidity/FormattedScope.h>
 #include <libsolidity/interface/Exceptions.h>
 
 #include <boost/noncopyable.hpp>
@@ -47,13 +48,22 @@ class SyntaxTest: AnalysisFramework
 public:
 	SyntaxTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _indent);
+	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false);
+
+	std::vector<SyntaxTestExpectation> const& expectations() const { return m_expectations; }
+	std::string const& source() const { return m_source; }
+	ErrorList const& errorList() const { return m_errorList; }
+	ErrorList const& compilerErrors() const { return m_compiler.errors(); }
+
+	void printExpected(std::ostream& _stream, std::string const& _linePrefix, bool const _formatted = false) const;
 
-	void printExpected(std::ostream& _stream, std::string const& _indent) const;
 	void printErrorList(
 		std::ostream& _stream,
 		ErrorList const& _errors,
-		std::string const& _indent
+		std::string const& _linePrefix,
+		bool const _ignoreWarnings,
+		bool const _lineNumbers,
+		bool const _formatted = false
 	) const;
 
 	static int registerTests(
@@ -66,6 +76,7 @@ private:
 	static std::string errorMessage(Error const& _e);
 	static std::string parseSource(std::istream& _stream);
 	static std::vector<SyntaxTestExpectation> parseExpectations(std::istream& _stream);
+	int offsetToLineNumber(int _location) const;
 
 	std::string m_source;
 	std::vector<SyntaxTestExpectation> m_expectations;
diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt
index a693ebab..febb0c26 100644
--- a/test/tools/CMakeLists.txt
+++ b/test/tools/CMakeLists.txt
@@ -1,2 +1,5 @@
 add_executable(solfuzzer fuzzer.cpp)
 target_link_libraries(solfuzzer PRIVATE libsolc evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES})
+
+add_executable(isoltest isoltest.cpp ../Options.cpp ../libsolidity/SyntaxTest.cpp ../libsolidity/AnalysisFramework.cpp)
+target_link_libraries(isoltest PRIVATE libsolc solidity evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp
new file mode 100644
index 00000000..668481cf
--- /dev/null
+++ b/test/tools/isoltest.cpp
@@ -0,0 +1,346 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <libdevcore/CommonIO.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/libsolidity/SyntaxTest.h>
+
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/program_options.hpp>
+
+#include <cstdlib>
+#include <iostream>
+#include <fstream>
+#include <queue>
+
+using namespace dev;
+using namespace dev::solidity;
+using namespace dev::solidity::test;
+using namespace dev::solidity::test::formatting;
+using namespace std;
+namespace po = boost::program_options;
+namespace fs = boost::filesystem;
+
+struct SyntaxTestStats
+{
+	int successCount;
+	int runCount;
+	operator bool() const { return successCount == runCount; }
+};
+
+class SyntaxTestTool
+{
+public:
+	SyntaxTestTool(string const& _name, fs::path const& _path, bool _formatted):
+		m_formatted(_formatted), m_name(_name), m_path(_path)
+	{}
+
+	enum class Result
+	{
+		Success,
+		Failure,
+		ParserError,
+		InputOutputError
+	};
+
+	Result process();
+
+	static SyntaxTestStats processPath(
+		fs::path const& _basepath,
+		fs::path const& _path,
+		bool const _formatted
+	);
+
+	static string editor;
+private:
+	enum class Request
+	{
+		Skip,
+		Rerun,
+		Quit
+	};
+
+	Request handleResponse(bool const _parserError);
+
+	void printContract() const;
+
+	bool const m_formatted;
+	string const m_name;
+	fs::path const m_path;
+	unique_ptr<SyntaxTest> m_test;
+};
+
+string SyntaxTestTool::editor;
+
+void SyntaxTestTool::printContract() const
+{
+	stringstream stream(m_test->source());
+	string line;
+	while (getline(stream, line))
+		cout << "    " << line << endl;
+	cout << endl;
+}
+
+SyntaxTestTool::Result SyntaxTestTool::process()
+{
+	bool success;
+	bool parserError = false;
+	std::stringstream outputMessages;
+
+	(FormattedScope(cout, m_formatted, {BOLD}) << m_name << ": ").flush();
+
+	try
+	{
+		m_test = unique_ptr<SyntaxTest>(new SyntaxTest(m_path.string()));
+	}
+	catch (std::exception const& _e)
+	{
+		FormattedScope(cout, m_formatted, {BOLD, RED}) << "cannot read test: " << _e.what() << endl;
+		return Result::InputOutputError;
+	}
+
+	try
+	{
+		success = m_test->run(outputMessages, "  ", m_formatted);
+	}
+	catch (...)
+	{
+		success = false;
+		parserError = true;
+	}
+
+	if (success)
+	{
+		FormattedScope(cout, m_formatted, {BOLD, GREEN}) << "OK" << endl;
+		return Result::Success;
+	}
+	else
+	{
+		FormattedScope(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl;
+
+		FormattedScope(cout, m_formatted, {BOLD, CYAN}) << "  Contract:" << endl;
+		printContract();
+
+		if (parserError)
+		{
+			cout << "  ";
+			FormattedScope(cout, m_formatted, {INVERSE, RED}) << "Parsing failed:" << endl;
+			m_test->printErrorList(cout, m_test->compilerErrors(), "    ", true, true, m_formatted);
+			cout << endl;
+			return Result::ParserError;
+		}
+		else
+		{
+			cout << outputMessages.str() << endl;
+			return Result::Failure;
+		}
+	}
+}
+
+SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _parserError)
+{
+	if (_parserError)
+		cout << "(e)dit/(s)kip/(q)uit? ";
+	else
+		cout << "(e)dit/(u)pdate expectations/(s)kip/(q)uit? ";
+	cout.flush();
+
+	while (true)
+	{
+		switch(readStandardInputChar())
+		{
+		case 's':
+			cout << endl;
+			return Request::Skip;
+		case 'u':
+			if (_parserError)
+				break;
+			else
+			{
+				cout << endl;
+				ofstream file(m_path.string(), ios::trunc);
+				file << m_test->source();
+				file << "// ----" << endl;
+				if (!m_test->errorList().empty())
+					m_test->printErrorList(file, m_test->errorList(), "// ", false, false, false);
+				return Request::Rerun;
+			}
+		case 'e':
+			cout << endl << endl;
+			if (system((editor + " \"" + m_path.string() + "\"").c_str()))
+				cerr << "Error running editor command." << endl << endl;
+			return Request::Rerun;
+		case 'q':
+			cout << endl;
+			return Request::Quit;
+		default:
+			break;
+		}
+	}
+}
+
+
+SyntaxTestStats SyntaxTestTool::processPath(
+	fs::path const& _basepath,
+	fs::path const& _path,
+	bool const _formatted
+)
+{
+	std::queue<fs::path> paths;
+	paths.push(_path);
+	int successCount = 0;
+	int runCount = 0;
+
+	while (!paths.empty())
+	{
+		auto currentPath = paths.front();
+
+		fs::path fullpath = _basepath / currentPath;
+		if (fs::is_directory(fullpath))
+		{
+			paths.pop();
+			for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
+				fs::directory_iterator(fullpath),
+				fs::directory_iterator()
+			))
+				paths.push(currentPath / entry.path().filename());
+		}
+		else
+		{
+			SyntaxTestTool testTool(currentPath.string(), fullpath, _formatted);
+			++runCount;
+			auto result = testTool.process();
+
+			switch(result)
+			{
+			case Result::Failure:
+			case Result::ParserError:
+				switch(testTool.handleResponse(result == Result::ParserError))
+				{
+				case Request::Quit:
+					return { successCount, runCount };
+				case Request::Rerun:
+					cout << "Re-running test case..." << endl;
+					--runCount;
+					break;
+				case Request::Skip:
+					paths.pop();
+					break;
+				}
+				break;
+			case Result::Success:
+				paths.pop();
+				++successCount;
+				break;
+			default:
+				// non-recoverable error; continue with next test case
+				paths.pop();
+				break;
+			}
+		}
+	}
+
+	return { successCount, runCount };
+
+}
+
+int main(int argc, char *argv[])
+{
+	if (getenv("EDITOR"))
+		SyntaxTestTool::editor = getenv("EDITOR");
+
+	fs::path testPath;
+	bool formatted = true;
+	po::options_description options(
+		R"(isoltest, tool for interactively managing test contracts.
+Usage: isoltest [Options] --testpath path
+Interactively validates test contracts.
+
+Allowed options)",
+		po::options_description::m_default_line_length,
+		po::options_description::m_default_line_length - 23);
+	options.add_options()
+		("help", "Show this help screen.")
+		("testpath", po::value<fs::path>(&testPath), "path to test files")
+		("no-color", "don't use colors")
+		("editor", po::value<string>(&SyntaxTestTool::editor), "editor for opening contracts");
+
+	po::variables_map arguments;
+	try
+	{
+		po::command_line_parser cmdLineParser(argc, argv);
+		cmdLineParser.options(options);
+		po::store(cmdLineParser.run(), arguments);
+
+		if (arguments.count("help"))
+		{
+			cout << options << endl;
+			return 0;
+		}
+
+		if (arguments.count("no-color"))
+			formatted = false;
+
+		po::notify(arguments);
+	}
+	catch (po::error const& _exception)
+	{
+		cerr << _exception.what() << endl;
+		return 1;
+	}
+
+	if (testPath.empty())
+	{
+		auto const searchPath =
+		{
+			fs::current_path() / ".." / ".." / ".." / "test",
+			fs::current_path() / ".." / ".." / "test",
+			fs::current_path() / ".." / "test",
+			fs::current_path() / "test",
+			fs::current_path()
+		};
+		for (auto const& basePath : searchPath)
+		{
+			fs::path syntaxTestPath = basePath / "libsolidity" / "syntaxTests";
+			if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath))
+			{
+				testPath = basePath;
+				break;
+			}
+		}
+	}
+
+	fs::path syntaxTestPath = testPath / "libsolidity" / "syntaxTests";
+
+	if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath))
+	{
+		auto stats = SyntaxTestTool::processPath(testPath / "libsolidity", "syntaxTests", formatted);
+
+		cout << endl << "Summary: ";
+		FormattedScope(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
+			stats.successCount << "/" << stats.runCount;
+		cout << " tests successful." << endl;
+
+		return stats ? 0 : 1;
+	}
+	else
+	{
+		cerr << "Test path not found. Use the --testpath argument." << endl;
+		return 1;
+	}
+}