Infrastructure for extracting syntax tests in separate test files.

9 files changed, 367 insertions, 52 deletions

diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp
index 997b610e..1f76c01b 100644
--- a/test/libsolidity/SolidityNameAndTypeResolution.cpp
+++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp
@@ -43,27 +43,6 @@ namespace test
 
 BOOST_FIXTURE_TEST_SUITE(SolidityNameAndTypeResolution, AnalysisFramework)
 
-BOOST_AUTO_TEST_CASE(smoke_test)
-{
-	char const* text = R"(
-		contract test {
-			uint256 stateVariable1;
-			function fun(uint256 arg1) public { uint256 y; y = arg1; }
-		}
-	)";
-	CHECK_SUCCESS(text);
-}
-
-BOOST_AUTO_TEST_CASE(double_stateVariable_declaration)
-{
-	char const* text = R"(
-		contract test {
-			uint256 variable;
-			uint128 variable;
-		}
-	)";
-	CHECK_ERROR(text, DeclarationError, "Identifier already declared.");
-}
 
 BOOST_AUTO_TEST_CASE(double_function_declaration)
 {
@@ -76,37 +55,6 @@ BOOST_AUTO_TEST_CASE(double_function_declaration)
 	CHECK_ERROR(text, DeclarationError, "Function with same name and arguments defined twice.");
 }
 
-BOOST_AUTO_TEST_CASE(double_variable_declaration)
-{
-	string text = R"(
-		contract test {
-			function f() pure public {
-				uint256 x;
-				if (true) { uint256 x; }
-			}
-		}
-	)";
-	CHECK_ERROR(text, DeclarationError, "Identifier already declared");
-}
-
-BOOST_AUTO_TEST_CASE(double_variable_declaration_050)
-{
-	string text = R"(
-		pragma experimental "v0.5.0";
-		contract test {
-			function f() pure public {
-				uint256 x;
-				if (true) { uint256 x; }
-			}
-		}
-	)";
-	CHECK_WARNING_ALLOW_MULTI(text, (vector<string>{
-		"This declaration shadows an existing declaration.",
-		"Unused local variable",
-		"Unused local variable"
-	}));
-}
-
 BOOST_AUTO_TEST_CASE(double_variable_declaration_disjoint_scope)
 {
 	string text = R"(
diff --git a/test/libsolidity/SyntaxTestParser.cpp b/test/libsolidity/SyntaxTestParser.cpp
new file mode 100644
index 00000000..c37caca5
--- /dev/null
+++ b/test/libsolidity/SyntaxTestParser.cpp
@@ -0,0 +1,97 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SyntaxTestParser.h>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/throw_exception.hpp>
+#include <cctype>
+#include <fstream>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+
+template<typename IteratorType>
+void skipWhitespace(IteratorType& it, IteratorType end)
+{
+	while (it != end && isspace(*it))
+		++it;
+}
+
+template<typename IteratorType>
+void skipSlashes(IteratorType& it, IteratorType end)
+{
+	while (it != end && *it == '/')
+		++it;
+}
+
+std::string SyntaxTestParser::parseSource(std::istream& _stream)
+{
+	std::string source;
+	string line;
+	string const delimiter("// ----");
+	while (getline(_stream, line))
+		if (boost::algorithm::starts_with(line, delimiter))
+			break;
+		else
+			source += line + "\n";
+	return source;
+}
+
+std::vector<SyntaxTestExpectation> SyntaxTestParser::parseExpectations(std::istream& _stream)
+{
+	std::vector<SyntaxTestExpectation> expectations;
+	std::string line;
+	while (getline(_stream, line))
+	{
+		auto it = line.begin();
+
+		skipSlashes(it, line.end());
+		skipWhitespace(it, line.end());
+
+		if (it == line.end()) continue;
+
+		auto typeBegin = it;
+		while (it != line.end() && *it != ':')
+			++it;
+		string errorType(typeBegin, it);
+
+		// skip colon
+		if (it != line.end()) it++;
+
+		skipWhitespace(it, line.end());
+
+		string errorMessage(it, line.end());
+		expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)});
+	}
+	return expectations;
+}
+
+SyntaxTest SyntaxTestParser::parse(string const& _filename)
+{
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\""));
+	file.exceptions(ios::badbit);
+
+	SyntaxTest result;
+	result.source = parseSource(file);
+	result.expectations = parseExpectations(file);
+	return result;
+}
diff --git a/test/libsolidity/SyntaxTestParser.h b/test/libsolidity/SyntaxTestParser.h
new file mode 100644
index 00000000..9e295a0b
--- /dev/null
+++ b/test/libsolidity/SyntaxTestParser.h
@@ -0,0 +1,57 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+struct SyntaxTestExpectation
+{
+	std::string type;
+	std::string message;
+};
+
+struct SyntaxTest
+{
+	std::string source;
+	std::vector<SyntaxTestExpectation> expectations;
+};
+
+class SyntaxTestParser
+{
+public:
+	SyntaxTestParser() = default;
+
+	SyntaxTest parse(std::string const& _filename);
+private:
+	std::string parseSource(std::istream& _stream);
+	std::vector<SyntaxTestExpectation> parseExpectations(std::istream& _stream);
+};
+
+}
+}
+}
diff --git a/test/libsolidity/SyntaxTester.cpp b/test/libsolidity/SyntaxTester.cpp
new file mode 100644
index 00000000..4e545760
--- /dev/null
+++ b/test/libsolidity/SyntaxTester.cpp
@@ -0,0 +1,134 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SyntaxTester.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/TestHelper.h>
+#include <boost/algorithm/string/replace.hpp>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+using namespace boost::unit_test;
+namespace fs = boost::filesystem;
+
+void SyntaxTester::runTest(SyntaxTest const& _test)
+{
+	vector<string> unexpectedErrors;
+	auto expectations = _test.expectations;
+	auto errorList = parseAnalyseAndReturnError(_test.source, true, true, true).second;
+
+	bool errorsMatch = true;
+
+	if (errorList.size() != expectations.size())
+		errorsMatch = false;
+	else
+	{
+		for (size_t i = 0; i < errorList.size(); i++)
+		{
+			if (
+				!(errorList[i]->typeName() == expectations[i].type) ||
+				!(errorMessage(*errorList[i]) == expectations[i].message)
+			)
+			{
+				errorsMatch = false;
+				break;
+			}
+		}
+	}
+
+	if (!errorsMatch)
+	{
+		string msg = "Test expectation mismatch.\nExpected result:\n";
+		if (expectations.empty())
+			msg += "\tSuccess\n";
+		else
+			for (auto const& expectation: expectations)
+				msg += "\t" + expectation.type + ": " + expectation.message + "\n";
+		msg += "Obtained result:\n";
+		if (errorList.empty())
+			msg += "\tSuccess\n";
+		else
+			for (auto const& error: errorList)
+				msg += "\t" + error->typeName() + ": " + errorMessage(*error) + "\n";
+		BOOST_ERROR(msg);
+	}
+}
+
+std::string SyntaxTester::errorMessage(Error const& _e)
+{
+	if (_e.comment())
+		return boost::replace_all_copy(*_e.comment(), "\n", "\\n");
+	else
+		return "NONE";
+}
+
+int SyntaxTester::registerTests(
+	test_suite& _suite,
+	fs::path const& _basepath,
+	fs::path const& _path
+)
+{
+
+	int numTestsAdded = 0;
+	fs::path fullpath = _basepath / _path;
+	if (fs::is_directory(fullpath))
+	{
+		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
+		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
+			fs::directory_iterator(fullpath),
+			fs::directory_iterator()
+		))
+			numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
+		_suite.add(sub_suite);
+	}
+	else
+	{
+		_suite.add(make_test_case(
+			[fullpath] { SyntaxTester().runTest(SyntaxTestParser().parse(fullpath.string())); },
+			_path.stem().string(),
+			_path.string(),
+			0
+		));
+		numTestsAdded = 1;
+	}
+	return numTestsAdded;
+}
+
+void SyntaxTester::registerTests()
+{
+	if(dev::test::Options::get().testPath.empty())
+		throw runtime_error(
+			"No path to the test files was specified. "
+			"Use the --testpath command line option or "
+			"the ETH_TEST_PATH environment variable."
+		);
+	auto testPath = fs::path(dev::test::Options::get().testPath);
+
+	if (fs::exists(testPath) && fs::is_directory(testPath))
+	{
+		int numTestsAdded = registerTests(
+			framework::master_test_suite(),
+			testPath / "libsolidity",
+			"syntaxTests"
+		);
+		solAssert(numTestsAdded > 0, "no syntax tests found in libsolidity/syntaxTests");
+	}
+	else
+		solAssert(false, "libsolidity/syntaxTests directory not found");
+}
diff --git a/test/libsolidity/SyntaxTester.h b/test/libsolidity/SyntaxTester.h
new file mode 100644
index 00000000..d61668aa
--- /dev/null
+++ b/test/libsolidity/SyntaxTester.h
@@ -0,0 +1,48 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/libsolidity/SyntaxTestParser.h>
+#include <boost/filesystem.hpp>
+#include <boost/test/unit_test.hpp>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+class SyntaxTester: public AnalysisFramework
+{
+public:
+	static void registerTests();
+private:
+	static int registerTests(
+		boost::unit_test::test_suite& _suite,
+		boost::filesystem::path const& _basepath,
+		boost::filesystem::path const& _path
+	);
+	static std::string errorMessage(Error const& _e);
+	void runTest(SyntaxTest const& _test);
+};
+
+}
+}
+}
diff --git a/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol
new file mode 100644
index 00000000..c5507b64
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol
@@ -0,0 +1,6 @@
+contract test {
+	uint256 variable;
+	uint128 variable;
+}
+// ----
+// DeclarationError: Identifier already declared.
diff --git a/test/libsolidity/syntaxTests/double_variable_declaration.sol b/test/libsolidity/syntaxTests/double_variable_declaration.sol
new file mode 100644
index 00000000..3349cfec
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_variable_declaration.sol
@@ -0,0 +1,8 @@
+contract test {
+	function f() pure public {
+		uint256 x;
+		if (true) { uint256 x; }
+	}
+}
+// ----
+// DeclarationError: Identifier already declared.
diff --git a/test/libsolidity/syntaxTests/double_variable_declaration_050.sol b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol
new file mode 100644
index 00000000..9c2d40d5
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol
@@ -0,0 +1,11 @@
+pragma experimental "v0.5.0";
+contract test {
+	function f() pure public {
+		uint256 x;
+		if (true) { uint256 x; }
+	}
+}
+// ----
+// Warning: This declaration shadows an existing declaration.
+// Warning: Unused local variable.
+// Warning: Unused local variable.
diff --git a/test/libsolidity/syntaxTests/smoke_test.sol b/test/libsolidity/syntaxTests/smoke_test.sol
new file mode 100644
index 00000000..2d48098a
--- /dev/null
+++ b/test/libsolidity/syntaxTests/smoke_test.sol
@@ -0,0 +1,6 @@
+contract test {
+	uint256 stateVariable1;
+	function fun(uint256 arg1) public { uint256 y; y = arg1; }
+}
+// ----
+// Warning: Function state mutability can be restricted to pure