Merge pull request #3479 from ethereum/multiError

Support searching inside multiple errors.

6 files changed, 184 insertions, 70 deletions

diff --git a/test/libsolidity/AnalysisFramework.cpp b/test/libsolidity/AnalysisFramework.cpp
index ea9703ea..a27e3222 100644
--- a/test/libsolidity/AnalysisFramework.cpp
+++ b/test/libsolidity/AnalysisFramework.cpp
@@ -36,7 +36,7 @@ using namespace dev;
 using namespace dev::solidity;
 using namespace dev::solidity::test;
 
-pair<SourceUnit const*, shared_ptr<Error const>>
+pair<SourceUnit const*, ErrorList>
 AnalysisFramework::parseAnalyseAndReturnError(
 	string const& _source,
 	bool _reportWarnings,
@@ -53,7 +53,7 @@ AnalysisFramework::parseAnalyseAndReturnError(
 
 	m_compiler.analyze();
 
-	std::shared_ptr<Error const> firstError;
+	ErrorList errors;
 	for (auto const& currentError: m_compiler.errors())
 	{
 		solAssert(currentError->comment(), "");
@@ -72,16 +72,15 @@ AnalysisFramework::parseAnalyseAndReturnError(
 
 		if (_reportWarnings || (currentError->type() != Error::Type::Warning))
 		{
-			if (firstError && !_allowMultipleErrors)
+			if (!_allowMultipleErrors && !errors.empty())
 			{
 				BOOST_FAIL("Multiple errors found: " + formatErrors());
 			}
-			if (!firstError)
-				firstError = currentError;
+			errors.emplace_back(std::move(currentError));
 		}
 	}
 
-	return make_pair(&m_compiler.ast(""), firstError);
+	return make_pair(&m_compiler.ast(""), errors);
 }
 
 SourceUnit const* AnalysisFramework::parseAndAnalyse(string const& _source)
@@ -89,23 +88,23 @@ SourceUnit const* AnalysisFramework::parseAndAnalyse(string const& _source)
 	auto sourceAndError = parseAnalyseAndReturnError(_source);
 	BOOST_REQUIRE(!!sourceAndError.first);
 	string message;
-	if (sourceAndError.second)
-		message = "Unexpected error: " + formatError(*sourceAndError.second);
-	BOOST_REQUIRE_MESSAGE(!sourceAndError.second, message);
+	if (!sourceAndError.second.empty())
+		message = "Unexpected error: " + formatErrors();
+	BOOST_REQUIRE_MESSAGE(sourceAndError.second.empty(), message);
 	return sourceAndError.first;
 }
 
 bool AnalysisFramework::success(string const& _source)
 {
-	return !parseAnalyseAndReturnError(_source).second;
+	return parseAnalyseAndReturnError(_source).second.empty();
 }
 
-Error AnalysisFramework::expectError(std::string const& _source, bool _warning, bool _allowMultiple)
+ErrorList AnalysisFramework::expectError(std::string const& _source, bool _warning, bool _allowMultiple)
 {
-	auto sourceAndError = parseAnalyseAndReturnError(_source, _warning, true, _allowMultiple);
-	BOOST_REQUIRE(!!sourceAndError.second);
-	BOOST_REQUIRE_MESSAGE(!!sourceAndError.first, "Expected error, but no error happened.");
-	return *sourceAndError.second;
+	auto sourceAndErrors = parseAnalyseAndReturnError(_source, _warning, true, _allowMultiple);
+	BOOST_REQUIRE(!sourceAndErrors.second.empty());
+	BOOST_REQUIRE_MESSAGE(!!sourceAndErrors.first, "Expected error, but no error happened.");
+	return sourceAndErrors.second;
 }
 
 string AnalysisFramework::formatErrors()
diff --git a/test/libsolidity/AnalysisFramework.h b/test/libsolidity/AnalysisFramework.h
index a566ba1d..6ecf4a5a 100644
--- a/test/libsolidity/AnalysisFramework.h
+++ b/test/libsolidity/AnalysisFramework.h
@@ -45,7 +45,7 @@ class AnalysisFramework
 {
 
 protected:
-	virtual std::pair<SourceUnit const*, std::shared_ptr<Error const>>
+	virtual std::pair<SourceUnit const*, ErrorList>
 	parseAnalyseAndReturnError(
 		std::string const& _source,
 		bool _reportWarnings = false,
@@ -55,7 +55,7 @@ protected:
 
 	SourceUnit const* parseAndAnalyse(std::string const& _source);
 	bool success(std::string const& _source);
-	Error expectError(std::string const& _source, bool _warning = false, bool _allowMultiple = false);
+	ErrorList expectError(std::string const& _source, bool _warning = false, bool _allowMultiple = false);
 
 	std::string formatErrors();
 	std::string formatError(Error const& _error);
@@ -70,34 +70,51 @@ protected:
 	dev::solidity::CompilerStack m_compiler;
 };
 
+// Asserts that the compilation down to typechecking
+// emits multiple errors of different types and messages, provided in the second argument.
+#define CHECK_ALLOW_MULTI(text, expectations) \
+do \
+{ \
+	ErrorList errors = expectError((text), true, true); \
+	auto message = searchErrors(errors, (expectations)); \
+	BOOST_CHECK_MESSAGE(message.empty(), message); \
+} while(0)
 
-#define CHECK_ERROR_OR_WARNING(text, typ, substring, warning, allowMulti) \
+#define CHECK_ERROR_OR_WARNING(text, typ, substrings, warning, allowMulti) \
 do \
 { \
-	Error err = expectError((text), (warning), (allowMulti)); \
-	BOOST_CHECK(err.type() == (Error::Type::typ)); \
-	BOOST_CHECK(searchErrorMessage(err, (substring))); \
+	ErrorList errors = expectError((text), (warning), (allowMulti)); \
+	std::vector<std::pair<Error::Type, std::string>> expectations; \
+	for (auto const& str: substrings) \
+		expectations.emplace_back((Error::Type::typ), str); \
+	auto message = searchErrors(errors, expectations); \
+	BOOST_CHECK_MESSAGE(message.empty(), message); \
 } while(0)
 
 // [checkError(text, type, substring)] asserts that the compilation down to typechecking
 // emits an error of type [type] and with a message containing [substring].
 #define CHECK_ERROR(text, type, substring) \
-CHECK_ERROR_OR_WARNING(text, type, substring, false, false)
+CHECK_ERROR_OR_WARNING(text, type, std::vector<std::string>{(substring)}, false, false)
 
 // [checkError(text, type, substring)] asserts that the compilation down to typechecking
-// emits an error of type [type] and with a message containing [substring].
-#define CHECK_ERROR_ALLOW_MULTI(text, type, substring) \
-CHECK_ERROR_OR_WARNING(text, type, substring, false, true)
+// emits multiple errors of the same type [type] and with a messages containing [substrings].
+// Because of the limitations of the preprocessor, you cannot use {{T1, "abc"}, {T2, "def"}} as arguments,
+// but have to replace them by (std::vector<std::pair<Error::Type, std::string>>{"abc", "def"})
+// (note the parentheses)
+#define CHECK_ERROR_ALLOW_MULTI(text, type, substrings) \
+CHECK_ERROR_OR_WARNING(text, type, substrings, false, true)
 
 // [checkWarning(text, substring)] asserts that the compilation down to typechecking
 // emits a warning and with a message containing [substring].
 #define CHECK_WARNING(text, substring) \
-CHECK_ERROR_OR_WARNING(text, Warning, substring, true, false)
+CHECK_ERROR_OR_WARNING(text, Warning, std::vector<std::string>{(substring)}, true, false)
 
 // [checkWarningAllowMulti(text, substring)] aserts that the compilation down to typechecking
 // emits a warning and with a message containing [substring].
-#define CHECK_WARNING_ALLOW_MULTI(text, substring) \
-CHECK_ERROR_OR_WARNING(text, Warning, substring, true, true)
+// Because of the limitations of the preprocessor, you cannot use {"abc", "def"} as arguments,
+// but have to replace them by (std::vector<std::string>{"abc", "def"}) (note the parentheses)
+#define CHECK_WARNING_ALLOW_MULTI(text, substrings) \
+CHECK_ERROR_OR_WARNING(text, Warning, substrings, true, true)
 
 // [checkSuccess(text)] asserts that the compilation down to typechecking succeeds.
 #define CHECK_SUCCESS(text) do { BOOST_CHECK(success((text))); } while(0)
@@ -107,9 +124,9 @@ do \
 { \
 	auto sourceAndError = parseAnalyseAndReturnError((text), true); \
 	std::string message; \
-	if (sourceAndError.second) \
-		message = formatError(*sourceAndError.second); \
-	BOOST_CHECK_MESSAGE(!sourceAndError.second, message); \
+	if (!sourceAndError.second.empty()) \
+		message = formatErrors();\
+	BOOST_CHECK_MESSAGE(sourceAndError.second.empty(), message); \
 } \
 while(0)
 
diff --git a/test/libsolidity/ErrorCheck.cpp b/test/libsolidity/ErrorCheck.cpp
index b1e94061..fba2c897 100644
--- a/test/libsolidity/ErrorCheck.cpp
+++ b/test/libsolidity/ErrorCheck.cpp
@@ -23,8 +23,19 @@
 #include <libdevcore/Exceptions.h>
 
 #include <string>
+#include <set>
 
 using namespace std;
+using namespace dev;
+using namespace dev::solidity;
+
+namespace
+{
+std::string errorMessage(Error const& _e)
+{
+	return _e.comment() ? *_e.comment() : "NONE";
+}
+}
 
 bool dev::solidity::searchErrorMessage(Error const& _err, std::string const& _substr)
 {
@@ -41,3 +52,31 @@ bool dev::solidity::searchErrorMessage(Error const& _err, std::string const& _su
 		cout << "Expected error message but found none." << endl;
 	return _substr.empty();
 }
+
+string dev::solidity::searchErrors(ErrorList const& _errors, vector<pair<Error::Type, string>> const& _expectations)
+{
+	auto expectations = _expectations;
+	for (auto const& error: _errors)
+	{
+		string msg = errorMessage(*error);
+		bool found = false;
+		for (auto it = expectations.begin(); it != expectations.end(); ++it)
+			if (msg.find(it->second) != string::npos && error->type() == it->first)
+			{
+				found = true;
+				expectations.erase(it);
+				break;
+			}
+		if (!found)
+			return "Unexpected error: " + error->typeName() + ": " + msg;
+	}
+	if (!expectations.empty())
+	{
+		string msg = "Expected error(s) not present:\n";
+		for (auto const& expectation: expectations)
+			msg += expectation.second + "\n";
+		return msg;
+	}
+
+	return "";
+}
diff --git a/test/libsolidity/ErrorCheck.h b/test/libsolidity/ErrorCheck.h
index a309a9d3..8ad81f85 100644
--- a/test/libsolidity/ErrorCheck.h
+++ b/test/libsolidity/ErrorCheck.h
@@ -23,10 +23,17 @@
 
 #include <libsolidity/interface/Exceptions.h>
 
+#include <vector>
+#include <tuple>
+
 namespace dev
 {
 namespace solidity
 {
 bool searchErrorMessage(Error const& _err, std::string const& _substr);
+/// Checks that all provided errors are of the given type and have a given substring in their
+/// description.
+/// If the expectations are not met, returns a nonempty description, otherwise an empty string.
+std::string searchErrors(ErrorList const& _errors, std::vector<std::pair<Error::Type, std::string>> const& _expectations);
 }
 }
diff --git a/test/libsolidity/SMTChecker.cpp b/test/libsolidity/SMTChecker.cpp
index 2a1609cc..8c955292 100644
--- a/test/libsolidity/SMTChecker.cpp
+++ b/test/libsolidity/SMTChecker.cpp
@@ -42,7 +42,7 @@ public:
 	}
 
 protected:
-	virtual std::pair<SourceUnit const*, std::shared_ptr<Error const>>
+	virtual std::pair<SourceUnit const*, ErrorList>
 	parseAnalyseAndReturnError(
 		std::string const& _source,
 		bool _reportWarnings = false,
@@ -102,8 +102,10 @@ BOOST_AUTO_TEST_CASE(warn_on_struct)
 			}
 		}
 	)";
-	/// Multiple warnings, should check for: Assertion checker does not yet implement this expression.
-	CHECK_WARNING_ALLOW_MULTI(text, "");
+	CHECK_WARNING_ALLOW_MULTI(text, (vector<string>{
+		"Assertion checker does not yet implement this expression.",
+		"Assertion checker does not yet support the type of this variable."
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(simple_assert)
diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp
index ee6a0633..b3a21671 100644
--- a/test/libsolidity/SolidityNameAndTypeResolution.cpp
+++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp
@@ -133,7 +133,7 @@ BOOST_AUTO_TEST_CASE(undeclared_name_is_not_fatal)
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, "Undeclared identifier.");
+	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, (vector<string>{"Undeclared identifier", "Undeclared identifier"}));
 }
 
 BOOST_AUTO_TEST_CASE(reference_to_later_declaration)
@@ -864,7 +864,7 @@ BOOST_AUTO_TEST_CASE(cyclic_inheritance)
 		contract A is B { }
 		contract B is A { }
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Definition of base has to precede definition of derived contract");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (vector<string>{"Definition of base has to precede definition of derived contract"}));
 }
 
 BOOST_AUTO_TEST_CASE(legal_override_direct)
@@ -1092,9 +1092,10 @@ BOOST_AUTO_TEST_CASE(modifier_overrides_function)
 		contract A { modifier mod(uint a) { _; } }
 		contract B is A { function mod(uint a) public { } }
 	)";
-	// Error: Identifier already declared.
-	// Error: Override changes modifier to function.
-	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, "Identifier already declared.");
+	CHECK_ALLOW_MULTI(text, (vector<pair<Error::Type, string>>{
+		{Error::Type::DeclarationError, "Identifier already declared"},
+		{Error::Type::TypeError, "Override changes modifier to function"}
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(function_overrides_modifier)
@@ -1103,9 +1104,10 @@ BOOST_AUTO_TEST_CASE(function_overrides_modifier)
 		contract A { function mod(uint a) public { } }
 		contract B is A { modifier mod(uint a) { _; } }
 	)";
-	// Error: Identifier already declared.
-	// Error: Override changes function to modifier.
-	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, "Identifier already declared.");
+	CHECK_ALLOW_MULTI(text, (vector<pair<Error::Type, string>>{
+		{Error::Type::DeclarationError, "Identifier already declared"},
+		{Error::Type::TypeError, "Override changes function to modifier"}
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(modifier_returns_value)
@@ -1342,7 +1344,10 @@ BOOST_AUTO_TEST_CASE(fallback_function_twice)
 			function() public { x = 3; }
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, "Function with same name and arguments defined twice.");
+	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, (vector<string>{
+		"Function with same name and arguments defined twice.",
+		"Only one fallback function is"
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(fallback_function_inheritance)
@@ -1677,7 +1682,11 @@ BOOST_AUTO_TEST_CASE(constant_input_parameter)
 			function f(uint[] constant a) public { }
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Illegal use of \"constant\" specifier.");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (vector<string>{
+		"Illegal use of \"constant\" specifier",
+		"Constants of non-value type not yet implemented",
+		"Uninitialized \"constant\" variable"
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(empty_name_return_parameter)
@@ -2626,7 +2635,10 @@ BOOST_AUTO_TEST_CASE(equal_overload)
 			function test(uint a) external {}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(sourceCode, DeclarationError, "Function with same name and arguments defined twice.");
+	CHECK_ALLOW_MULTI(sourceCode, (vector<pair<Error::Type, string>>{
+		{Error::Type::DeclarationError, "Function with same name and arguments defined twice."},
+		{Error::Type::TypeError, "Overriding function visibility differs"}
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(uninitialized_var)
@@ -3128,7 +3140,10 @@ BOOST_AUTO_TEST_CASE(library_constructor)
 			function Lib();
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Constructor cannot be defined in libraries.");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (vector<std::string>{
+		"Constructor cannot be defined in libraries.",
+		"Constructor must be implemented if declared."
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(valid_library)
@@ -3855,7 +3870,10 @@ BOOST_AUTO_TEST_CASE(left_value_in_conditional_expression_not_supported_yet)
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Conditional expression as left value is not supported yet.");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{
+		"Conditional expression as left value is not supported yet.",
+		"Expression has to be an lvalue"
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(conditional_expression_with_different_struct)
@@ -4074,7 +4092,11 @@ BOOST_AUTO_TEST_CASE(varM_disqualified_as_keyword)
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, "Identifier not found or not unique.");
+	CHECK_ERROR_ALLOW_MULTI(text, DeclarationError, (std::vector<std::string>{
+		"Identifier not found or not unique.",
+		"Identifier not found or not unique.",
+		"Identifier not found or not unique."
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(modifier_is_not_a_valid_typename)
@@ -4103,7 +4125,7 @@ BOOST_AUTO_TEST_CASE(modifier_is_not_a_valid_typename_is_not_fatal)
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Name has to refer to a struct, enum or contract.");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{"Name has to refer to a struct, enum or contract."}));
 }
 
 BOOST_AUTO_TEST_CASE(function_is_not_a_valid_typename)
@@ -4842,7 +4864,10 @@ BOOST_AUTO_TEST_CASE(unused_return_value_callcode)
 			}
 		}
 	)";
-	CHECK_WARNING_ALLOW_MULTI(text, "Return value of low-level calls not used");
+	CHECK_WARNING_ALLOW_MULTI(text, (std::vector<std::string>{
+		"Return value of low-level calls not used",
+		"\"callcode\" has been deprecated"
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(unused_return_value_delegatecall)
@@ -5022,9 +5047,9 @@ BOOST_AUTO_TEST_CASE(warn_nonpresent_pragma)
 {
 	char const* text = "contract C {}";
 	auto sourceAndError = parseAnalyseAndReturnError(text, true, false);
-	BOOST_REQUIRE(!!sourceAndError.second);
+	BOOST_REQUIRE(!sourceAndError.second.empty());
 	BOOST_REQUIRE(!!sourceAndError.first);
-	BOOST_CHECK(searchErrorMessage(*sourceAndError.second, "Source file does not specify required compiler version!"));
+	BOOST_CHECK(searchErrorMessage(*sourceAndError.second.front(), "Source file does not specify required compiler version!"));
 }
 
 BOOST_AUTO_TEST_CASE(unsatisfied_version)
@@ -5033,10 +5058,10 @@ BOOST_AUTO_TEST_CASE(unsatisfied_version)
 		pragma solidity ^99.99.0;
 	)";
 	auto sourceAndError = parseAnalyseAndReturnError(text, false, false, false);
-	BOOST_REQUIRE(!!sourceAndError.second);
+	BOOST_REQUIRE(!sourceAndError.second.empty());
 	BOOST_REQUIRE(!!sourceAndError.first);
-	BOOST_CHECK(sourceAndError.second->type() == Error::Type::SyntaxError);
-	BOOST_CHECK(searchErrorMessage(*sourceAndError.second, "Source file requires different compiler version"));
+	BOOST_CHECK(sourceAndError.second.front()->type() == Error::Type::SyntaxError);
+	BOOST_CHECK(searchErrorMessage(*sourceAndError.second.front(), "Source file requires different compiler version"));
 }
 
 BOOST_AUTO_TEST_CASE(invalid_constructor_statemutability)
@@ -5182,7 +5207,7 @@ BOOST_AUTO_TEST_CASE(payable_internal_function_type_is_not_fatal)
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Only external function types can be payable.");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{"Only external function types can be payable."}));
 }
 
 BOOST_AUTO_TEST_CASE(call_value_on_non_payable_function_type)
@@ -5876,7 +5901,10 @@ BOOST_AUTO_TEST_CASE(invalid_address_length_long)
 			}
 		}
 	)";
-	CHECK_WARNING_ALLOW_MULTI(text, "This looks like an address but has an invalid checksum.");
+	CHECK_ALLOW_MULTI(text, (std::vector<std::pair<Error::Type, std::string>>{
+		{Error::Type::Warning, "This looks like an address but has an invalid checksum."},
+		{Error::Type::TypeError, "not implicitly convertible"}
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(address_test_for_bug_in_implementation)
@@ -5949,7 +5977,11 @@ BOOST_AUTO_TEST_CASE(cyclic_dependency_for_constants)
 			uint constant d = 2 + a;
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "a has a cyclic dependency via c");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{
+		"a has a cyclic dependency via c",
+		"c has a cyclic dependency via d",
+		"d has a cyclic dependency via a"
+	}));
 	text = R"(
 		contract C {
 			uint constant a = b * c;
@@ -5977,7 +6009,10 @@ BOOST_AUTO_TEST_CASE(interface_constructor)
 			function I();
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Constructor cannot be defined in interfaces");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{
+		"Constructor cannot be defined in interfaces",
+		"Constructor must be implemented if declared.",
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(interface_functions)
@@ -6660,15 +6695,23 @@ BOOST_AUTO_TEST_CASE(returndatacopy_as_variable)
 	char const* text = R"(
 		contract c { function f() public { uint returndatasize; assembly { returndatasize }}}
 	)";
-	CHECK_WARNING_ALLOW_MULTI(text, "Variable is shadowed in inline assembly by an instruction of the same name");
+	CHECK_ALLOW_MULTI(text, (std::vector<std::pair<Error::Type, std::string>>{
+		{Error::Type::Warning, "Variable is shadowed in inline assembly by an instruction of the same name"},
+		{Error::Type::DeclarationError, "Unbalanced stack"},
+		{Error::Type::Warning, "only available after the Metropolis"}
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(create2_as_variable)
 {
 	char const* text = R"(
-		contract c { function f() public { uint create2; assembly { create2(0, 0, 0, 0) }}}
+		contract c { function f() public { uint create2; assembly { create2(0, 0, 0, 0) } }}
 	)";
-	CHECK_WARNING_ALLOW_MULTI(text, "Variable is shadowed in inline assembly by an instruction of the same name");
+	CHECK_ALLOW_MULTI(text, (std::vector<std::pair<Error::Type, std::string>>{
+		{Error::Type::Warning, "Variable is shadowed in inline assembly by an instruction of the same name"},
+		{Error::Type::Warning, "only available after the Metropolis"},
+		{Error::Type::DeclarationError, "Unbalanced stack"}
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(warn_unspecified_storage)
@@ -6728,7 +6771,7 @@ BOOST_AUTO_TEST_CASE(storage_location_non_array_or_struct_disallowed_is_not_fata
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "Storage location can only be given for array or struct types.");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{"Storage location can only be given for array or struct types."}));
 }
 
 BOOST_AUTO_TEST_CASE(implicit_conversion_disallowed)
@@ -6926,7 +6969,11 @@ BOOST_AUTO_TEST_CASE(do_not_crash_on_not_lvalue)
 			}
 		}
 	)";
-	CHECK_ERROR_ALLOW_MULTI(text, TypeError, "is not callable");
+	CHECK_ERROR_ALLOW_MULTI(text, TypeError, (std::vector<std::string>{
+		"is not callable",
+		"Expression has to be an lvalue",
+		"Type int_const 2 is not implicitly"
+	}));
 }
 
 BOOST_AUTO_TEST_CASE(builtin_reject_gas)
@@ -7129,11 +7176,11 @@ BOOST_AUTO_TEST_CASE(experimental_pragma)
 		pragma experimental __test;
 	)";
 	CHECK_WARNING(text, "Experimental features are turned on. Do not use experimental features on live deployments.");
-//	text = R"(
-//		pragma experimental __test;
-//		pragma experimental __test;
-//	)";
-//	CHECK_ERROR_ALLOW_MULTI(text, SyntaxError, "Duplicate experimental feature name.");
+	text = R"(
+		pragma experimental __test;
+		pragma experimental __test;
+	)";
+	CHECK_ERROR_ALLOW_MULTI(text, SyntaxError, (std::vector<std::string>{"Duplicate experimental feature name."}));
 }
 
 BOOST_AUTO_TEST_CASE(reject_interface_creation)
@@ -7196,7 +7243,10 @@ BOOST_AUTO_TEST_CASE(tight_packing_literals)
 			}
 		}
 	)";
-//	CHECK_WARNING(text, "The type of \"int_const 1\" was inferred as uint8.");
+	CHECK_WARNING_ALLOW_MULTI(text, (std::vector<std::string>{
+		"The type of \"int_const 1\" was inferred as uint8.",
+		"\"sha3\" has been deprecated in favour of \"keccak256\""
+	}));
 	text = R"(
 		contract C {
 			function f() pure public returns (bytes32) {