Merge pull request #5571 from ethereum/develop

Version 0.5.1

18 files changed, 3139 insertions, 0 deletions

diff --git a/liblangutil/CMakeLists.txt b/liblangutil/CMakeLists.txt
new file mode 100644
index 00000000..dfcccfce
--- /dev/null
+++ b/liblangutil/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Solidity Commons Library (Solidity related sharing bits between libsolidity and libyul)
+set(sources
+	CharStream.cpp
+	ErrorReporter.cpp
+	Exceptions.cpp
+	ParserBase.cpp
+	Scanner.cpp
+	SourceReferenceFormatter.cpp
+	Token.cpp
+)
+
+add_library(langutil ${sources})
+target_link_libraries(langutil PUBLIC devcore)
diff --git a/liblangutil/CharStream.cpp b/liblangutil/CharStream.cpp
new file mode 100644
index 00000000..aee7cb3e
--- /dev/null
+++ b/liblangutil/CharStream.cpp
@@ -0,0 +1,108 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+	This file is derived from the file "scanner.cc", which was part of the
+	V8 project. The original copyright header follows:
+
+	Copyright 2006-2012, the V8 project authors. All rights reserved.
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are
+	met:
+
+	* Redistributions of source code must retain the above copyright
+	  notice, this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above
+	  copyright notice, this list of conditions and the following
+	  disclaimer in the documentation and/or other materials provided
+	  with the distribution.
+	* Neither the name of Google Inc. nor the names of its
+	  contributors may be used to endorse or promote products derived
+	  from this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity scanner.
+ */
+
+#include <liblangutil/CharStream.h>
+#include <liblangutil/Exceptions.h>
+
+using namespace std;
+using namespace langutil;
+
+char CharStream::advanceAndGet(size_t _chars)
+{
+	if (isPastEndOfInput())
+		return 0;
+	m_position += _chars;
+	if (isPastEndOfInput())
+		return 0;
+	return m_source[m_position];
+}
+
+char CharStream::rollback(size_t _amount)
+{
+	solAssert(m_position >= _amount, "");
+	m_position -= _amount;
+	return get();
+}
+
+string CharStream::lineAtPosition(int _position) const
+{
+	// if _position points to \n, it returns the line before the \n
+	using size_type = string::size_type;
+	size_type searchStart = min<size_type>(m_source.size(), _position);
+	if (searchStart > 0)
+		searchStart--;
+	size_type lineStart = m_source.rfind('\n', searchStart);
+	if (lineStart == string::npos)
+		lineStart = 0;
+	else
+		lineStart++;
+	return m_source.substr(lineStart, min(m_source.find('\n', lineStart),
+										  m_source.size()) - lineStart);
+}
+
+tuple<int, int> CharStream::translatePositionToLineColumn(int _position) const
+{
+	using size_type = string::size_type;
+	size_type searchPosition = min<size_type>(m_source.size(), _position);
+	int lineNumber = count(m_source.begin(), m_source.begin() + searchPosition, '\n');
+	size_type lineStart;
+	if (searchPosition == 0)
+		lineStart = 0;
+	else
+	{
+		lineStart = m_source.rfind('\n', searchPosition - 1);
+		lineStart = lineStart == string::npos ? 0 : lineStart + 1;
+	}
+	return tuple<int, int>(lineNumber, searchPosition - lineStart);
+}
+
+
diff --git a/liblangutil/CharStream.h b/liblangutil/CharStream.h
new file mode 100644
index 00000000..f92beb30
--- /dev/null
+++ b/liblangutil/CharStream.h
@@ -0,0 +1,100 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+	This file is derived from the file "scanner.h", which was part of the
+	V8 project. The original copyright header follows:
+
+	Copyright 2006-2012, the V8 project authors. All rights reserved.
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are
+	met:
+
+	* Redistributions of source code must retain the above copyright
+	  notice, this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above
+	  copyright notice, this list of conditions and the following
+	  disclaimer in the documentation and/or other materials provided
+	  with the distribution.
+	* Neither the name of Google Inc. nor the names of its
+	  contributors may be used to endorse or promote products derived
+	  from this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity scanner.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <tuple>
+
+namespace langutil
+{
+
+/**
+ * Bidirectional stream of characters.
+ *
+ * This CharStream is used by lexical analyzers as the source.
+ */
+class CharStream
+{
+public:
+	CharStream(): m_position(0) {}
+	explicit CharStream(std::string const& _source, std::string const& name):
+		m_source(_source), m_name(name), m_position(0) {}
+
+	int position() const { return m_position; }
+	bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); }
+
+	char get(size_t _charsForward = 0) const { return m_source[m_position + _charsForward]; }
+	char advanceAndGet(size_t _chars = 1);
+	char rollback(size_t _amount);
+
+	void reset() { m_position = 0; }
+
+	std::string const& source() const noexcept { return m_source; }
+	std::string const& name() const noexcept { return m_name; }
+
+	///@{
+	///@name Error printing helper functions
+	/// Functions that help pretty-printing parse errors
+	/// Do only use in error cases, they are quite expensive.
+	std::string lineAtPosition(int _position) const;
+	std::tuple<int, int> translatePositionToLineColumn(int _position) const;
+	///@}
+
+private:
+	std::string m_source;
+	std::string m_name;
+	size_t m_position;
+};
+
+}
diff --git a/liblangutil/EVMVersion.h b/liblangutil/EVMVersion.h
new file mode 100644
index 00000000..657727ac
--- /dev/null
+++ b/liblangutil/EVMVersion.h
@@ -0,0 +1,94 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * EVM versioning.
+ */
+
+#pragma once
+
+#include <string>
+
+#include <boost/optional.hpp>
+#include <boost/operators.hpp>
+
+namespace dev
+{
+namespace solidity
+{
+
+/**
+ * A version specifier of the EVM we want to compile to.
+ * Defaults to the latest version.
+ */
+class EVMVersion:
+	boost::less_than_comparable<EVMVersion>,
+	boost::equality_comparable<EVMVersion>
+{
+public:
+	EVMVersion() {}
+
+	static EVMVersion homestead() { return {Version::Homestead}; }
+	static EVMVersion tangerineWhistle() { return {Version::TangerineWhistle}; }
+	static EVMVersion spuriousDragon() { return {Version::SpuriousDragon}; }
+	static EVMVersion byzantium() { return {Version::Byzantium}; }
+	static EVMVersion constantinople() { return {Version::Constantinople}; }
+
+	static boost::optional<EVMVersion> fromString(std::string const& _version)
+	{
+		for (auto const& v: {homestead(), tangerineWhistle(), spuriousDragon(), byzantium(), constantinople()})
+			if (_version == v.name())
+				return v;
+		return {};
+	}
+
+	bool operator==(EVMVersion const& _other) const { return m_version == _other.m_version; }
+	bool operator<(EVMVersion const& _other) const { return m_version < _other.m_version; }
+
+	std::string name() const
+	{
+		switch (m_version)
+		{
+		case Version::Homestead: return "homestead";
+		case Version::TangerineWhistle: return "tangerineWhistle";
+		case Version::SpuriousDragon: return "spuriousDragon";
+		case Version::Byzantium: return "byzantium";
+		case Version::Constantinople: return "constantinople";
+		}
+		return "INVALID";
+	}
+
+	/// Has the RETURNDATACOPY and RETURNDATASIZE opcodes.
+	bool supportsReturndata() const { return *this >= byzantium(); }
+	bool hasStaticCall() const { return *this >= byzantium(); }
+	bool hasBitwiseShifting() const { return *this >= constantinople(); }
+	bool hasCreate2() const { return *this >= constantinople(); }
+
+	/// Whether we have to retain the costs for the call opcode itself (false),
+	/// or whether we can just forward easily all remaining gas (true).
+	bool canOverchargeGasForCall() const { return *this >= tangerineWhistle(); }
+
+private:
+	enum class Version { Homestead, TangerineWhistle, SpuriousDragon, Byzantium, Constantinople };
+
+	EVMVersion(Version _version): m_version(_version) {}
+
+	Version m_version = Version::Byzantium;
+};
+
+
+}
+}
diff --git a/liblangutil/ErrorReporter.cpp b/liblangutil/ErrorReporter.cpp
new file mode 100644
index 00000000..5b6e0072
--- /dev/null
+++ b/liblangutil/ErrorReporter.cpp
@@ -0,0 +1,226 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Rhett <roadriverrail@gmail.com>
+ * @date 2017
+ * Error helper class.
+ */
+
+#include <liblangutil/ErrorReporter.h>
+#include <liblangutil/SourceLocation.h>
+#include <memory>
+
+using namespace std;
+using namespace dev;
+using namespace langutil;
+
+ErrorReporter& ErrorReporter::operator=(ErrorReporter const& _errorReporter)
+{
+	if (&_errorReporter == this)
+		return *this;
+	m_errorList = _errorReporter.m_errorList;
+	return *this;
+}
+
+
+void ErrorReporter::warning(string const& _description)
+{
+	error(Error::Type::Warning, SourceLocation(), _description);
+}
+
+void ErrorReporter::warning(
+	SourceLocation const& _location,
+	string const& _description
+)
+{
+	error(Error::Type::Warning, _location, _description);
+}
+
+void ErrorReporter::warning(
+	SourceLocation const& _location,
+	string const& _description,
+	SecondarySourceLocation const& _secondaryLocation
+)
+{
+	error(Error::Type::Warning, _location, _secondaryLocation, _description);
+}
+
+void ErrorReporter::error(Error::Type _type, SourceLocation const& _location, string const& _description)
+{
+	if (checkForExcessiveErrors(_type))
+		return;
+
+	auto err = make_shared<Error>(_type);
+	*err <<
+		errinfo_sourceLocation(_location) <<
+		errinfo_comment(_description);
+
+	m_errorList.push_back(err);
+}
+
+void ErrorReporter::error(Error::Type _type, SourceLocation const& _location, SecondarySourceLocation const& _secondaryLocation, string const& _description)
+{
+	if (checkForExcessiveErrors(_type))
+		return;
+
+	auto err = make_shared<Error>(_type);
+	*err <<
+		errinfo_sourceLocation(_location) <<
+		errinfo_secondarySourceLocation(_secondaryLocation) <<
+		errinfo_comment(_description);
+
+	m_errorList.push_back(err);
+}
+
+bool ErrorReporter::checkForExcessiveErrors(Error::Type _type)
+{
+	if (_type == Error::Type::Warning)
+	{
+		m_warningCount++;
+
+		if (m_warningCount == c_maxWarningsAllowed)
+		{
+			auto err = make_shared<Error>(Error::Type::Warning);
+			*err << errinfo_comment("There are more than 256 warnings. Ignoring the rest.");
+			m_errorList.push_back(err);
+		}
+
+		if (m_warningCount >= c_maxWarningsAllowed)
+			return true;
+	}
+	else
+	{
+		m_errorCount++;
+
+		if (m_errorCount > c_maxErrorsAllowed)
+		{
+			auto err = make_shared<Error>(Error::Type::Warning);
+			*err << errinfo_comment("There are more than 256 errors. Aborting.");
+			m_errorList.push_back(err);
+			BOOST_THROW_EXCEPTION(FatalError());
+		}
+	}
+
+	return false;
+}
+
+void ErrorReporter::fatalError(Error::Type _type, SourceLocation const& _location, string const& _description)
+{
+	error(_type, _location, _description);
+	BOOST_THROW_EXCEPTION(FatalError());
+}
+
+ErrorList const& ErrorReporter::errors() const
+{
+	return m_errorList;
+}
+
+void ErrorReporter::clear()
+{
+	m_errorList.clear();
+}
+
+void ErrorReporter::declarationError(SourceLocation const& _location, SecondarySourceLocation const&_secondaryLocation, string const& _description)
+{
+	error(
+		Error::Type::DeclarationError,
+		_location,
+		_secondaryLocation,
+		_description
+	);
+}
+
+void ErrorReporter::declarationError(SourceLocation const& _location, string const& _description)
+{
+	error(
+		Error::Type::DeclarationError,
+		_location,
+		_description
+	);
+}
+
+void ErrorReporter::fatalDeclarationError(SourceLocation const& _location, std::string const& _description)
+{
+	fatalError(
+		Error::Type::DeclarationError,
+		_location,
+		_description);
+}
+
+void ErrorReporter::parserError(SourceLocation const& _location, string const& _description)
+{
+	error(
+		Error::Type::ParserError,
+		_location,
+		_description
+	);
+}
+
+void ErrorReporter::fatalParserError(SourceLocation const& _location, string const& _description)
+{
+	fatalError(
+		Error::Type::ParserError,
+		_location,
+		_description
+	);
+}
+
+void ErrorReporter::syntaxError(SourceLocation const& _location, string const& _description)
+{
+	error(
+		Error::Type::SyntaxError,
+		_location,
+		_description
+	);
+}
+
+void ErrorReporter::typeError(SourceLocation const& _location, SecondarySourceLocation const& _secondaryLocation, string const& _description)
+{
+	error(
+		Error::Type::TypeError,
+		_location,
+		_secondaryLocation,
+		_description
+	);
+}
+
+void ErrorReporter::typeError(SourceLocation const& _location, string const& _description)
+{
+	error(
+		Error::Type::TypeError,
+		_location,
+		_description
+	);
+}
+
+
+void ErrorReporter::fatalTypeError(SourceLocation const& _location, string const& _description)
+{
+	fatalError(Error::Type::TypeError,
+		_location,
+		_description
+	);
+}
+
+void ErrorReporter::docstringParsingError(string const& _description)
+{
+	error(
+		Error::Type::DocstringParsingError,
+		SourceLocation(),
+		_description
+	);
+}
diff --git a/liblangutil/ErrorReporter.h b/liblangutil/ErrorReporter.h
new file mode 100644
index 00000000..d90e652e
--- /dev/null
+++ b/liblangutil/ErrorReporter.h
@@ -0,0 +1,120 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Rhett <roadriverrail@gmail.com>
+ * @date 2017
+ * Error reporting helper class.
+ */
+
+#pragma once
+
+#include <liblangutil/Exceptions.h>
+#include <liblangutil/SourceLocation.h>
+
+namespace langutil
+{
+
+class ErrorReporter
+{
+public:
+
+	explicit ErrorReporter(ErrorList& _errors):
+		m_errorList(_errors) { }
+
+	ErrorReporter(ErrorReporter const& _errorReporter) noexcept:
+		m_errorList(_errorReporter.m_errorList) { }
+
+	ErrorReporter& operator=(ErrorReporter const& _errorReporter);
+
+	void warning(std::string const& _description);
+
+	void warning(SourceLocation const& _location, std::string const& _description);
+
+	void warning(
+		SourceLocation const& _location,
+		std::string const& _description,
+		SecondarySourceLocation const& _secondaryLocation
+	);
+
+	void error(
+		Error::Type _type,
+		SourceLocation const& _location,
+		std::string const& _description
+	);
+
+	void declarationError(
+		SourceLocation const& _location,
+		SecondarySourceLocation const& _secondaryLocation,
+		std::string const& _description
+	);
+
+	void declarationError(SourceLocation const& _location, std::string const& _description);
+
+	void fatalDeclarationError(SourceLocation const& _location, std::string const& _description);
+
+	void parserError(SourceLocation const& _location, std::string const& _description);
+
+	void fatalParserError(SourceLocation const& _location, std::string const& _description);
+
+	void syntaxError(SourceLocation const& _location, std::string const& _description);
+
+	void typeError(
+		SourceLocation const& _location,
+		SecondarySourceLocation const& _secondaryLocation = SecondarySourceLocation(),
+		std::string const& _description = std::string()
+	);
+
+	void typeError(SourceLocation const& _location, std::string const& _description);
+
+	void fatalTypeError(SourceLocation const& _location, std::string const& _description);
+
+	void docstringParsingError(std::string const& _description);
+
+	ErrorList const& errors() const;
+
+	void clear();
+
+	/// @returns true iff there is any error (ignores warnings).
+	bool hasErrors() const
+	{
+		return m_errorCount > 0;
+	}
+
+private:
+	void error(Error::Type _type,
+		SourceLocation const& _location,
+		SecondarySourceLocation const& _secondaryLocation,
+		std::string const& _description = std::string());
+
+	void fatalError(Error::Type _type,
+		SourceLocation const& _location = SourceLocation(),
+		std::string const& _description = std::string());
+
+	// @returns true if error shouldn't be stored
+	bool checkForExcessiveErrors(Error::Type _type);
+
+	ErrorList& m_errorList;
+
+	unsigned m_errorCount = 0;
+	unsigned m_warningCount = 0;
+
+	const unsigned c_maxWarningsAllowed = 256;
+	const unsigned c_maxErrorsAllowed = 256;
+};
+
+}
+
diff --git a/liblangutil/Exceptions.cpp b/liblangutil/Exceptions.cpp
new file mode 100644
index 00000000..346313d5
--- /dev/null
+++ b/liblangutil/Exceptions.cpp
@@ -0,0 +1,66 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Liana <liana@ethdev.com>
+ * @date 2015
+ * Solidity exception hierarchy.
+ */
+
+#include <liblangutil/Exceptions.h>
+
+using namespace std;
+using namespace dev;
+using namespace langutil;
+
+Error::Error(Type _type, SourceLocation const& _location, string const& _description):
+	m_type(_type)
+{
+	switch(m_type)
+	{
+	case Type::DeclarationError:
+		m_typeName = "DeclarationError";
+		break;
+	case Type::DocstringParsingError:
+		m_typeName = "DocstringParsingError";
+		break;
+	case Type::ParserError:
+		m_typeName = "ParserError";
+		break;
+	case Type::SyntaxError:
+		m_typeName = "SyntaxError";
+		break;
+	case Type::TypeError:
+		m_typeName = "TypeError";
+		break;
+	case Type::Warning:
+		m_typeName = "Warning";
+		break;
+	}
+
+	if (!_location.isEmpty())
+		*this << errinfo_sourceLocation(_location);
+	if (!_description.empty())
+		*this << errinfo_comment(_description);
+}
+
+Error::Error(Error::Type _type, const std::string& _description, const SourceLocation& _location):
+	Error(_type)
+{
+	if (!_location.isEmpty())
+		*this << errinfo_sourceLocation(_location);
+	*this << errinfo_comment(_description);
+}
diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h
new file mode 100644
index 00000000..5ad31ab2
--- /dev/null
+++ b/liblangutil/Exceptions.h
@@ -0,0 +1,131 @@
+/*
+    This file is part of solidity.
+
+    solidity is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    solidity is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity exception hierarchy.
+ */
+
+#pragma once
+
+#include <string>
+#include <utility>
+#include <vector>
+#include <memory>
+#include <libdevcore/Exceptions.h>
+#include <libdevcore/Assertions.h>
+#include <liblangutil/SourceLocation.h>
+
+namespace langutil
+{
+class Error;
+using ErrorList = std::vector<std::shared_ptr<Error const>>;
+
+struct CompilerError: virtual dev::Exception {};
+struct InternalCompilerError: virtual dev::Exception {};
+struct FatalError: virtual dev::Exception {};
+struct UnimplementedFeatureError: virtual dev::Exception {};
+
+/// Assertion that throws an InternalCompilerError containing the given description if it is not met.
+#define solAssert(CONDITION, DESCRIPTION) \
+        assertThrow(CONDITION, ::langutil::InternalCompilerError, DESCRIPTION)
+
+#define solUnimplementedAssert(CONDITION, DESCRIPTION) \
+        assertThrow(CONDITION, ::langutil::UnimplementedFeatureError, DESCRIPTION)
+
+#define solUnimplemented(DESCRIPTION) \
+        solUnimplementedAssert(false, DESCRIPTION)
+
+class Error: virtual public dev::Exception
+{
+public:
+	enum class Type
+	{
+		DeclarationError,
+		DocstringParsingError,
+		ParserError,
+		TypeError,
+		SyntaxError,
+		Warning
+	};
+
+	explicit Error(
+		Type _type,
+		SourceLocation const& _location = SourceLocation(),
+		std::string const& _description = std::string()
+	);
+
+	Error(Type _type, std::string const& _description, SourceLocation const& _location = SourceLocation());
+
+	Type type() const { return m_type; }
+	std::string const& typeName() const { return m_typeName; }
+
+	/// helper functions
+	static Error const* containsErrorOfType(ErrorList const& _list, Error::Type _type)
+	{
+		for (auto e: _list)
+		{
+			if (e->type() == _type)
+				return e.get();
+		}
+		return nullptr;
+	}
+	static bool containsOnlyWarnings(ErrorList const& _list)
+	{
+		for (auto e: _list)
+		{
+			if (e->type() != Type::Warning)
+				return false;
+		}
+		return true;
+	}
+private:
+	Type m_type;
+	std::string m_typeName;
+};
+
+using errorSourceLocationInfo = std::pair<std::string, SourceLocation>;
+
+class SecondarySourceLocation
+{
+public:
+	SecondarySourceLocation& append(std::string const& _errMsg, SourceLocation const& _sourceLocation)
+	{
+		infos.push_back(std::make_pair(_errMsg, _sourceLocation));
+		return *this;
+	}
+
+	/// Limits the number of secondary source locations to 32 and appends a notice to the
+	/// error message.
+	void limitSize(std::string& _message)
+	{
+		size_t occurrences = infos.size();
+		if (occurrences > 32)
+		{
+			infos.resize(32);
+			_message += " Truncated from " + std::to_string(occurrences) + " to the first 32 occurrences.";
+		}
+	}
+
+	std::vector<errorSourceLocationInfo> infos;
+};
+
+using errinfo_sourceLocation = boost::error_info<struct tag_sourceLocation, SourceLocation>;
+using errinfo_secondarySourceLocation = boost::error_info<struct tag_secondarySourceLocation, SecondarySourceLocation>;
+
+
+}
diff --git a/liblangutil/ParserBase.cpp b/liblangutil/ParserBase.cpp
new file mode 100644
index 00000000..391af291
--- /dev/null
+++ b/liblangutil/ParserBase.cpp
@@ -0,0 +1,109 @@
+/*
+    This file is part of solidity.
+
+    solidity is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    solidity is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2016
+ * Solidity parser shared functionality.
+ */
+
+#include <liblangutil/ParserBase.h>
+#include <liblangutil/Scanner.h>
+#include <liblangutil/ErrorReporter.h>
+
+using namespace std;
+using namespace langutil;
+
+int ParserBase::position() const
+{
+	return m_scanner->currentLocation().start;
+}
+
+int ParserBase::endPosition() const
+{
+	return m_scanner->currentLocation().end;
+}
+
+Token ParserBase::currentToken() const
+{
+	return m_scanner->currentToken();
+}
+
+Token ParserBase::peekNextToken() const
+{
+	return m_scanner->peekNextToken();
+}
+
+std::string ParserBase::currentLiteral() const
+{
+	return m_scanner->currentLiteral();
+}
+
+Token ParserBase::advance()
+{
+	return m_scanner->next();
+}
+
+void ParserBase::expectToken(Token _value, bool _advance)
+{
+	Token tok = m_scanner->currentToken();
+	if (tok != _value)
+	{
+		auto tokenName = [this](Token _token)
+		{
+			if (_token == Token::Identifier)
+				return string("identifier");
+			else if (_token == Token::EOS)
+				return string("end of source");
+			else if (TokenTraits::isReservedKeyword(_token))
+				return string("reserved keyword '") + TokenTraits::friendlyName(_token) + "'";
+			else if (TokenTraits::isElementaryTypeName(_token)) //for the sake of accuracy in reporting
+			{
+				ElementaryTypeNameToken elemTypeName = m_scanner->currentElementaryTypeNameToken();
+				return string("'") + elemTypeName.toString() + "'";
+			}
+			else
+				return string("'") + TokenTraits::friendlyName(_token) + "'";
+		};
+
+		fatalParserError(string("Expected ") + tokenName(_value) + string(" but got ") + tokenName(tok));
+	}
+	if (_advance)
+		m_scanner->next();
+}
+
+void ParserBase::increaseRecursionDepth()
+{
+	m_recursionDepth++;
+	if (m_recursionDepth >= 2560)
+		fatalParserError("Maximum recursion depth reached during parsing.");
+}
+
+void ParserBase::decreaseRecursionDepth()
+{
+	solAssert(m_recursionDepth > 0, "");
+	m_recursionDepth--;
+}
+
+void ParserBase::parserError(string const& _description)
+{
+	m_errorReporter.parserError(SourceLocation(position(), endPosition(), source()), _description);
+}
+
+void ParserBase::fatalParserError(string const& _description)
+{
+	m_errorReporter.fatalParserError(SourceLocation(position(), endPosition(), source()), _description);
+}
diff --git a/liblangutil/ParserBase.h b/liblangutil/ParserBase.h
new file mode 100644
index 00000000..855201e2
--- /dev/null
+++ b/liblangutil/ParserBase.h
@@ -0,0 +1,92 @@
+/*
+    This file is part of solidity.
+
+    solidity is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    solidity is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2016
+ * Solidity parser shared functionality.
+ */
+
+#pragma once
+
+#include <liblangutil/Token.h>
+#include <liblangutil/Scanner.h>
+#include <memory>
+#include <string>
+
+namespace langutil
+{
+
+class ErrorReporter;
+class Scanner;
+
+class ParserBase
+{
+public:
+	explicit ParserBase(ErrorReporter& errorReporter): m_errorReporter(errorReporter) {}
+
+	std::shared_ptr<CharStream> source() const { return m_scanner->charStream(); }
+
+protected:
+	/// Utility class that creates an error and throws an exception if the
+	/// recursion depth is too deep.
+	class RecursionGuard
+	{
+	public:
+		explicit RecursionGuard(ParserBase& _parser): m_parser(_parser)
+		{
+			m_parser.increaseRecursionDepth();
+		}
+		~RecursionGuard() { m_parser.decreaseRecursionDepth(); }
+	private:
+		ParserBase& m_parser;
+	};
+
+	/// Start position of the current token
+	int position() const;
+	/// End position of the current token
+	int endPosition() const;
+
+	///@{
+	///@name Helper functions
+	/// If current token value is not _value, throw exception otherwise advance token.
+	void expectToken(Token _value, bool _advance = true);
+	Token currentToken() const;
+	Token peekNextToken() const;
+	std::string currentLiteral() const;
+	Token advance();
+	///@}
+
+	/// Increases the recursion depth and throws an exception if it is too deep.
+	void increaseRecursionDepth();
+	void decreaseRecursionDepth();
+
+	/// Creates a @ref ParserError and annotates it with the current position and the
+	/// given @a _description.
+	void parserError(std::string const& _description);
+
+	/// Creates a @ref ParserError and annotates it with the current position and the
+	/// given @a _description. Throws the FatalError.
+	void fatalParserError(std::string const& _description);
+
+	std::shared_ptr<Scanner> m_scanner;
+	/// The reference to the list of errors and warning to add errors/warnings during parsing
+	ErrorReporter& m_errorReporter;
+	/// Current recursion depth during parsing.
+	size_t m_recursionDepth = 0;
+};
+
+}
diff --git a/liblangutil/Scanner.cpp b/liblangutil/Scanner.cpp
new file mode 100644
index 00000000..ac298bd5
--- /dev/null
+++ b/liblangutil/Scanner.cpp
@@ -0,0 +1,904 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+	This file is derived from the file "scanner.cc", which was part of the
+	V8 project. The original copyright header follows:
+
+	Copyright 2006-2012, the V8 project authors. All rights reserved.
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are
+	met:
+
+	* Redistributions of source code must retain the above copyright
+	  notice, this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above
+	  copyright notice, this list of conditions and the following
+	  disclaimer in the documentation and/or other materials provided
+	  with the distribution.
+	* Neither the name of Google Inc. nor the names of its
+	  contributors may be used to endorse or promote products derived
+	  from this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity scanner.
+ */
+
+#include <liblangutil/Exceptions.h>
+#include <liblangutil/Scanner.h>
+#include <algorithm>
+#include <ostream>
+#include <tuple>
+
+using namespace std;
+
+namespace langutil
+{
+
+namespace
+{
+bool isDecimalDigit(char c)
+{
+	return '0' <= c && c <= '9';
+}
+bool isHexDigit(char c)
+{
+	return isDecimalDigit(c)
+		   || ('a' <= c && c <= 'f')
+		   || ('A' <= c && c <= 'F');
+}
+bool isLineTerminator(char c)
+{
+	return c == '\n';
+}
+bool isWhiteSpace(char c)
+{
+	return c == ' ' || c == '\n' || c == '\t' || c == '\r';
+}
+bool isIdentifierStart(char c)
+{
+	return c == '_' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
+}
+bool isIdentifierPart(char c)
+{
+	return isIdentifierStart(c) || isDecimalDigit(c);
+}
+int hexValue(char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	else if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	else if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+	else return -1;
+}
+} // end anonymous namespace
+
+std::string to_string(ScannerError _errorCode)
+{
+	switch (_errorCode)
+	{
+		case ScannerError::NoError: return "No error.";
+		case ScannerError::IllegalToken: return "Invalid token.";
+		case ScannerError::IllegalHexString: return "Expected even number of hex-nibbles within double-quotes.";
+		case ScannerError::IllegalHexDigit: return "Hexadecimal digit missing or invalid.";
+		case ScannerError::IllegalCommentTerminator: return "Expected multi-line comment-terminator.";
+		case ScannerError::IllegalEscapeSequence: return "Invalid escape sequence.";
+		case ScannerError::IllegalStringEndQuote: return "Expected string end-quote.";
+		case ScannerError::IllegalNumberSeparator: return "Invalid use of number separator '_'.";
+		case ScannerError::IllegalExponent: return "Invalid exponent.";
+		case ScannerError::IllegalNumberEnd: return "Identifier-start is not allowed at end of a number.";
+		case ScannerError::OctalNotAllowed: return "Octal numbers not allowed.";
+		default:
+			solAssert(false, "Unhandled case in to_string(ScannerError)");
+			return "";
+	}
+}
+
+std::ostream& operator<<(std::ostream& os, ScannerError _errorCode)
+{
+	os << to_string(_errorCode);
+	return os;
+}
+
+/// Scoped helper for literal recording. Automatically drops the literal
+/// if aborting the scanning before it's complete.
+enum LiteralType {
+	LITERAL_TYPE_STRING,
+	LITERAL_TYPE_NUMBER, // not really different from string type in behaviour
+	LITERAL_TYPE_COMMENT
+};
+
+class LiteralScope
+{
+public:
+	explicit LiteralScope(Scanner* _self, enum LiteralType _type): m_type(_type)
+	, m_scanner(_self)
+	, m_complete(false)
+	{
+		if (_type == LITERAL_TYPE_COMMENT)
+			m_scanner->m_nextSkippedComment.literal.clear();
+		else
+			m_scanner->m_nextToken.literal.clear();
+	}
+	~LiteralScope()
+	{
+		if (!m_complete)
+		{
+			if (m_type == LITERAL_TYPE_COMMENT)
+				m_scanner->m_nextSkippedComment.literal.clear();
+			else
+				m_scanner->m_nextToken.literal.clear();
+		}
+	}
+	void complete() { m_complete = true; }
+
+private:
+	enum LiteralType m_type;
+	Scanner* m_scanner;
+	bool m_complete;
+}; // end of LiteralScope class
+
+
+void Scanner::reset(CharStream _source)
+{
+	m_source = make_shared<CharStream>(std::move(_source));
+	reset();
+}
+
+void Scanner::reset(std::shared_ptr<CharStream> _source)
+{
+	solAssert(_source.get() != nullptr, "You MUST provide a CharStream when resetting.");
+	m_source = _source;
+	reset();
+}
+
+void Scanner::reset()
+{
+	m_source->reset();
+	m_char = m_source->get();
+	skipWhitespace();
+	scanToken();
+	next();
+}
+
+bool Scanner::scanHexByte(char& o_scannedByte)
+{
+	char x = 0;
+	for (int i = 0; i < 2; i++)
+	{
+		int d = hexValue(m_char);
+		if (d < 0)
+		{
+			rollback(i);
+			return false;
+		}
+		x = x * 16 + d;
+		advance();
+	}
+	o_scannedByte = x;
+	return true;
+}
+
+bool Scanner::scanUnicode(unsigned & o_codepoint)
+{
+	unsigned x = 0;
+	for (int i = 0; i < 4; i++)
+	{
+		int d = hexValue(m_char);
+		if (d < 0)
+		{
+			rollback(i);
+			return false;
+		}
+		x = x * 16 + d;
+		advance();
+	}
+	o_codepoint = x;
+	return true;
+}
+
+// This supports codepoints between 0000 and FFFF.
+void Scanner::addUnicodeAsUTF8(unsigned codepoint)
+{
+	if (codepoint <= 0x7f)
+		addLiteralChar(codepoint);
+	else if (codepoint <= 0x7ff)
+	{
+		addLiteralChar(0xc0 | (codepoint >> 6));
+		addLiteralChar(0x80 | (codepoint & 0x3f));
+	}
+	else
+	{
+		addLiteralChar(0xe0 | (codepoint >> 12));
+		addLiteralChar(0x80 | ((codepoint >> 6) & 0x3f));
+		addLiteralChar(0x80 | (codepoint & 0x3f));
+	}
+}
+
+// Ensure that tokens can be stored in a byte.
+BOOST_STATIC_ASSERT(TokenTraits::count() <= 0x100);
+
+Token Scanner::next()
+{
+	m_currentToken = m_nextToken;
+	m_skippedComment = m_nextSkippedComment;
+	scanToken();
+
+	return m_currentToken.token;
+}
+
+Token Scanner::selectToken(char _next, Token _then, Token _else)
+{
+	advance();
+	if (m_char == _next)
+		return selectToken(_then);
+	else
+		return _else;
+}
+
+bool Scanner::skipWhitespace()
+{
+	int const startPosition = sourcePos();
+	while (isWhiteSpace(m_char))
+		advance();
+	// Return whether or not we skipped any characters.
+	return sourcePos() != startPosition;
+}
+
+void Scanner::skipWhitespaceExceptUnicodeLinebreak()
+{
+	while (isWhiteSpace(m_char) && !isUnicodeLinebreak())
+		advance();
+}
+
+Token Scanner::skipSingleLineComment()
+{
+	// Line terminator is not part of the comment. If it is a
+	// non-ascii line terminator, it will result in a parser error.
+	while (!isUnicodeLinebreak())
+		if (!advance()) break;
+
+	return Token::Whitespace;
+}
+
+Token Scanner::scanSingleLineDocComment()
+{
+	LiteralScope literal(this, LITERAL_TYPE_COMMENT);
+	advance(); //consume the last '/' at ///
+
+	skipWhitespaceExceptUnicodeLinebreak();
+
+	while (!isSourcePastEndOfInput())
+	{
+		if (isLineTerminator(m_char))
+		{
+			// check if next line is also a documentation comment
+			skipWhitespace();
+			if (!m_source->isPastEndOfInput(3) &&
+				m_source->get(0) == '/' &&
+				m_source->get(1) == '/' &&
+				m_source->get(2) == '/')
+			{
+				addCommentLiteralChar('\n');
+				m_char = m_source->advanceAndGet(3);
+			}
+			else
+				break; // next line is not a documentation comment, we are done
+
+		}
+		else if (isUnicodeLinebreak())
+			// Any line terminator that is not '\n' is considered to end the
+			// comment.
+			break;
+		addCommentLiteralChar(m_char);
+		advance();
+	}
+	literal.complete();
+	return Token::CommentLiteral;
+}
+
+Token Scanner::skipMultiLineComment()
+{
+	advance();
+	while (!isSourcePastEndOfInput())
+	{
+		char ch = m_char;
+		advance();
+
+		// If we have reached the end of the multi-line comment, we
+		// consume the '/' and insert a whitespace. This way all
+		// multi-line comments are treated as whitespace.
+		if (ch == '*' && m_char == '/')
+		{
+			m_char = ' ';
+			return Token::Whitespace;
+		}
+	}
+	// Unterminated multi-line comment.
+	return setError(ScannerError::IllegalCommentTerminator);
+}
+
+Token Scanner::scanMultiLineDocComment()
+{
+	LiteralScope literal(this, LITERAL_TYPE_COMMENT);
+	bool endFound = false;
+	bool charsAdded = false;
+
+	while (isWhiteSpace(m_char) && !isLineTerminator(m_char))
+		advance();
+
+	while (!isSourcePastEndOfInput())
+	{
+		//handle newlines in multline comments
+		if (isLineTerminator(m_char))
+		{
+			skipWhitespace();
+			if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '*')
+			{ // it is unknown if this leads to the end of the comment
+				addCommentLiteralChar('*');
+				advance();
+			}
+			else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) != '/')
+			{ // skip first '*' in subsequent lines
+				if (charsAdded)
+					addCommentLiteralChar('\n');
+				m_char = m_source->advanceAndGet(2);
+			}
+			else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
+			{ // if after newline the comment ends, don't insert the newline
+				m_char = m_source->advanceAndGet(2);
+				endFound = true;
+				break;
+			}
+			else if (charsAdded)
+				addCommentLiteralChar('\n');
+		}
+
+		if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
+		{
+			m_char = m_source->advanceAndGet(2);
+			endFound = true;
+			break;
+		}
+		addCommentLiteralChar(m_char);
+		charsAdded = true;
+		advance();
+	}
+	literal.complete();
+	if (!endFound)
+		return setError(ScannerError::IllegalCommentTerminator);
+	else
+		return Token::CommentLiteral;
+}
+
+Token Scanner::scanSlash()
+{
+	int firstSlashPosition = sourcePos();
+	advance();
+	if (m_char == '/')
+	{
+		if (!advance()) /* double slash comment directly before EOS */
+			return Token::Whitespace;
+		else if (m_char == '/')
+		{
+			// doxygen style /// comment
+			Token comment;
+			m_nextSkippedComment.location.start = firstSlashPosition;
+			comment = scanSingleLineDocComment();
+			m_nextSkippedComment.location.end = sourcePos();
+			m_nextSkippedComment.token = comment;
+			return Token::Whitespace;
+		}
+		else
+			return skipSingleLineComment();
+	}
+	else if (m_char == '*')
+	{
+		// doxygen style /** natspec comment
+		if (!advance()) /* slash star comment before EOS */
+			return setError(ScannerError::IllegalCommentTerminator);
+		else if (m_char == '*')
+		{
+			advance(); //consume the last '*' at /**
+
+			// "/**/"
+			if (m_char == '/')
+			{
+				advance(); //skip the closing slash
+				return Token::Whitespace;
+			}
+			// we actually have a multiline documentation comment
+			Token comment;
+			m_nextSkippedComment.location.start = firstSlashPosition;
+			comment = scanMultiLineDocComment();
+			m_nextSkippedComment.location.end = sourcePos();
+			m_nextSkippedComment.token = comment;
+			if (comment == Token::Illegal)
+				return Token::Illegal; // error already set
+			else
+				return Token::Whitespace;
+		}
+		else
+			return skipMultiLineComment();
+	}
+	else if (m_char == '=')
+		return selectToken(Token::AssignDiv);
+	else
+		return Token::Div;
+}
+
+void Scanner::scanToken()
+{
+	m_nextToken.error = ScannerError::NoError;
+	m_nextToken.literal.clear();
+	m_nextToken.extendedTokenInfo = make_tuple(0, 0);
+	m_nextSkippedComment.literal.clear();
+	m_nextSkippedComment.extendedTokenInfo = make_tuple(0, 0);
+
+	Token token;
+	// M and N are for the purposes of grabbing different type sizes
+	unsigned m;
+	unsigned n;
+	do
+	{
+		// Remember the position of the next token
+		m_nextToken.location.start = sourcePos();
+		switch (m_char)
+		{
+		case '"':
+		case '\'':
+			token = scanString();
+			break;
+		case '<':
+			// < <= << <<=
+			advance();
+			if (m_char == '=')
+				token = selectToken(Token::LessThanOrEqual);
+			else if (m_char == '<')
+				token = selectToken('=', Token::AssignShl, Token::SHL);
+			else
+				token = Token::LessThan;
+			break;
+		case '>':
+			// > >= >> >>= >>> >>>=
+			advance();
+			if (m_char == '=')
+				token = selectToken(Token::GreaterThanOrEqual);
+			else if (m_char == '>')
+			{
+				// >> >>= >>> >>>=
+				advance();
+				if (m_char == '=')
+					token = selectToken(Token::AssignSar);
+				else if (m_char == '>')
+					token = selectToken('=', Token::AssignShr, Token::SHR);
+				else
+					token = Token::SAR;
+			}
+			else
+				token = Token::GreaterThan;
+			break;
+		case '=':
+			// = == =>
+			advance();
+			if (m_char == '=')
+				token = selectToken(Token::Equal);
+			else if (m_char == '>')
+				token = selectToken(Token::Arrow);
+			else
+				token = Token::Assign;
+			break;
+		case '!':
+			// ! !=
+			advance();
+			if (m_char == '=')
+				token = selectToken(Token::NotEqual);
+			else
+				token = Token::Not;
+			break;
+		case '+':
+			// + ++ +=
+			advance();
+			if (m_char == '+')
+				token = selectToken(Token::Inc);
+			else if (m_char == '=')
+				token = selectToken(Token::AssignAdd);
+			else
+				token = Token::Add;
+			break;
+		case '-':
+			// - -- -=
+			advance();
+			if (m_char == '-')
+				token = selectToken(Token::Dec);
+			else if (m_char == '=')
+				token = selectToken(Token::AssignSub);
+			else
+				token = Token::Sub;
+			break;
+		case '*':
+			// * ** *=
+			advance();
+			if (m_char == '*')
+				token = selectToken(Token::Exp);
+			else if (m_char == '=')
+				token = selectToken(Token::AssignMul);
+			else
+				token = Token::Mul;
+			break;
+		case '%':
+			// % %=
+			token = selectToken('=', Token::AssignMod, Token::Mod);
+			break;
+		case '/':
+			// /  // /* /=
+			token = scanSlash();
+			break;
+		case '&':
+			// & && &=
+			advance();
+			if (m_char == '&')
+				token = selectToken(Token::And);
+			else if (m_char == '=')
+				token = selectToken(Token::AssignBitAnd);
+			else
+				token = Token::BitAnd;
+			break;
+		case '|':
+			// | || |=
+			advance();
+			if (m_char == '|')
+				token = selectToken(Token::Or);
+			else if (m_char == '=')
+				token = selectToken(Token::AssignBitOr);
+			else
+				token = Token::BitOr;
+			break;
+		case '^':
+			// ^ ^=
+			token = selectToken('=', Token::AssignBitXor, Token::BitXor);
+			break;
+		case '.':
+			// . Number
+			advance();
+			if (isDecimalDigit(m_char))
+				token = scanNumber('.');
+			else
+				token = Token::Period;
+			break;
+		case ':':
+			token = selectToken(Token::Colon);
+			break;
+		case ';':
+			token = selectToken(Token::Semicolon);
+			break;
+		case ',':
+			token = selectToken(Token::Comma);
+			break;
+		case '(':
+			token = selectToken(Token::LParen);
+			break;
+		case ')':
+			token = selectToken(Token::RParen);
+			break;
+		case '[':
+			token = selectToken(Token::LBrack);
+			break;
+		case ']':
+			token = selectToken(Token::RBrack);
+			break;
+		case '{':
+			token = selectToken(Token::LBrace);
+			break;
+		case '}':
+			token = selectToken(Token::RBrace);
+			break;
+		case '?':
+			token = selectToken(Token::Conditional);
+			break;
+		case '~':
+			token = selectToken(Token::BitNot);
+			break;
+		default:
+			if (isIdentifierStart(m_char))
+			{
+				tie(token, m, n) = scanIdentifierOrKeyword();
+
+				// Special case for hexadecimal literals
+				if (token == Token::Hex)
+				{
+					// reset
+					m = 0;
+					n = 0;
+
+					// Special quoted hex string must follow
+					if (m_char == '"' || m_char == '\'')
+						token = scanHexString();
+					else
+						token = setError(ScannerError::IllegalToken);
+				}
+			}
+			else if (isDecimalDigit(m_char))
+				token = scanNumber();
+			else if (skipWhitespace())
+				token = Token::Whitespace;
+			else if (isSourcePastEndOfInput())
+				token = Token::EOS;
+			else
+				token = selectErrorToken(ScannerError::IllegalToken);
+			break;
+		}
+		// Continue scanning for tokens as long as we're just skipping
+		// whitespace.
+	}
+	while (token == Token::Whitespace);
+	m_nextToken.location.end = sourcePos();
+	m_nextToken.token = token;
+	m_nextToken.extendedTokenInfo = make_tuple(m, n);
+}
+
+bool Scanner::scanEscape()
+{
+	char c = m_char;
+	advance();
+	// Skip escaped newlines.
+	if (isLineTerminator(c))
+		return true;
+	switch (c)
+	{
+	case '\'':  // fall through
+	case '"':  // fall through
+	case '\\':
+		break;
+	case 'b':
+		c = '\b';
+		break;
+	case 'f':
+		c = '\f';
+		break;
+	case 'n':
+		c = '\n';
+		break;
+	case 'r':
+		c = '\r';
+		break;
+	case 't':
+		c = '\t';
+		break;
+	case 'v':
+		c = '\v';
+		break;
+	case 'u':
+	{
+		unsigned codepoint;
+		if (!scanUnicode(codepoint))
+			return false;
+		addUnicodeAsUTF8(codepoint);
+		return true;
+	}
+	case 'x':
+		if (!scanHexByte(c))
+			return false;
+		break;
+	default:
+		return false;
+	}
+
+	addLiteralChar(c);
+	return true;
+}
+
+bool Scanner::isUnicodeLinebreak()
+{
+	if (0x0a <= m_char && m_char <= 0x0d)
+		// line feed, vertical tab, form feed, carriage return
+		return true;
+	else if (!m_source->isPastEndOfInput(1) && uint8_t(m_source->get(0)) == 0xc2 && uint8_t(m_source->get(1)) == 0x85)
+		// NEL - U+0085, C2 85 in utf8
+		return true;
+	else if (!m_source->isPastEndOfInput(2) && uint8_t(m_source->get(0)) == 0xe2 && uint8_t(m_source->get(1)) == 0x80 && (
+		uint8_t(m_source->get(2)) == 0xa8 || uint8_t(m_source->get(2)) == 0xa9
+	))
+		// LS - U+2028, E2 80 A8  in utf8
+		// PS - U+2029, E2 80 A9  in utf8
+		return true;
+	else
+		return false;
+}
+
+Token Scanner::scanString()
+{
+	char const quote = m_char;
+	advance();  // consume quote
+	LiteralScope literal(this, LITERAL_TYPE_STRING);
+	while (m_char != quote && !isSourcePastEndOfInput() && !isUnicodeLinebreak())
+	{
+		char c = m_char;
+		advance();
+		if (c == '\\')
+		{
+			if (isSourcePastEndOfInput() || !scanEscape())
+				return setError(ScannerError::IllegalEscapeSequence);
+		}
+		else
+			addLiteralChar(c);
+	}
+	if (m_char != quote)
+		return setError(ScannerError::IllegalStringEndQuote);
+	literal.complete();
+	advance();  // consume quote
+	return Token::StringLiteral;
+}
+
+Token Scanner::scanHexString()
+{
+	char const quote = m_char;
+	advance();  // consume quote
+	LiteralScope literal(this, LITERAL_TYPE_STRING);
+	while (m_char != quote && !isSourcePastEndOfInput())
+	{
+		char c = m_char;
+		if (!scanHexByte(c))
+			// can only return false if hex-byte is incomplete (only one hex digit instead of two)
+			return setError(ScannerError::IllegalHexString);
+		addLiteralChar(c);
+	}
+
+	if (m_char != quote)
+		return setError(ScannerError::IllegalStringEndQuote);
+
+	literal.complete();
+	advance();  // consume quote
+	return Token::StringLiteral;
+}
+
+// Parse for regex [:digit:]+(_[:digit:]+)*
+void Scanner::scanDecimalDigits()
+{
+	// MUST begin with a decimal digit.
+	if (!isDecimalDigit(m_char))
+		return;
+
+	// May continue with decimal digit or underscore for grouping.
+	do addLiteralCharAndAdvance();
+	while (!m_source->isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
+
+	// Defer further validation of underscore to SyntaxChecker.
+}
+
+Token Scanner::scanNumber(char _charSeen)
+{
+	enum { DECIMAL, HEX, BINARY } kind = DECIMAL;
+	LiteralScope literal(this, LITERAL_TYPE_NUMBER);
+	if (_charSeen == '.')
+	{
+		// we have already seen a decimal point of the float
+		addLiteralChar('.');
+		if (m_char == '_')
+			return setError(ScannerError::IllegalToken);
+		scanDecimalDigits();  // we know we have at least one digit
+	}
+	else
+	{
+		solAssert(_charSeen == 0, "");
+		// if the first character is '0' we must check for octals and hex
+		if (m_char == '0')
+		{
+			addLiteralCharAndAdvance();
+			// either 0, 0exxx, 0Exxx, 0.xxx or a hex number
+			if (m_char == 'x')
+			{
+				// hex number
+				kind = HEX;
+				addLiteralCharAndAdvance();
+				if (!isHexDigit(m_char))
+					return setError(ScannerError::IllegalHexDigit); // we must have at least one hex digit after 'x'
+
+				while (isHexDigit(m_char) || m_char == '_') // We keep the underscores for later validation
+					addLiteralCharAndAdvance();
+			}
+			else if (isDecimalDigit(m_char))
+				// We do not allow octal numbers
+				return setError(ScannerError::OctalNotAllowed);
+		}
+		// Parse decimal digits and allow trailing fractional part.
+		if (kind == DECIMAL)
+		{
+			scanDecimalDigits();  // optional
+			if (m_char == '.')
+			{
+				if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
+				{
+					// Assume the input may be a floating point number with leading '_' in fraction part.
+					// Recover by consuming it all but returning `Illegal` right away.
+					addLiteralCharAndAdvance(); // '.'
+					addLiteralCharAndAdvance(); // '_'
+					scanDecimalDigits();
+				}
+				if (m_source->isPastEndOfInput() || !isDecimalDigit(m_source->get(1)))
+				{
+					// A '.' has to be followed by a number.
+					literal.complete();
+					return Token::Number;
+				}
+				addLiteralCharAndAdvance();
+				scanDecimalDigits();
+			}
+		}
+	}
+	// scan exponent, if any
+	if (m_char == 'e' || m_char == 'E')
+	{
+		solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
+		if (kind != DECIMAL)
+			return setError(ScannerError::IllegalExponent);
+		else if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
+		{
+			// Recover from wrongly placed underscore as delimiter in literal with scientific
+			// notation by consuming until the end.
+			addLiteralCharAndAdvance(); // 'e'
+			addLiteralCharAndAdvance(); // '_'
+			scanDecimalDigits();
+			literal.complete();
+			return Token::Number;
+		}
+		// scan exponent
+		addLiteralCharAndAdvance(); // 'e' | 'E'
+		if (m_char == '+' || m_char == '-')
+			addLiteralCharAndAdvance();
+		if (!isDecimalDigit(m_char)) // we must have at least one decimal digit after 'e'/'E'
+			return setError(ScannerError::IllegalExponent);
+		scanDecimalDigits();
+	}
+	// The source character immediately following a numeric literal must
+	// not be an identifier start or a decimal digit; see ECMA-262
+	// section 7.8.3, page 17 (note that we read only one decimal digit
+	// if the value is 0).
+	if (isDecimalDigit(m_char) || isIdentifierStart(m_char))
+		return setError(ScannerError::IllegalNumberEnd);
+	literal.complete();
+	return Token::Number;
+}
+
+tuple<Token, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
+{
+	solAssert(isIdentifierStart(m_char), "");
+	LiteralScope literal(this, LITERAL_TYPE_STRING);
+	addLiteralCharAndAdvance();
+	// Scan the rest of the identifier characters.
+	while (isIdentifierPart(m_char)) //get full literal
+		addLiteralCharAndAdvance();
+	literal.complete();
+	return TokenTraits::fromIdentifierOrKeyword(m_nextToken.literal);
+}
+
+
+}
diff --git a/liblangutil/Scanner.h b/liblangutil/Scanner.h
new file mode 100644
index 00000000..72d0072f
--- /dev/null
+++ b/liblangutil/Scanner.h
@@ -0,0 +1,248 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+	This file is derived from the file "scanner.h", which was part of the
+	V8 project. The original copyright header follows:
+
+	Copyright 2006-2012, the V8 project authors. All rights reserved.
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are
+	met:
+
+	* Redistributions of source code must retain the above copyright
+	  notice, this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above
+	  copyright notice, this list of conditions and the following
+	  disclaimer in the documentation and/or other materials provided
+	  with the distribution.
+	* Neither the name of Google Inc. nor the names of its
+	  contributors may be used to endorse or promote products derived
+	  from this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity scanner.
+ */
+
+#pragma once
+
+#include <liblangutil/Token.h>
+#include <liblangutil/CharStream.h>
+#include <liblangutil/SourceLocation.h>
+#include <libdevcore/Common.h>
+#include <libdevcore/CommonData.h>
+#include <iosfwd>
+
+namespace langutil
+{
+
+class AstRawString;
+class AstValueFactory;
+class ParserRecorder;
+
+enum class ScannerError
+{
+	NoError,
+
+	IllegalToken,
+	IllegalHexString,
+	IllegalHexDigit,
+	IllegalCommentTerminator,
+	IllegalEscapeSequence,
+	IllegalStringEndQuote,
+	IllegalNumberSeparator,
+	IllegalExponent,
+	IllegalNumberEnd,
+
+	OctalNotAllowed,
+};
+
+std::string to_string(ScannerError _errorCode);
+std::ostream& operator<<(std::ostream& os, ScannerError _errorCode);
+
+class Scanner
+{
+	friend class LiteralScope;
+public:
+	explicit Scanner(std::shared_ptr<CharStream> _source) { reset(std::move(_source)); }
+	explicit Scanner(CharStream _source = CharStream()) { reset(std::move(_source)); }
+
+	std::string const& source() const noexcept { return m_source->source(); }
+
+	std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
+
+	/// Resets the scanner as if newly constructed with _source as input.
+	void reset(CharStream _source);
+	void reset(std::shared_ptr<CharStream> _source);
+	/// Resets scanner to the start of input.
+	void reset();
+
+	/// @returns the next token and advances input
+	Token next();
+
+	///@{
+	///@name Information about the current token
+
+	/// @returns the current token
+	Token currentToken() const
+	{
+		return m_currentToken.token;
+	}
+	ElementaryTypeNameToken currentElementaryTypeNameToken() const
+	{
+		unsigned firstSize;
+		unsigned secondSize;
+		std::tie(firstSize, secondSize) = m_currentToken.extendedTokenInfo;
+		return ElementaryTypeNameToken(m_currentToken.token, firstSize, secondSize);
+	}
+
+	SourceLocation currentLocation() const { return m_currentToken.location; }
+	std::string const& currentLiteral() const { return m_currentToken.literal; }
+	std::tuple<unsigned, unsigned> const& currentTokenInfo() const { return m_currentToken.extendedTokenInfo; }
+
+	/// Retrieves the last error that occurred during lexical analysis.
+	/// @note If no error occurred, the value is undefined.
+	ScannerError currentError() const noexcept { return m_currentToken.error; }
+	///@}
+
+	///@{
+	///@name Information about the current comment token
+
+	SourceLocation currentCommentLocation() const { return m_skippedComment.location; }
+	std::string const& currentCommentLiteral() const { return m_skippedComment.literal; }
+	/// Called by the parser during FunctionDefinition parsing to clear the current comment
+	void clearCurrentCommentLiteral() { m_skippedComment.literal.clear(); }
+
+	///@}
+
+	///@{
+	///@name Information about the next token
+
+	/// @returns the next token without advancing input.
+	Token peekNextToken() const { return m_nextToken.token; }
+	SourceLocation peekLocation() const { return m_nextToken.location; }
+	std::string const& peekLiteral() const { return m_nextToken.literal; }
+	///@}
+
+	///@{
+	///@name Error printing helper functions
+	/// Functions that help pretty-printing parse errors
+	/// Do only use in error cases, they are quite expensive.
+	std::string lineAtPosition(int _position) const { return m_source->lineAtPosition(_position); }
+	std::tuple<int, int> translatePositionToLineColumn(int _position) const { return m_source->translatePositionToLineColumn(_position); }
+	std::string sourceAt(SourceLocation const& _location) const
+	{
+		solAssert(!_location.isEmpty(), "");
+		solAssert(m_source.get() == _location.source.get(), "CharStream memory locations must match.");
+		return m_source->source().substr(_location.start, _location.end - _location.start);
+	}
+	///@}
+
+private:
+	inline Token setError(ScannerError _error) noexcept
+	{
+		m_nextToken.error = _error;
+		return Token::Illegal;
+	}
+
+	/// Used for the current and look-ahead token and comments
+	struct TokenDesc
+	{
+		Token token;
+		SourceLocation location;
+		std::string literal;
+		ScannerError error = ScannerError::NoError;
+		std::tuple<unsigned, unsigned> extendedTokenInfo;
+	};
+
+	///@{
+	///@name Literal buffer support
+	inline void addLiteralChar(char c) { m_nextToken.literal.push_back(c); }
+	inline void addCommentLiteralChar(char c) { m_nextSkippedComment.literal.push_back(c); }
+	inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); }
+	void addUnicodeAsUTF8(unsigned codepoint);
+	///@}
+
+	bool advance() { m_char = m_source->advanceAndGet(); return !m_source->isPastEndOfInput(); }
+	void rollback(int _amount) { m_char = m_source->rollback(_amount); }
+
+	inline Token selectErrorToken(ScannerError _err) { advance(); return setError(_err); }
+	inline Token selectToken(Token _tok) { advance(); return _tok; }
+	/// If the next character is _next, advance and return _then, otherwise return _else.
+	inline Token selectToken(char _next, Token _then, Token _else);
+
+	bool scanHexByte(char& o_scannedByte);
+	bool scanUnicode(unsigned& o_codepoint);
+
+	/// Scans a single Solidity token.
+	void scanToken();
+
+	/// Skips all whitespace and @returns true if something was skipped.
+	bool skipWhitespace();
+	/// Skips all whitespace that are neither '\r' nor '\n'.
+	void skipWhitespaceExceptUnicodeLinebreak();
+	Token skipSingleLineComment();
+	Token skipMultiLineComment();
+
+	void scanDecimalDigits();
+	Token scanNumber(char _charSeen = 0);
+	std::tuple<Token, unsigned, unsigned> scanIdentifierOrKeyword();
+
+	Token scanString();
+	Token scanHexString();
+	Token scanSingleLineDocComment();
+	Token scanMultiLineDocComment();
+	/// Scans a slash '/' and depending on the characters returns the appropriate token
+	Token scanSlash();
+
+	/// Scans an escape-sequence which is part of a string and adds the
+	/// decoded character to the current literal. Returns true if a pattern
+	/// is scanned.
+	bool scanEscape();
+
+	/// @returns true iff we are currently positioned at a unicode line break.
+	bool isUnicodeLinebreak();
+
+	/// Return the current source position.
+	int sourcePos() const { return m_source->position(); }
+	bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
+
+	TokenDesc m_skippedComment;  // desc for current skipped comment
+	TokenDesc m_nextSkippedComment; // desc for next skipped comment
+
+	TokenDesc m_currentToken;  // desc for current token (as returned by Next())
+	TokenDesc m_nextToken;     // desc for next token (one token look-ahead)
+
+	std::shared_ptr<CharStream> m_source;
+
+	/// one character look-ahead, equals 0 at end of input
+	char m_char;
+};
+
+}
diff --git a/liblangutil/SourceLocation.h b/liblangutil/SourceLocation.h
new file mode 100644
index 00000000..2d18a7d1
--- /dev/null
+++ b/liblangutil/SourceLocation.h
@@ -0,0 +1,97 @@
+/*
+    This file is part of solidity.
+
+    solidity is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    solidity is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Lefteris Karapetsas <lefteris@ethdev.com>
+ * @date 2015
+ * Represents a location in a source file
+ */
+
+#pragma once
+
+#include <libdevcore/Common.h> // defines noexcept macro for MSVC
+#include <liblangutil/CharStream.h>
+#include <memory>
+#include <string>
+#include <ostream>
+#include <tuple>
+
+namespace langutil
+{
+
+/**
+ * Representation of an interval of source positions.
+ * The interval includes start and excludes end.
+ */
+struct SourceLocation
+{
+	SourceLocation(): start(-1), end(-1), source{nullptr} { }
+	SourceLocation(int _start, int _end, std::shared_ptr<CharStream> _source):
+		start(_start), end(_end), source{std::move(_source)} { }
+
+	bool operator==(SourceLocation const& _other) const
+	{
+		return source.get() == _other.source.get() && start == _other.start && end == _other.end;
+	}
+	bool operator!=(SourceLocation const& _other) const { return !operator==(_other); }
+	inline bool operator<(SourceLocation const& _other) const;
+	inline bool contains(SourceLocation const& _other) const;
+	inline bool intersects(SourceLocation const& _other) const;
+
+	bool isEmpty() const { return start == -1 && end == -1; }
+
+	int start;
+	int end;
+	std::shared_ptr<CharStream> source;
+};
+
+/// Stream output for Location (used e.g. in boost exceptions).
+inline std::ostream& operator<<(std::ostream& _out, SourceLocation const& _location)
+{
+	if (_location.isEmpty())
+		return _out << "NO_LOCATION_SPECIFIED";
+
+	if (_location.source)
+		_out << _location.source->name();
+
+	_out << "[" << _location.start << "," << _location.end << ")";
+
+	return _out;
+}
+
+bool SourceLocation::operator<(SourceLocation const& _other) const
+{
+	if (!source|| !_other.source)
+		return std::make_tuple(int(!!source), start, end) < std::make_tuple(int(!!_other.source), _other.start, _other.end);
+	else
+		return std::make_tuple(source->name(), start, end) < std::make_tuple(_other.source->name(), _other.start, _other.end);
+}
+
+bool SourceLocation::contains(SourceLocation const& _other) const
+{
+	if (isEmpty() || _other.isEmpty() || source.get() != _other.source.get())
+		return false;
+	return start <= _other.start && _other.end <= end;
+}
+
+bool SourceLocation::intersects(SourceLocation const& _other) const
+{
+	if (isEmpty() || _other.isEmpty() || source.get() != _other.source.get())
+		return false;
+	return _other.start < end && start < _other.end;
+}
+
+}
diff --git a/liblangutil/SourceReferenceFormatter.cpp b/liblangutil/SourceReferenceFormatter.cpp
new file mode 100644
index 00000000..8ac05b4e
--- /dev/null
+++ b/liblangutil/SourceReferenceFormatter.cpp
@@ -0,0 +1,129 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Formatting functions for errors referencing positions and locations in the source.
+ */
+
+#include <liblangutil/SourceReferenceFormatter.h>
+#include <liblangutil/Scanner.h>
+#include <liblangutil/Exceptions.h>
+
+using namespace std;
+using namespace dev;
+using namespace langutil;
+
+void SourceReferenceFormatter::printSourceLocation(SourceLocation const* _location)
+{
+	if (!_location || !_location->source)
+		return; // Nothing we can print here
+	auto const& scanner = m_scannerFromSourceName(_location->source->name());
+	int startLine;
+	int startColumn;
+	tie(startLine, startColumn) = scanner.translatePositionToLineColumn(_location->start);
+	int endLine;
+	int endColumn;
+	tie(endLine, endColumn) = scanner.translatePositionToLineColumn(_location->end);
+	if (startLine == endLine)
+	{
+		string line = scanner.lineAtPosition(_location->start);
+
+		int locationLength = endColumn - startColumn;
+		if (locationLength > 150)
+		{
+			line = line.substr(0, startColumn + 35) + " ... " + line.substr(endColumn - 35);
+			endColumn = startColumn + 75;
+			locationLength = 75;
+		}
+		if (line.length() > 150)
+		{
+			int len = line.length();
+			line = line.substr(max(0, startColumn - 35), min(startColumn, 35) + min(locationLength + 35, len - startColumn));
+			if (startColumn + locationLength + 35 < len)
+				line += " ...";
+			if (startColumn > 35)
+			{
+				line = " ... " + line;
+				startColumn = 40;
+			}
+			endColumn = startColumn + locationLength;
+		}
+
+		m_stream << line << endl;
+
+		for_each(
+			line.cbegin(),
+			line.cbegin() + startColumn,
+			[this](char const& ch) { m_stream << (ch == '\t' ? '\t' : ' '); }
+		);
+		m_stream << "^";
+		if (endColumn > startColumn + 2)
+			m_stream << string(endColumn - startColumn - 2, '-');
+		if (endColumn > startColumn + 1)
+			m_stream << "^";
+		m_stream << endl;
+	}
+	else
+		m_stream <<
+			scanner.lineAtPosition(_location->start) <<
+			endl <<
+			string(startColumn, ' ') <<
+			"^ (Relevant source part starts here and spans across multiple lines)." <<
+			endl;
+}
+
+void SourceReferenceFormatter::printSourceName(SourceLocation const* _location)
+{
+	if (!_location || !_location->source)
+		return; // Nothing we can print here
+	auto const& scanner = m_scannerFromSourceName(_location->source->name());
+	int startLine;
+	int startColumn;
+	tie(startLine, startColumn) = scanner.translatePositionToLineColumn(_location->start);
+	m_stream << _location->source->name() << ":" << (startLine + 1) << ":" << (startColumn + 1) << ": ";
+}
+
+void SourceReferenceFormatter::printExceptionInformation(
+	dev::Exception const& _exception,
+	string const& _name
+)
+{
+	SourceLocation const* location = boost::get_error_info<errinfo_sourceLocation>(_exception);
+	auto secondarylocation = boost::get_error_info<errinfo_secondarySourceLocation>(_exception);
+
+	printSourceName(location);
+
+	m_stream << _name;
+	if (string const* description = boost::get_error_info<errinfo_comment>(_exception))
+		m_stream << ": " << *description << endl;
+	else
+		m_stream << endl;
+
+	printSourceLocation(location);
+
+	if (secondarylocation && !secondarylocation->infos.empty())
+	{
+		for (auto info: secondarylocation->infos)
+		{
+			printSourceName(&info.second);
+			m_stream << info.first << endl;
+			printSourceLocation(&info.second);
+		}
+		m_stream << endl;
+	}
+}
diff --git a/liblangutil/SourceReferenceFormatter.h b/liblangutil/SourceReferenceFormatter.h
new file mode 100644
index 00000000..0ef3ca00
--- /dev/null
+++ b/liblangutil/SourceReferenceFormatter.h
@@ -0,0 +1,76 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Formatting functions for errors referencing positions and locations in the source.
+ */
+
+#pragma once
+
+#include <ostream>
+#include <sstream>
+#include <functional>
+
+namespace dev
+{
+struct Exception; // forward
+}
+
+namespace langutil
+{
+struct SourceLocation;
+class Scanner;
+
+class SourceReferenceFormatter
+{
+public:
+	using ScannerFromSourceNameFun = std::function<langutil::Scanner const&(std::string const&)>;
+
+	explicit SourceReferenceFormatter(
+		std::ostream& _stream,
+		ScannerFromSourceNameFun _scannerFromSourceName
+	):
+		m_stream(_stream),
+		m_scannerFromSourceName(std::move(_scannerFromSourceName))
+	{}
+
+	/// Prints source location if it is given.
+	void printSourceLocation(langutil::SourceLocation const* _location);
+	void printExceptionInformation(dev::Exception const& _exception, std::string const& _name);
+
+	static std::string formatExceptionInformation(
+		dev::Exception const& _exception,
+		std::string const& _name,
+		ScannerFromSourceNameFun const& _scannerFromSourceName
+	)
+	{
+		std::ostringstream errorOutput;
+
+		SourceReferenceFormatter formatter(errorOutput, _scannerFromSourceName);
+		formatter.printExceptionInformation(_exception, _name);
+		return errorOutput.str();
+	}
+private:
+	/// Prints source name if location is given.
+	void printSourceName(langutil::SourceLocation const* _location);
+
+	std::ostream& m_stream;
+	ScannerFromSourceNameFun m_scannerFromSourceName;
+};
+
+}
diff --git a/liblangutil/Token.cpp b/liblangutil/Token.cpp
new file mode 100644
index 00000000..cbfd4a8c
--- /dev/null
+++ b/liblangutil/Token.cpp
@@ -0,0 +1,204 @@
+// Copyright 2006-2012, the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+//      notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+//      copyright notice, this list of conditions and the following
+//      disclaimer in the documentation and/or other materials provided
+//      with the distribution.
+//    * Neither the name of Google Inc. nor the names of its
+//      contributors may be used to endorse or promote products derived
+//      from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Modifications as part of solidity under the following license:
+//
+// solidity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// solidity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+#include <liblangutil/Token.h>
+#include <boost/range/iterator_range.hpp>
+#include <map>
+
+using namespace std;
+
+namespace langutil
+{
+
+void ElementaryTypeNameToken::assertDetails(Token _baseType, unsigned const& _first, unsigned const& _second)
+{
+	solAssert(TokenTraits::isElementaryTypeName(_baseType), "Expected elementary type name: " + string(TokenTraits::toString(_baseType)));
+	if (_baseType == Token::BytesM)
+	{
+		solAssert(_second == 0, "There should not be a second size argument to type bytesM.");
+		solAssert(_first <= 32, "No elementary type bytes" + to_string(_first) + ".");
+	}
+	else if (_baseType == Token::UIntM || _baseType == Token::IntM)
+	{
+		solAssert(_second == 0, "There should not be a second size argument to type " + string(TokenTraits::toString(_baseType)) + ".");
+		solAssert(
+			_first <= 256 && _first % 8 == 0,
+			"No elementary type " + string(TokenTraits::toString(_baseType)) + to_string(_first) + "."
+		);
+	}
+	else if (_baseType == Token::UFixedMxN || _baseType == Token::FixedMxN)
+	{
+		solAssert(
+			_first >= 8 && _first <= 256 && _first % 8 == 0 && _second <= 80,
+			"No elementary type " + string(TokenTraits::toString(_baseType)) + to_string(_first) + "x" + to_string(_second) + "."
+		);
+	}
+	m_token = _baseType;
+	m_firstNumber = _first;
+	m_secondNumber = _second;
+}
+
+namespace TokenTraits
+{
+
+char const* toString(Token tok)
+{
+	switch (tok)
+	{
+#define T(name, string, precedence) case Token::name: return string;
+		TOKEN_LIST(T, T)
+#undef T
+		default: // Token::NUM_TOKENS:
+			return "";
+	}
+}
+
+char const* name(Token tok)
+{
+#define T(name, string, precedence) #name,
+	static char const* const names[TokenTraits::count()] = { TOKEN_LIST(T, T) };
+#undef T
+
+	solAssert(static_cast<size_t>(tok) < TokenTraits::count(), "");
+	return names[static_cast<size_t>(tok)];
+}
+
+std::string friendlyName(Token tok)
+{
+	char const* ret = toString(tok);
+	if (ret)
+		return std::string(ret);
+
+	ret = name(tok);
+	solAssert(ret != nullptr, "");
+	return std::string(ret);
+}
+
+#define T(name, string, precedence) precedence,
+int precedence(Token tok)
+{
+	int8_t const static precs[TokenTraits::count()] =
+	{
+		TOKEN_LIST(T, T)
+	};
+	return precs[static_cast<size_t>(tok)];
+}
+#undef T
+
+int parseSize(string::const_iterator _begin, string::const_iterator _end)
+{
+	try
+	{
+		unsigned int m = boost::lexical_cast<int>(boost::make_iterator_range(_begin, _end));
+		return m;
+	}
+	catch(boost::bad_lexical_cast const&)
+	{
+		return -1;
+	}
+}
+
+static Token keywordByName(string const& _name)
+{
+	// The following macros are used inside TOKEN_LIST and cause non-keyword tokens to be ignored
+	// and keywords to be put inside the keywords variable.
+#define KEYWORD(name, string, precedence) {string, Token::name},
+#define TOKEN(name, string, precedence)
+	static const map<string, Token> keywords({TOKEN_LIST(TOKEN, KEYWORD)});
+#undef KEYWORD
+#undef TOKEN
+	auto it = keywords.find(_name);
+	return it == keywords.end() ? Token::Identifier : it->second;
+}
+
+tuple<Token, unsigned int, unsigned int> fromIdentifierOrKeyword(string const& _literal)
+{
+	auto positionM = find_if(_literal.begin(), _literal.end(), ::isdigit);
+	if (positionM != _literal.end())
+	{
+		string baseType(_literal.begin(), positionM);
+		auto positionX = find_if_not(positionM, _literal.end(), ::isdigit);
+		int m = parseSize(positionM, positionX);
+		Token keyword = keywordByName(baseType);
+		if (keyword == Token::Bytes)
+		{
+			if (0 < m && m <= 32 && positionX == _literal.end())
+				return make_tuple(Token::BytesM, m, 0);
+		}
+		else if (keyword == Token::UInt || keyword == Token::Int)
+		{
+			if (0 < m && m <= 256 && m % 8 == 0 && positionX == _literal.end())
+			{
+				if (keyword == Token::UInt)
+					return make_tuple(Token::UIntM, m, 0);
+				else
+					return make_tuple(Token::IntM, m, 0);
+			}
+		}
+		else if (keyword == Token::UFixed || keyword == Token::Fixed)
+		{
+			if (
+				positionM < positionX &&
+				positionX < _literal.end() &&
+				*positionX == 'x' &&
+				all_of(positionX + 1, _literal.end(), ::isdigit)
+			) {
+				int n = parseSize(positionX + 1, _literal.end());
+				if (
+					8 <= m && m <= 256 && m % 8 == 0 &&
+					0 <= n && n <= 80
+				) {
+					if (keyword == Token::UFixed)
+						return make_tuple(Token::UFixedMxN, m, n);
+					else
+						return make_tuple(Token::FixedMxN, m, n);
+				}
+			}
+		}
+		return make_tuple(Token::Identifier, 0, 0);
+	}
+
+	return make_tuple(keywordByName(_literal), 0, 0);
+}
+
+}
+}
diff --git a/liblangutil/Token.h b/liblangutil/Token.h
new file mode 100644
index 00000000..f832fdf7
--- /dev/null
+++ b/liblangutil/Token.h
@@ -0,0 +1,376 @@
+// Copyright 2006-2012, the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+//      notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+//      copyright notice, this list of conditions and the following
+//      disclaimer in the documentation and/or other materials provided
+//      with the distribution.
+//    * Neither the name of Google Inc. nor the names of its
+//      contributors may be used to endorse or promote products derived
+//      from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Modifications as part of solidity under the following license:
+//
+// solidity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// solidity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+
+#include <libdevcore/Common.h>
+#include <liblangutil/Exceptions.h>
+#include <liblangutil/UndefMacros.h>
+
+#include <iosfwd>
+#include <string>
+#include <tuple>
+
+namespace langutil
+{
+
+// TOKEN_LIST takes a list of 3 macros M, all of which satisfy the
+// same signature M(name, string, precedence), where name is the
+// symbolic token name, string is the corresponding syntactic symbol
+// (or nullptr, for literals), and precedence is the precedence (or 0).
+// The parameters are invoked for token categories as follows:
+//
+//   T: Non-keyword tokens
+//   K: Keyword tokens
+
+// IGNORE_TOKEN is a convenience macro that can be supplied as
+// an argument (at any position) for a TOKEN_LIST call. It does
+// nothing with tokens belonging to the respective category.
+
+#define IGNORE_TOKEN(name, string, precedence)
+
+#define TOKEN_LIST(T, K)												\
+	/* End of source indicator. */										\
+	T(EOS, "EOS", 0)													\
+																		\
+	/* Punctuators (ECMA-262, section 7.7, page 15). */				\
+	T(LParen, "(", 0)                                                   \
+	T(RParen, ")", 0)                                                   \
+	T(LBrack, "[", 0)                                                   \
+	T(RBrack, "]", 0)                                                   \
+	T(LBrace, "{", 0)                                                   \
+	T(RBrace, "}", 0)                                                   \
+	T(Colon, ":", 0)                                                    \
+	T(Semicolon, ";", 0)                                                \
+	T(Period, ".", 0)                                                   \
+	T(Conditional, "?", 3)                                              \
+	T(Arrow, "=>", 0)                                                   \
+	\
+	/* Assignment operators. */										\
+	/* IsAssignmentOp() relies on this block of enum values being */	\
+	/* contiguous and sorted in the same order!*/						\
+	T(Assign, "=", 2)                                                   \
+	/* The following have to be in exactly the same order as the simple binary operators*/ \
+	T(AssignBitOr, "|=", 2)                                           \
+	T(AssignBitXor, "^=", 2)                                          \
+	T(AssignBitAnd, "&=", 2)                                          \
+	T(AssignShl, "<<=", 2)                                            \
+	T(AssignSar, ">>=", 2)                                            \
+	T(AssignShr, ">>>=", 2)                                           \
+	T(AssignAdd, "+=", 2)                                             \
+	T(AssignSub, "-=", 2)                                             \
+	T(AssignMul, "*=", 2)                                             \
+	T(AssignDiv, "/=", 2)                                             \
+	T(AssignMod, "%=", 2)                                             \
+	\
+	/* Binary operators sorted by precedence. */                       \
+	/* IsBinaryOp() relies on this block of enum values */             \
+	/* being contiguous and sorted in the same order! */               \
+	T(Comma, ",", 1)                                                   \
+	T(Or, "||", 4)                                                     \
+	T(And, "&&", 5)                                                    \
+	T(BitOr, "|", 8)                                                   \
+	T(BitXor, "^", 9)                                                  \
+	T(BitAnd, "&", 10)                                                 \
+	T(SHL, "<<", 11)                                                   \
+	T(SAR, ">>", 11)                                                   \
+	T(SHR, ">>>", 11)                                                  \
+	T(Add, "+", 12)                                                    \
+	T(Sub, "-", 12)                                                    \
+	T(Mul, "*", 13)                                                    \
+	T(Div, "/", 13)                                                    \
+	T(Mod, "%", 13)                                                    \
+	T(Exp, "**", 14)                                                   \
+	\
+	/* Compare operators sorted by precedence. */                      \
+	/* IsCompareOp() relies on this block of enum values */            \
+	/* being contiguous and sorted in the same order! */               \
+	T(Equal, "==", 6)                                                  \
+	T(NotEqual, "!=", 6)                                               \
+	T(LessThan, "<", 7)                                                \
+	T(GreaterThan, ">", 7)                                             \
+	T(LessThanOrEqual, "<=", 7)                                        \
+	T(GreaterThanOrEqual, ">=", 7)                                     \
+	\
+	/* Unary operators. */                                             \
+	/* IsUnaryOp() relies on this block of enum values */              \
+	/* being contiguous and sorted in the same order! */               \
+	T(Not, "!", 0)                                                     \
+	T(BitNot, "~", 0)                                                  \
+	T(Inc, "++", 0)                                                    \
+	T(Dec, "--", 0)                                                    \
+	K(Delete, "delete", 0)                                             \
+	\
+	/* Keywords */                                                     \
+	K(Anonymous, "anonymous", 0)                                       \
+	K(As, "as", 0)                                                     \
+	K(Assembly, "assembly", 0)                                         \
+	K(Break, "break", 0)                                               \
+	K(Constant, "constant", 0)                                         \
+	K(Constructor, "constructor", 0)                                   \
+	K(Continue, "continue", 0)                                         \
+	K(Contract, "contract", 0)                                         \
+	K(Do, "do", 0)                                                     \
+	K(Else, "else", 0)                                                 \
+	K(Enum, "enum", 0)                                                 \
+	K(Emit, "emit", 0)                                                 \
+	K(Event, "event", 0)                                               \
+	K(External, "external", 0)                                         \
+	K(For, "for", 0)                                                   \
+	K(Function, "function", 0)                                         \
+	K(Hex, "hex", 0)                                                   \
+	K(If, "if", 0)                                                     \
+	K(Indexed, "indexed", 0)                                           \
+	K(Interface, "interface", 0)                                       \
+	K(Internal, "internal", 0)                                         \
+	K(Import, "import", 0)                                             \
+	K(Is, "is", 0)                                                     \
+	K(Library, "library", 0)                                           \
+	K(Mapping, "mapping", 0)                                           \
+	K(Memory, "memory", 0)                                             \
+	K(Modifier, "modifier", 0)                                         \
+	K(New, "new", 0)                                                   \
+	K(Payable, "payable", 0)                                           \
+	K(Public, "public", 0)                                             \
+	K(Pragma, "pragma", 0)                                             \
+	K(Private, "private", 0)                                           \
+	K(Pure, "pure", 0)                                                 \
+	K(Return, "return", 0)                                             \
+	K(Returns, "returns", 0)                                           \
+	K(Storage, "storage", 0)                                           \
+	K(CallData, "calldata", 0)                                         \
+	K(Struct, "struct", 0)                                             \
+	K(Throw, "throw", 0)                                               \
+	K(Using, "using", 0)                                               \
+	K(Var, "var", 0)                                                   \
+	K(View, "view", 0)                                                 \
+	K(While, "while", 0)                                               \
+	\
+	/* Ether subdenominations */                                       \
+	K(SubWei, "wei", 0)                                                \
+	K(SubSzabo, "szabo", 0)                                            \
+	K(SubFinney, "finney", 0)                                          \
+	K(SubEther, "ether", 0)                                            \
+	K(SubSecond, "seconds", 0)                                         \
+	K(SubMinute, "minutes", 0)                                         \
+	K(SubHour, "hours", 0)                                             \
+	K(SubDay, "days", 0)                                               \
+	K(SubWeek, "weeks", 0)                                             \
+	K(SubYear, "years", 0)                                             \
+	/* type keywords*/                                                 \
+	K(Int, "int", 0)                                                   \
+	K(UInt, "uint", 0)                                                 \
+	K(Bytes, "bytes", 0)                                               \
+	K(Byte, "byte", 0)                                                 \
+	K(String, "string", 0)                                             \
+	K(Address, "address", 0)                                           \
+	K(Bool, "bool", 0)                                                 \
+	K(Fixed, "fixed", 0)                                               \
+	K(UFixed, "ufixed", 0)                                             \
+	T(IntM, "intM", 0)                                                 \
+	T(UIntM, "uintM", 0)                                               \
+	T(BytesM, "bytesM", 0)                                             \
+	T(FixedMxN, "fixedMxN", 0)                                         \
+	T(UFixedMxN, "ufixedMxN", 0)                                       \
+	T(TypesEnd, nullptr, 0) /* used as type enum end marker */         \
+	\
+	/* Literals */                                                     \
+	K(TrueLiteral, "true", 0)                                          \
+	K(FalseLiteral, "false", 0)                                        \
+	T(Number, nullptr, 0)                                              \
+	T(StringLiteral, nullptr, 0)                                       \
+	T(CommentLiteral, nullptr, 0)                                      \
+	\
+	/* Identifiers (not keywords or future reserved words). */         \
+	T(Identifier, nullptr, 0)                                          \
+	\
+	/* Keywords reserved for future use. */                            \
+	K(Abstract, "abstract", 0)                                         \
+	K(After, "after", 0)                                               \
+	K(Alias, "alias", 0)                                               \
+	K(Apply, "apply", 0)                                               \
+	K(Auto, "auto", 0)                                                 \
+	K(Case, "case", 0)                                                 \
+	K(Catch, "catch", 0)                                               \
+	K(CopyOf, "copyof", 0)                                             \
+	K(Default, "default", 0)                                           \
+	K(Define, "define", 0)                                             \
+	K(Final, "final", 0)                                               \
+	K(Immutable, "immutable", 0)                                       \
+	K(Implements, "implements", 0)                                     \
+	K(In, "in", 0)                                                     \
+	K(Inline, "inline", 0)                                             \
+	K(Let, "let", 0)                                                   \
+	K(Macro, "macro", 0)                                               \
+	K(Match, "match", 0)                                               \
+	K(Mutable, "mutable", 0)                                           \
+	K(NullLiteral, "null", 0)                                          \
+	K(Of, "of", 0)                                                     \
+	K(Override, "override", 0)                                         \
+	K(Partial, "partial", 0)                                           \
+	K(Promise, "promise", 0)                                           \
+	K(Reference, "reference", 0)                                       \
+	K(Relocatable, "relocatable", 0)                                   \
+	K(Sealed, "sealed", 0)                                             \
+	K(Sizeof, "sizeof", 0)                                             \
+	K(Static, "static", 0)                                             \
+	K(Supports, "supports", 0)                                         \
+	K(Switch, "switch", 0)                                             \
+	K(Try, "try", 0)                                                   \
+	K(Type, "type", 0)                                                 \
+	K(Typedef, "typedef", 0)                                           \
+	K(TypeOf, "typeof", 0)                                             \
+	K(Unchecked, "unchecked", 0)                                       \
+	\
+	/* Illegal token - not able to scan. */                            \
+	T(Illegal, "ILLEGAL", 0)                                           \
+	\
+	/* Scanner-internal use only. */                                   \
+	T(Whitespace, nullptr, 0)
+
+// All token values.
+// attention! msvc issue:
+// http://stackoverflow.com/questions/9567868/compile-errors-after-adding-v8-to-my-project-c2143-c2059
+// @todo: avoid TOKEN_LIST macro
+enum class Token : unsigned int {
+#define T(name, string, precedence) name,
+	TOKEN_LIST(T, T)
+	NUM_TOKENS
+#undef T
+};
+
+namespace TokenTraits
+{
+	constexpr size_t count() { return static_cast<size_t>(Token::NUM_TOKENS); }
+
+	// Predicates
+	constexpr bool isElementaryTypeName(Token tok) { return Token::Int <= tok && tok < Token::TypesEnd; }
+	constexpr bool isAssignmentOp(Token tok) { return Token::Assign <= tok && tok <= Token::AssignMod; }
+	constexpr bool isBinaryOp(Token op) { return Token::Comma <= op && op <= Token::Exp; }
+	constexpr bool isCommutativeOp(Token op) { return op == Token::BitOr || op == Token::BitXor || op == Token::BitAnd ||
+		 op == Token::Add || op == Token::Mul || op == Token::Equal || op == Token::NotEqual; }
+	constexpr bool isArithmeticOp(Token op) { return Token::Add <= op && op <= Token::Exp; }
+	constexpr bool isCompareOp(Token op) { return Token::Equal <= op && op <= Token::GreaterThanOrEqual; }
+
+	constexpr bool isBitOp(Token op) { return (Token::BitOr <= op && op <= Token::BitAnd) || op == Token::BitNot; }
+	constexpr bool isBooleanOp(Token op) { return (Token::Or <= op && op <= Token::And) || op == Token::Not; }
+	constexpr bool isUnaryOp(Token op) { return (Token::Not <= op && op <= Token::Delete) || op == Token::Add || op == Token::Sub; }
+	constexpr bool isCountOp(Token op) { return op == Token::Inc || op == Token::Dec; }
+	constexpr bool isShiftOp(Token op) { return (Token::SHL <= op) && (op <= Token::SHR); }
+	constexpr bool isVariableVisibilitySpecifier(Token op) { return op == Token::Public || op == Token::Private || op == Token::Internal; }
+	constexpr bool isVisibilitySpecifier(Token op) { return isVariableVisibilitySpecifier(op) || op == Token::External; }
+	constexpr bool isLocationSpecifier(Token op) { return op == Token::Memory || op == Token::Storage || op == Token::CallData; }
+
+	constexpr bool isStateMutabilitySpecifier(Token op, bool _allowConstant = true)
+	{
+		return (op == Token::Constant && _allowConstant)
+			|| op == Token::Pure || op == Token::View || op == Token::Payable;
+	}
+
+	constexpr bool isEtherSubdenomination(Token op) { return op == Token::SubWei || op == Token::SubSzabo || op == Token::SubFinney || op == Token::SubEther; }
+	constexpr bool isTimeSubdenomination(Token op) { return op == Token::SubSecond || op == Token::SubMinute || op == Token::SubHour || op == Token::SubDay || op == Token::SubWeek || op == Token::SubYear; }
+	constexpr bool isReservedKeyword(Token op) { return (Token::Abstract <= op && op <= Token::Unchecked); }
+
+	inline Token AssignmentToBinaryOp(Token op)
+	{
+		solAssert(isAssignmentOp(op) && op != Token::Assign, "");
+		return static_cast<Token>(static_cast<int>(op) + (static_cast<int>(Token::BitOr) - static_cast<int>(Token::AssignBitOr)));
+	}
+
+	// @returns the precedence > 0 for binary and compare
+	// operators; returns 0 otherwise.
+	int precedence(Token tok);
+
+	std::tuple<Token, unsigned int, unsigned int> fromIdentifierOrKeyword(std::string const& _literal);
+
+	// @returns a string corresponding to the C++ token name
+	// (e.g. "LT" for the token LT).
+	char const* name(Token tok);
+
+	// @returns a string corresponding to the JS token string
+	// (.e., "<" for the token LT) or nullptr if the token doesn't
+	// have a (unique) string (e.g. an IDENTIFIER).
+	char const* toString(Token tok);
+
+	std::string friendlyName(Token tok);
+}
+
+inline std::ostream& operator<<(std::ostream& os, Token token)
+{
+	os << TokenTraits::friendlyName(token);
+	return os;
+}
+
+class ElementaryTypeNameToken
+{
+public:
+	ElementaryTypeNameToken(Token _token, unsigned const& _firstNumber, unsigned const& _secondNumber)
+	{
+		assertDetails(_token, _firstNumber, _secondNumber);
+	}
+
+	unsigned int firstNumber() const { return m_firstNumber; }
+	unsigned int secondNumber() const { return m_secondNumber; }
+	Token token() const { return m_token; }
+
+	///if tokValue is set to true, then returns the actual token type name, otherwise, returns full type
+	std::string toString(bool const& tokenValue = false) const
+	{
+		std::string name = TokenTraits::toString(m_token);
+		if (tokenValue || (firstNumber() == 0 && secondNumber() == 0))
+			return name;
+		solAssert(name.size() >= 3, "Token name size should be greater than 3. Should not reach here.");
+		if (m_token == Token::FixedMxN || m_token == Token::UFixedMxN)
+			return name.substr(0, name.size() - 3) + std::to_string(m_firstNumber) + "x" + std::to_string(m_secondNumber);
+		else
+			return name.substr(0, name.size() - 1) + std::to_string(m_firstNumber);
+	}
+
+private:
+	Token m_token;
+	unsigned int m_firstNumber;
+	unsigned int m_secondNumber;
+	/// throws if type is not properly sized
+	void assertDetails(Token _baseType, unsigned const& _first, unsigned const& _second);
+};
+
+}
diff --git a/liblangutil/UndefMacros.h b/liblangutil/UndefMacros.h
new file mode 100644
index 00000000..d96e242e
--- /dev/null
+++ b/liblangutil/UndefMacros.h
@@ -0,0 +1,46 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file UndefMacros.h
+ * @author Lefteris  <lefteris@ethdev.com>
+ * @date 2015
+ *
+ * This header should be used to #undef some really evil macros defined by
+ * windows.h which result in conflict with our Token.h
+ */
+#pragma once
+
+#if defined(_MSC_VER) || defined(__MINGW32__)
+
+#undef DELETE
+#undef IN
+#undef VOID
+#undef THIS
+#undef CONST
+
+// Conflicting define on MinGW in windows.h
+// windows.h(19): #define interface struct
+#ifdef interface
+#undef interface
+#endif
+
+#elif defined(DELETE) || defined(IN) || defined(VOID) || defined(THIS) || defined(CONST) || defined(interface)
+
+#error "The preceding macros in this header file are reserved for V8's "\
+"TOKEN_LIST. Please add a platform specific define above to undefine "\
+"overlapping macros."
+
+#endif