added two functions in Token to handle long identifiers, redid fromIdentifierOrKeyword, and made complementary changes in scanner and parser

5 files changed, 138 insertions, 55 deletions

diff --git a/libsolidity/parsing/Parser.cpp b/libsolidity/parsing/Parser.cpp
index 74ec5cda..e579f18b 100644
--- a/libsolidity/parsing/Parser.cpp
+++ b/libsolidity/parsing/Parser.cpp
@@ -590,7 +590,10 @@ ASTPointer<TypeName> Parser::parseTypeName(bool _allowVar)
 	Token::Value token = m_scanner->currentToken();
 	if (Token::isElementaryTypeName(token))
 	{
-		ElementaryTypeNameToken elemTypeName(token, m_scanner->currentTokenInfo());
+		unsigned firstSize;
+		unsigned secondSize;
+		tie(firstSize, secondSize) = m_scanner->currentTokenInfo();
+		ElementaryTypeNameToken elemTypeName(token, firstSize, secondSize);
 		type = ASTNodeFactory(*this).createNode<ElementaryTypeName>(elemTypeName);
 		m_scanner->next();
 	}
@@ -628,11 +631,14 @@ ASTPointer<Mapping> Parser::parseMapping()
 	expectToken(Token::Mapping);
 	expectToken(Token::LParen);
 	ASTPointer<ElementaryTypeName> keyType;
-	if (Token::isElementaryTypeName(m_scanner->currentToken()))
-		keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>
-			(ElementaryTypeNameToken(m_scanner->currentToken(), m_scanner->currentTokenInfo()));
-	else
+	Token::Value token = m_scanner->currentToken();
+	if (!Token::isElementaryTypeName(token))
 		fatalParserError(string("Expected elementary type name for mapping key type"));
+	unsigned firstSize;
+	unsigned secondSize;
+	tie(firstSize, secondSize) = m_scanner->currentTokenInfo();
+	ElementaryTypeNameToken elemTypeName(token, firstSize, secondSize);
+	keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>(elemTypeName);
 	m_scanner->next();
 	expectToken(Token::Arrow);
 	bool const allowVar = false;
@@ -829,7 +835,10 @@ ASTPointer<Statement> Parser::parseSimpleStatement(ASTPointer<ASTString> const&
 	else
 	{
 		startedWithElementary = true;
-		ElementaryTypeNameToken elemToken(m_scanner->currentToken(), m_scanner->currentTokenInfo());
+		unsigned firstNum;
+		unsigned secondNum;
+		tie(firstNum, secondNum) = m_scanner->currentTokenInfo();
+		ElementaryTypeNameToken elemToken(m_scanner->currentToken(), firstNum, secondNum);
 		path.push_back(ASTNodeFactory(*this).createNode<ElementaryTypeNameExpression>(elemToken));
 		m_scanner->next();
 	}
@@ -1138,7 +1147,10 @@ ASTPointer<Expression> Parser::parsePrimaryExpression()
 		if (Token::isElementaryTypeName(token))
 		{
 			//used for casts
-			ElementaryTypeNameToken elementaryExpression(m_scanner->currentToken(), m_scanner->currentTokenInfo());
+			unsigned firstSize;
+			unsigned secondSize;
+			tie(firstSize, secondSize) = m_scanner->currentTokenInfo();
+			ElementaryTypeNameToken elementaryExpression(m_scanner->currentToken(), firstSize, secondSize);
 			expression = nodeFactory.createNode<ElementaryTypeNameExpression>(elementaryExpression);
 			m_scanner->next();
 		}
diff --git a/libsolidity/parsing/Scanner.cpp b/libsolidity/parsing/Scanner.cpp
index 9307b100..510d283e 100644
--- a/libsolidity/parsing/Scanner.cpp
+++ b/libsolidity/parsing/Scanner.cpp
@@ -381,12 +381,14 @@ Token::Value Scanner::scanSlash()
 void Scanner::scanToken()
 {
 	m_nextToken.literal.clear();
-	m_nextToken.extendedTokenInfo.clear();
+	m_nextToken.extendedTokenInfo = make_tuple(0, 0);
 	m_nextSkippedComment.literal.clear();
-	m_nextSkippedComment.extendedTokenInfo.clear();
+	m_nextSkippedComment.extendedTokenInfo = make_tuple(0, 0);
 
 	Token::Value token;
-	string tokenExtension;
+	// M and N are for the purposes of grabbing different type sizes
+	unsigned M;
+	unsigned N;
 	do
 	{
 		// Remember the position of the next token
@@ -554,7 +556,7 @@ void Scanner::scanToken()
 			break;
 		default:
 			if (isIdentifierStart(m_char))
-				tie(token, tokenExtension) = scanIdentifierOrKeyword();
+				tie(token, M, N) = scanIdentifierOrKeyword();
 			else if (isDecimalDigit(m_char))
 				token = scanNumber();
 			else if (skipWhitespace())
@@ -571,7 +573,7 @@ void Scanner::scanToken()
 	while (token == Token::Whitespace);
 	m_nextToken.location.end = sourcePos();
 	m_nextToken.token = token;
-	m_nextToken.extendedTokenInfo = tokenExtension;
+	m_nextToken.extendedTokenInfo = make_tuple(M,N);
 }
 
 bool Scanner::scanEscape()
@@ -709,7 +711,7 @@ Token::Value Scanner::scanNumber(char _charSeen)
 	return Token::Number;
 }
 
-tuple<Token::Value, string> Scanner::scanIdentifierOrKeyword()
+tuple<Token::Value, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
 {
 	solAssert(isIdentifierStart(m_char), "");
 	LiteralScope literal(this, LITERAL_TYPE_STRING);
diff --git a/libsolidity/parsing/Scanner.h b/libsolidity/parsing/Scanner.h
index fa1f118b..8dde922d 100644
--- a/libsolidity/parsing/Scanner.h
+++ b/libsolidity/parsing/Scanner.h
@@ -122,7 +122,7 @@ public:
 
 	SourceLocation currentLocation() const { return m_currentToken.location; }
 	std::string const& currentLiteral() const { return m_currentToken.literal; }
-	std::string const& currentTokenInfo() const { return m_currentToken.extendedTokenInfo; }
+	std::tuple<unsigned, unsigned> const& currentTokenInfo() const { return m_currentToken.extendedTokenInfo; }
 	///@}
 
 	///@{
@@ -161,7 +161,7 @@ private:
 		Token::Value token;
 		SourceLocation location;
 		std::string literal;
-		std::string extendedTokenInfo;
+		std::tuple<unsigned, unsigned> extendedTokenInfo;
 	};
 
 	///@{
@@ -192,7 +192,7 @@ private:
 
 	void scanDecimalDigits();
 	Token::Value scanNumber(char _charSeen = 0);
-	std::tuple<Token::Value, std::string> scanIdentifierOrKeyword();
+	std::tuple<Token::Value, unsigned, unsigned> scanIdentifierOrKeyword();
 
 	Token::Value scanString();
 	Token::Value scanSingleLineDocComment();
diff --git a/libsolidity/parsing/Token.cpp b/libsolidity/parsing/Token.cpp
index eaa65344..b07b5c1b 100644
--- a/libsolidity/parsing/Token.cpp
+++ b/libsolidity/parsing/Token.cpp
@@ -50,36 +50,65 @@ namespace dev
 namespace solidity
 {
 
-tuple<string, unsigned int, unsigned int> ElementaryTypeNameToken::parseDetails(Token::Value _baseType, string const& _details)
+void ElementaryTypeNameToken::parseDetails(Token::Value _baseType, unsigned const& _first, unsigned const& _second)
 {
 	solAssert(Token::isElementaryTypeName(_baseType), "");
 	string baseType = Token::toString(_baseType);
-	if (_details.length() == 0)
-		return make_tuple(baseType, 0, 0);
+	if (_first == 0 && _second == 0)
+	{
+		m_name = baseType;
+		m_firstNumber = _first;
+		m_secondNumber = _second;
+	}
 
 	if (baseType == "bytesM")
 	{
+		solAssert(_second == 0, "There should not be a second size argument to type bytesM.");
 		for (unsigned m = 1; m <= 32; m++)
-			if (to_string(m) == _details)
-				return make_tuple(baseType.substr(0, baseType.size()-1) + to_string(m), m, 0);
+			if (m == _first)
+			{
+				m_name = baseType.substr(0, baseType.size()-1) + to_string(_first);
+				m_firstNumber = _first;
+				m_secondNumber = _second;
+			}
 	}
 	else if (baseType == "uintM" || baseType == "intM")
 	{
+		solAssert(_second == 0, "There should not be a second size argument to type " + baseType + ".");
 		for (unsigned m = 8; m <= 256; m+=8)
-			if (to_string(m) == _details)
-				return make_tuple(baseType.substr(0, baseType.size()-1) + to_string(m), m, 0);
+			if (m == _first)
+			{
+				m_name = baseType.substr(0, baseType.size()-1) + to_string(_first);
+				m_firstNumber = _first;
+				m_secondNumber = _second;
+			}
 	}
 	else if (baseType == "ufixedMxN" || baseType == "fixedMxN")
 	{
 		for (unsigned m = 0; m <= 256; m+=8)
 			for (unsigned n = 8; m + n <= 256; n+=8)
-				if ((to_string(m) + "x" + to_string(n)) == _details)
-					return make_tuple(baseType.substr(0, baseType.size()-3) + to_string(m) + "x" + to_string(n), m, n);
+				if (m == _first && n == _second)
+				{
+					m_name = baseType.substr(0, baseType.size()-3) + 
+							to_string(_first) + 
+							"x" +
+							to_string(_second);
+					m_firstNumber = _first;
+					m_secondNumber = _second;
+				}
 	}
 	
-	BOOST_THROW_EXCEPTION(Error(Error::Type::TypeError) << 
-		errinfo_comment("Cannot create elementary type name token out of type " + baseType + " and size " + _details)
-	);	
+	if (m_name.empty())
+		BOOST_THROW_EXCEPTION(Error(Error::Type::TypeError) << 
+			errinfo_comment(
+				"Cannot create elementary type name token out of type " + 
+				baseType +
+				" and size(s) " +
+				to_string(_first) +
+				" and " +
+				to_string(_second)
+			)
+		);	
 }
 
 #define T(name, string, precedence) #name,
@@ -112,33 +141,68 @@ char const Token::m_tokenType[] =
 {
 	TOKEN_LIST(KT, KK)
 };
-tuple<Token::Value, string> Token::fromIdentifierOrKeyword(const string& _literal)
+unsigned Token::extractM(string _literal)
+{
+	try
+	{
+		unsigned short m = stoi(_literal.substr(_literal.find_first_of("0123456789")));
+		return m;
+	}
+	catch(out_of_range& e)
+	{
+		return 0;
+	}
+}
+pair<unsigned, unsigned> Token::extractMxN(string _literal)
+{
+	try
+	{
+		unsigned short m = stoi(_literal.substr(0, _literal.find_last_of("x") - 1));
+		unsigned short n = stoi(_literal.substr(_literal.find_last_of("x") + 1));
+		return make_pair(m, n);
+	}
+	catch (out_of_range& e)
+	{
+		return make_pair(0, 0);
+	}
+}
+tuple<Token::Value, unsigned short, unsigned short> Token::fromIdentifierOrKeyword(string const& _literal)
 {
-	string token = _literal;
-	string details;
-	if (_literal == "uintM" || _literal == "intM" || _literal == "fixedMxN" || _literal == "ufixedMxN" || _literal == "bytesM")
-		return make_pair(Token::Identifier, details);
 	if (_literal.find_first_of("0123456789") != string::npos)
 	{
 		string baseType = _literal.substr(0, _literal.find_first_of("0123456789"));
-		short m = stoi(_literal.substr(_literal.find_first_of("0123456789")));
 		if (baseType == "bytes")
 		{
-			details = (0 < m && m <= 32) ? to_string(m) : "";
-			token = details.empty() ? _literal : baseType + "M";
+			unsigned short m = extractM(_literal);
+			if (0 < m && m <= 32)
+				return make_tuple(Token::BytesM, m, 0);
+			return make_tuple(Token::Identifier, 0, 0);
 		}
 		else if (baseType == "uint" || baseType == "int")
 		{
-			details = (0 < m && m <= 256 && m % 8 == 0) ? to_string(m) : "";
-			token = details.empty() ? _literal : baseType + "M";
+			unsigned short m = extractM(_literal);			
+			if (0 < m && m <= 256 && m % 8 == 0)
+			{
+				if (baseType == "uint")
+					return make_tuple(Token::UIntM, m, 0);
+				else
+					return make_tuple(Token::IntM, m, 0);
+			}
+			return make_tuple(Token::Identifier, 0, 0);
 		}
 		else if (baseType == "ufixed" || baseType == "fixed")
 		{
-			m = stoi(to_string(m).substr(0, to_string(m).find_first_of("x") - 1));
-			short n = stoi(_literal.substr(_literal.find_last_of("x") + 1));
-			details = (0 < n + m && n + m <= 256 && ((n % 8 == 0) && (m % 8 == 0))) ? 
-						to_string(m) + "x" + to_string(n) : "";
-			token = details.empty() ? _literal : baseType + "MxN" ;
+			unsigned short m;
+			unsigned short n;
+			tie(m, n) = extractMxN(_literal);
+			if (0 < n + m && n + m <= 256 && ((n % 8 == 0) && (m % 8 == 0)))
+			{
+				if (baseType == "ufixed")
+					return make_tuple(Token::UFixedMxN, m, n);
+				else
+					return make_tuple(Token::FixedMxN, m, n);
+			}
+			return make_tuple(Token::Identifier, 0, 0);
 		}
 	}
 	// The following macros are used inside TOKEN_LIST and cause non-keyword tokens to be ignored
@@ -148,8 +212,8 @@ tuple<Token::Value, string> Token::fromIdentifierOrKeyword(const string& _litera
 	static const map<string, Token::Value> keywords({TOKEN_LIST(TOKEN, KEYWORD)});
 #undef KEYWORD
 #undef TOKEN
-	auto it = keywords.find(token);
-	return it == keywords.end() ? make_pair(Token::Identifier, details) : make_pair(it->second, details);
+	auto it = keywords.find(_literal);
+	return it == keywords.end() ? make_tuple(Token::Identifier, 0, 0) : make_tuple(it->second, 0, 0);
 }
 
 #undef KT
diff --git a/libsolidity/parsing/Token.h b/libsolidity/parsing/Token.h
index 55b801db..7193627f 100644
--- a/libsolidity/parsing/Token.h
+++ b/libsolidity/parsing/Token.h
@@ -189,19 +189,19 @@ namespace solidity
 	K(After, "after", 0)                                               \
 	/* type keywords*/                                                 \
 	K(Int, "int", 0)                                                   \
-	K(IntM, "intM", 0)                                                 \
+	T(IntM, "intM", 0)                                                 \
 	K(UInt, "uint", 0)                                                 \
-	K(UIntM, "uintM", 0)                                               \
+	T(UIntM, "uintM", 0)                                               \
 	K(Bytes, "bytes", 0)                                               \
-	K(BytesM, "bytesM", 0)                                             \
+	T(BytesM, "bytesM", 0)                                             \
 	K(Byte, "byte", 0)                                                 \
 	K(String, "string", 0)                                             \
 	K(Address, "address", 0)                                           \
 	K(Bool, "bool", 0)                                                 \
 	K(Fixed, "fixed", 0)                                               \
-	K(FixedMxN, "fixedMxN", 0)                                         \
+	T(FixedMxN, "fixedMxN", 0)                                         \
 	K(UFixed, "ufixed", 0)                                             \
-	K(UFixedMxN, "ufixedMxN", 0)                                       \
+	T(UFixedMxN, "ufixedMxN", 0)                                       \
 	T(TypesEnd, NULL, 0) /* used as type enum end marker */            \
 	\
 	/* Literals */                                                     \
@@ -302,9 +302,14 @@ public:
 		return m_precedence[tok];
 	}
 
-	static std::tuple<Token::Value, std::string> fromIdentifierOrKeyword(std::string const& _name);
+	static std::tuple<Token::Value, unsigned short, unsigned short> fromIdentifierOrKeyword(std::string const& _literal);
 
 private:
+	// extractM and extractMxN provide a safe way to extract numbers, 
+	// if out_of_range error is thrown, they returns 0s, therefore securing 
+	// the variable's identity as an identifier.
+	static unsigned extractM(std::string _literal);
+	static std::pair<unsigned, unsigned> extractMxN(std::string _literal);
 	static char const* const m_name[NUM_TOKENS];
 	static char const* const m_string[NUM_TOKENS];
 	static int8_t const m_precedence[NUM_TOKENS];
@@ -314,9 +319,9 @@ private:
 class ElementaryTypeNameToken
 {
 public:
-	ElementaryTypeNameToken(Token::Value _token, std::string const& _detail)
+	ElementaryTypeNameToken(Token::Value _token, unsigned const& _firstNumber, unsigned const& _secondNumber)
 	{
-		std::tie(m_name, m_firstNumber, m_secondNumber) = parseDetails(_token, _detail);
+		parseDetails(_token, _firstNumber, _secondNumber);
 		m_token = _token;
 	}
 
@@ -331,8 +336,8 @@ private:
 	std::string m_name;
 	unsigned int m_firstNumber;
 	unsigned int m_secondNumber;
-	/// throws if _details is malformed
-	std::tuple<std::string, unsigned int, unsigned int> parseDetails(Token::Value _baseType, std::string const& _details);
+	/// throws if type is not properly sized
+	void parseDetails(Token::Value _baseType, unsigned const& _first, unsigned const& _second);
 };
 
 }