solidity scanner takes triple slash doc comments into account

- Conditionally scanning for the documentation comments and gettings their
  contents.

- Adding tests for this functionality of the scanner

3 files changed, 44 insertions, 12 deletions

diff --git a/Scanner.cpp b/Scanner.cpp
index b13e52d7..382d07a9 100644
--- a/Scanner.cpp
+++ b/Scanner.cpp
@@ -102,13 +102,13 @@ int HexValue(char c)
 }
 } // end anonymous namespace
 
-void Scanner::reset(CharStream const& _source)
+void Scanner::reset(CharStream const& _source, bool _skipDocumentationComments)
 {
 	m_source = _source;
 	m_char = m_source.get();
 	skipWhitespace();
-	scanToken();
-	next();
+	scanToken(_skipDocumentationComments);
+	next(_skipDocumentationComments);
 }
 
 
@@ -134,10 +134,10 @@ bool Scanner::scanHexByte(char& o_scannedByte)
 // Ensure that tokens can be stored in a byte.
 BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
 
-Token::Value Scanner::next()
+Token::Value Scanner::next(bool _skipDocumentationComments)
 {
 	m_current_token = m_next_token;
-	scanToken();
+	scanToken(_skipDocumentationComments);
 	return m_current_token.token;
 }
 
@@ -171,6 +171,21 @@ Token::Value Scanner::skipSingleLineComment()
 	return Token::WHITESPACE;
 }
 
+// For the moment this function simply consumes a single line triple slash doc comment
+Token::Value Scanner::scanDocumentationComment()
+{
+	LiteralScope literal(this);
+	advance(); //consume the last '/'
+	while (!isSourcePastEndOfInput() && !IsLineTerminator(m_char))
+	{
+		char c = m_char;
+		advance();
+		addLiteralChar(c);
+	}
+	literal.Complete();
+	return Token::COMMENT_LITERAL;
+}
+
 Token::Value Scanner::skipMultiLineComment()
 {
 	if (asserts(m_char == '*'))
@@ -194,7 +209,7 @@ Token::Value Scanner::skipMultiLineComment()
 	return Token::ILLEGAL;
 }
 
-void Scanner::scanToken()
+void Scanner::scanToken(bool _skipDocumentationComments)
 {
 	m_next_token.literal.clear();
 	Token::Value token;
@@ -297,7 +312,14 @@ void Scanner::scanToken()
 			// /  // /* /=
 			advance();
 			if (m_char == '/')
-				token = skipSingleLineComment();
+			{
+				if (!advance()) /* double slash comment directly before EOS */
+					token = Token::WHITESPACE;
+				else if (!_skipDocumentationComments)
+					token = scanDocumentationComment();
+				else
+					token = skipSingleLineComment();
+			}
 			else if (m_char == '*')
 				token = skipMultiLineComment();
 			else if (m_char == '=')
diff --git a/Scanner.h b/Scanner.h
index 997365f3..fd48d569 100644
--- a/Scanner.h
+++ b/Scanner.h
@@ -111,19 +111,27 @@ public:
 	};
 
 	Scanner() { reset(CharStream()); }
-	explicit Scanner(CharStream const& _source) { reset(_source); }
+	explicit Scanner(CharStream const& _source, bool _skipDocumentationComments = true)
+	{
+		reset(_source, _skipDocumentationComments);
+	}
 
 	/// Resets the scanner as if newly constructed with _input as input.
-	void reset(CharStream const& _source);
+	void reset(CharStream const& _source, bool _skipDocumentationComments = true);
 
 	/// Returns the next token and advances input.
-	Token::Value next();
+	Token::Value next(bool _skipDocumentationComments = true);
 
 	///@{
 	///@name Information about the current token
 
 	/// Returns the current token
-	Token::Value getCurrentToken() { return m_current_token.token; }
+	Token::Value getCurrentToken(bool _skipDocumentationComments = true)
+	{
+		if (!_skipDocumentationComments)
+			next(_skipDocumentationComments);
+		return m_current_token.token;
+	}
 	Location getCurrentLocation() const { return m_current_token.location; }
 	std::string const& getCurrentLiteral() const { return m_current_token.literal; }
 	///@}
@@ -172,7 +180,7 @@ private:
 	bool scanHexByte(char& o_scannedByte);
 
 	/// Scans a single JavaScript token.
-	void scanToken();
+	void scanToken(bool _skipDocumentationComments = true);
 
 	/// Skips all whitespace and @returns true if something was skipped.
 	bool skipWhitespace();
@@ -184,6 +192,7 @@ private:
 	Token::Value scanIdentifierOrKeyword();
 
 	Token::Value scanString();
+	Token::Value scanDocumentationComment();
 
 	/// Scans an escape-sequence which is part of a string and adds the
 	/// decoded character to the current literal. Returns true if a pattern
diff --git a/Token.h b/Token.h
index 67971c3d..39a84bcc 100644
--- a/Token.h
+++ b/Token.h
@@ -281,6 +281,7 @@ namespace solidity
 	K(FALSE_LITERAL, "false", 0)                                       \
 	T(NUMBER, NULL, 0)                                                 \
 	T(STRING_LITERAL, NULL, 0)                                         \
+	T(COMMENT_LITERAL, NULL, 0)                                         \
 	\
 	/* Identifiers (not keywords or future reserved words). */         \
 	T(IDENTIFIER, NULL, 0)                                             \