aboutsummaryrefslogtreecommitdiffstats
path: root/libsolidity/parsing/Scanner.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'libsolidity/parsing/Scanner.cpp')
-rw-r--r--libsolidity/parsing/Scanner.cpp84
1 files changed, 53 insertions, 31 deletions
diff --git a/libsolidity/parsing/Scanner.cpp b/libsolidity/parsing/Scanner.cpp
index 6541f6c2..dbe1f389 100644
--- a/libsolidity/parsing/Scanner.cpp
+++ b/libsolidity/parsing/Scanner.cpp
@@ -243,22 +243,17 @@ bool Scanner::skipWhitespace()
return sourcePos() != startPosition;
}
-bool Scanner::skipWhitespaceExceptLF()
+void Scanner::skipWhitespaceExceptUnicodeLinebreak()
{
- int const startPosition = sourcePos();
- while (isWhiteSpace(m_char) && !isLineTerminator(m_char))
+ while (isWhiteSpace(m_char) && !isUnicodeLinebreak())
advance();
- // Return whether or not we skipped any characters.
- return sourcePos() != startPosition;
}
Token::Value Scanner::skipSingleLineComment()
{
- // The line terminator at the end of the line is not considered
- // to be part of the single-line comment; it is recognized
- // separately by the lexical grammar and becomes part of the
- // stream of input elements for the syntactic grammar
- while (!isLineTerminator(m_char))
+ // Line terminator is not part of the comment. If it is a
+ // non-ascii line terminator, it will result in a parser error.
+ while (!isUnicodeLinebreak())
if (!advance()) break;
return Token::Whitespace;
@@ -268,7 +263,9 @@ Token::Value Scanner::scanSingleLineDocComment()
{
LiteralScope literal(this, LITERAL_TYPE_COMMENT);
advance(); //consume the last '/' at ///
- skipWhitespaceExceptLF();
+
+ skipWhitespaceExceptUnicodeLinebreak();
+
while (!isSourcePastEndOfInput())
{
if (isLineTerminator(m_char))
@@ -287,6 +284,10 @@ Token::Value Scanner::scanSingleLineDocComment()
break; // next line is not a documentation comment, we are done
}
+ else if (isUnicodeLinebreak())
+ // Any line terminator that is not '\n' is considered to end the
+ // comment.
+ break;
addCommentLiteralChar(m_char);
advance();
}
@@ -321,6 +322,9 @@ Token::Value Scanner::scanMultiLineDocComment()
bool endFound = false;
bool charsAdded = false;
+ while (isWhiteSpace(m_char) && !isLineTerminator(m_char))
+ advance();
+
while (!isSourcePastEndOfInput())
{
//handle newlines in multline comments
@@ -372,7 +376,7 @@ Token::Value Scanner::scanSlash()
if (m_char == '/')
{
if (!advance()) /* double slash comment directly before EOS */
- return Token::Whitespace;
+ return Token::Whitespace;
else if (m_char == '/')
{
// doxygen style /// comment
@@ -390,24 +394,27 @@ Token::Value Scanner::scanSlash()
{
// doxygen style /** natspec comment
if (!advance()) /* slash star comment before EOS */
- return Token::Whitespace;
+ return Token::Illegal;
else if (m_char == '*')
{
advance(); //consume the last '*' at /**
- skipWhitespaceExceptLF();
- // special case of a closed normal multiline comment
- if (!m_source.isPastEndOfInput() && m_source.get(0) == '/')
- advance(); //skip the closing slash
- else // we actually have a multiline documentation comment
+ // "/**/"
+ if (m_char == '/')
{
- Token::Value comment;
- m_nextSkippedComment.location.start = firstSlashPosition;
- comment = scanMultiLineDocComment();
- m_nextSkippedComment.location.end = sourcePos();
- m_nextSkippedComment.token = comment;
+ advance(); //skip the closing slash
+ return Token::Whitespace;
}
- return Token::Whitespace;
+ // we actually have a multiline documentation comment
+ Token::Value comment;
+ m_nextSkippedComment.location.start = firstSlashPosition;
+ comment = scanMultiLineDocComment();
+ m_nextSkippedComment.location.end = sourcePos();
+ m_nextSkippedComment.token = comment;
+ if (comment == Token::Illegal)
+ return Token::Illegal;
+ else
+ return Token::Whitespace;
}
else
return skipMultiLineComment();
@@ -435,11 +442,6 @@ void Scanner::scanToken()
m_nextToken.location.start = sourcePos();
switch (m_char)
{
- case '\n':
- case ' ':
- case '\t':
- token = selectToken(Token::Whitespace);
- break;
case '"':
case '\'':
token = scanString();
@@ -675,18 +677,38 @@ bool Scanner::scanEscape()
if (!scanHexByte(c))
return false;
break;
+ default:
+ return false;
}
addLiteralChar(c);
return true;
}
+bool Scanner::isUnicodeLinebreak()
+{
+ if (0x0a <= m_char && m_char <= 0x0d)
+ // line feed, vertical tab, form feed, carriage return
+ return true;
+ else if (!m_source.isPastEndOfInput(1) && uint8_t(m_source.get(0)) == 0xc2 && uint8_t(m_source.get(1)) == 0x85)
+ // NEL - U+0085, C2 85 in utf8
+ return true;
+ else if (!m_source.isPastEndOfInput(2) && uint8_t(m_source.get(0)) == 0xe2 && uint8_t(m_source.get(1)) == 0x80 && (
+ uint8_t(m_source.get(2)) == 0xa8 || uint8_t(m_source.get(2)) == 0xa9
+ ))
+ // LS - U+2028, E2 80 A8 in utf8
+ // PS - U+2029, E2 80 A9 in utf8
+ return true;
+ else
+ return false;
+}
+
Token::Value Scanner::scanString()
{
char const quote = m_char;
advance(); // consume quote
LiteralScope literal(this, LITERAL_TYPE_STRING);
- while (m_char != quote && !isSourcePastEndOfInput() && !isLineTerminator(m_char))
+ while (m_char != quote && !isSourcePastEndOfInput() && !isUnicodeLinebreak())
{
char c = m_char;
advance();
@@ -710,7 +732,7 @@ Token::Value Scanner::scanHexString()
char const quote = m_char;
advance(); // consume quote
LiteralScope literal(this, LITERAL_TYPE_STRING);
- while (m_char != quote && !isSourcePastEndOfInput() && !isLineTerminator(m_char))
+ while (m_char != quote && !isSourcePastEndOfInput())
{
char c = m_char;
if (!scanHexByte(c))