From 3174f5eca34d8a86f3105fae93f333fdb5535e4d Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Tue, 18 Nov 2014 18:50:40 +0100 Subject: solidity scanner takes triple slash doc comments into account - Conditionally scanning for the documentation comments and gettings their contents. - Adding tests for this functionality of the scanner --- Scanner.h | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'Scanner.h') diff --git a/Scanner.h b/Scanner.h index 997365f3..fd48d569 100644 --- a/Scanner.h +++ b/Scanner.h @@ -111,19 +111,27 @@ public: }; Scanner() { reset(CharStream()); } - explicit Scanner(CharStream const& _source) { reset(_source); } + explicit Scanner(CharStream const& _source, bool _skipDocumentationComments = true) + { + reset(_source, _skipDocumentationComments); + } /// Resets the scanner as if newly constructed with _input as input. - void reset(CharStream const& _source); + void reset(CharStream const& _source, bool _skipDocumentationComments = true); /// Returns the next token and advances input. - Token::Value next(); + Token::Value next(bool _skipDocumentationComments = true); ///@{ ///@name Information about the current token /// Returns the current token - Token::Value getCurrentToken() { return m_current_token.token; } + Token::Value getCurrentToken(bool _skipDocumentationComments = true) + { + if (!_skipDocumentationComments) + next(_skipDocumentationComments); + return m_current_token.token; + } Location getCurrentLocation() const { return m_current_token.location; } std::string const& getCurrentLiteral() const { return m_current_token.literal; } ///@} @@ -172,7 +180,7 @@ private: bool scanHexByte(char& o_scannedByte); /// Scans a single JavaScript token. - void scanToken(); + void scanToken(bool _skipDocumentationComments = true); /// Skips all whitespace and @returns true if something was skipped. bool skipWhitespace(); @@ -184,6 +192,7 @@ private: Token::Value scanIdentifierOrKeyword(); Token::Value scanString(); + Token::Value scanDocumentationComment(); /// Scans an escape-sequence which is part of a string and adds the /// decoded character to the current literal. Returns true if a pattern -- cgit v1.2.3 From 4e6d3a38cd780457a6a3cfb480f880af802f661a Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Wed, 19 Nov 2014 02:02:30 +0100 Subject: fixing typo and alignment --- Scanner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Scanner.h') diff --git a/Scanner.h b/Scanner.h index fd48d569..402f1aea 100644 --- a/Scanner.h +++ b/Scanner.h @@ -179,7 +179,7 @@ private: bool scanHexByte(char& o_scannedByte); - /// Scans a single JavaScript token. + /// Scans a single Solidity token. void scanToken(bool _skipDocumentationComments = true); /// Skips all whitespace and @returns true if something was skipped. -- cgit v1.2.3 From 43961a552d294757f631e9b1a0b90017002fd196 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Wed, 19 Nov 2014 16:21:42 +0100 Subject: documentation comments are now always skipped but saved as special tokens at the Scanner --- Scanner.h | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) (limited to 'Scanner.h') diff --git a/Scanner.h b/Scanner.h index 402f1aea..23007fe1 100644 --- a/Scanner.h +++ b/Scanner.h @@ -111,31 +111,34 @@ public: }; Scanner() { reset(CharStream()); } - explicit Scanner(CharStream const& _source, bool _skipDocumentationComments = true) - { - reset(_source, _skipDocumentationComments); - } + explicit Scanner(CharStream const& _source) { reset(_source); } /// Resets the scanner as if newly constructed with _input as input. - void reset(CharStream const& _source, bool _skipDocumentationComments = true); + void reset(CharStream const& _source); - /// Returns the next token and advances input. - Token::Value next(bool _skipDocumentationComments = true); + /// Returns the next token and advances input. If called from reset() + /// and ScanToken() found a documentation token then next should be called + /// with _change_skipped_comment=true + Token::Value next(bool _change_skipped_comment = false); ///@{ ///@name Information about the current token /// Returns the current token - Token::Value getCurrentToken(bool _skipDocumentationComments = true) + Token::Value getCurrentToken() { - if (!_skipDocumentationComments) - next(_skipDocumentationComments); return m_current_token.token; } Location getCurrentLocation() const { return m_current_token.location; } std::string const& getCurrentLiteral() const { return m_current_token.literal; } ///@} + ///@{ + ///@name Information about the current comment token + Location getCurrentCommentLocation() const { return m_skipped_comment.location; } + std::string const& getCurrentCommentLiteral() const { return m_skipped_comment.literal; } + ///@} + ///@{ ///@name Information about the next token @@ -154,7 +157,7 @@ public: ///@} private: - // Used for the current and look-ahead token. + // Used for the current and look-ahead token and comments struct TokenDesc { Token::Value token; @@ -166,6 +169,7 @@ private: ///@name Literal buffer support inline void startNewLiteral() { m_next_token.literal.clear(); } inline void addLiteralChar(char c) { m_next_token.literal.push_back(c); } + inline void addCommentLiteralChar(char c) { m_next_skipped_comment.literal.push_back(c); } inline void dropLiteral() { m_next_token.literal.clear(); } inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); } ///@} @@ -179,8 +183,9 @@ private: bool scanHexByte(char& o_scannedByte); - /// Scans a single Solidity token. - void scanToken(bool _skipDocumentationComments = true); + /// Scans a single Solidity token. Returns true if the scanned token was + /// a skipped documentation comment. False in all other cases. + bool scanToken(); /// Skips all whitespace and @returns true if something was skipped. bool skipWhitespace(); @@ -203,6 +208,9 @@ private: int getSourcePos() { return m_source.getPos(); } bool isSourcePastEndOfInput() { return m_source.isPastEndOfInput(); } + TokenDesc m_skipped_comment; // desc for current skipped comment + TokenDesc m_next_skipped_comment; // desc for next skiped comment + TokenDesc m_current_token; // desc for current token (as returned by Next()) TokenDesc m_next_token; // desc for next token (one token look-ahead) -- cgit v1.2.3 From b4b0e37e7a105b952c3dd3e3dd939e0b7150809d Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 20 Nov 2014 22:08:16 +0100 Subject: styling fixes --- Scanner.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Scanner.h') diff --git a/Scanner.h b/Scanner.h index 23007fe1..0a6778ec 100644 --- a/Scanner.h +++ b/Scanner.h @@ -119,7 +119,7 @@ public: /// Returns the next token and advances input. If called from reset() /// and ScanToken() found a documentation token then next should be called /// with _change_skipped_comment=true - Token::Value next(bool _change_skipped_comment = false); + Token::Value next(bool _changeSkippedComment = false); ///@{ ///@name Information about the current token @@ -157,7 +157,7 @@ public: ///@} private: - // Used for the current and look-ahead token and comments + /// Used for the current and look-ahead token and comments struct TokenDesc { Token::Value token; -- cgit v1.2.3 From a93916b5f9df5a71c890150ad705147d040264df Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 20 Nov 2014 23:18:05 +0100 Subject: extra comments scanning test --- Scanner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Scanner.h') diff --git a/Scanner.h b/Scanner.h index 0a6778ec..94c67840 100644 --- a/Scanner.h +++ b/Scanner.h @@ -118,7 +118,7 @@ public: /// Returns the next token and advances input. If called from reset() /// and ScanToken() found a documentation token then next should be called - /// with _change_skipped_comment=true + /// with _changeSkippedComment=true Token::Value next(bool _changeSkippedComment = false); ///@{ -- cgit v1.2.3 From cda2532de6886a1b6735387b807490b5ff4556ae Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 20 Nov 2014 23:56:24 +0100 Subject: cleaning up the external interface of Scanner::next(). No special cases --- Scanner.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'Scanner.h') diff --git a/Scanner.h b/Scanner.h index 94c67840..5dfe7a33 100644 --- a/Scanner.h +++ b/Scanner.h @@ -116,10 +116,8 @@ public: /// Resets the scanner as if newly constructed with _input as input. void reset(CharStream const& _source); - /// Returns the next token and advances input. If called from reset() - /// and ScanToken() found a documentation token then next should be called - /// with _changeSkippedComment=true - Token::Value next(bool _changeSkippedComment = false); + /// Returns the next token and advances input + Token::Value next(); ///@{ ///@name Information about the current token -- cgit v1.2.3