From 7fa892eca93cc1b3fd75eb9341c11f0a471970d9 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Wed, 14 Mar 2018 19:15:48 +0100 Subject: Add interactive test tool isoltest. --- test/tools/isoltest.cpp | 346 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 test/tools/isoltest.cpp (limited to 'test/tools/isoltest.cpp') diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp new file mode 100644 index 00000000..668481cf --- /dev/null +++ b/test/tools/isoltest.cpp @@ -0,0 +1,346 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +using namespace dev; +using namespace dev::solidity; +using namespace dev::solidity::test; +using namespace dev::solidity::test::formatting; +using namespace std; +namespace po = boost::program_options; +namespace fs = boost::filesystem; + +struct SyntaxTestStats +{ + int successCount; + int runCount; + operator bool() const { return successCount == runCount; } +}; + +class SyntaxTestTool +{ +public: + SyntaxTestTool(string const& _name, fs::path const& _path, bool _formatted): + m_formatted(_formatted), m_name(_name), m_path(_path) + {} + + enum class Result + { + Success, + Failure, + ParserError, + InputOutputError + }; + + Result process(); + + static SyntaxTestStats processPath( + fs::path const& _basepath, + fs::path const& _path, + bool const _formatted + ); + + static string editor; +private: + enum class Request + { + Skip, + Rerun, + Quit + }; + + Request handleResponse(bool const _parserError); + + void printContract() const; + + bool const m_formatted; + string const m_name; + fs::path const m_path; + unique_ptr m_test; +}; + +string SyntaxTestTool::editor; + +void SyntaxTestTool::printContract() const +{ + stringstream stream(m_test->source()); + string line; + while (getline(stream, line)) + cout << " " << line << endl; + cout << endl; +} + +SyntaxTestTool::Result SyntaxTestTool::process() +{ + bool success; + bool parserError = false; + std::stringstream outputMessages; + + (FormattedScope(cout, m_formatted, {BOLD}) << m_name << ": ").flush(); + + try + { + m_test = unique_ptr(new SyntaxTest(m_path.string())); + } + catch (std::exception const& _e) + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << "cannot read test: " << _e.what() << endl; + return Result::InputOutputError; + } + + try + { + success = m_test->run(outputMessages, " ", m_formatted); + } + catch (...) + { + success = false; + parserError = true; + } + + if (success) + { + FormattedScope(cout, m_formatted, {BOLD, GREEN}) << "OK" << endl; + return Result::Success; + } + else + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl; + + FormattedScope(cout, m_formatted, {BOLD, CYAN}) << " Contract:" << endl; + printContract(); + + if (parserError) + { + cout << " "; + FormattedScope(cout, m_formatted, {INVERSE, RED}) << "Parsing failed:" << endl; + m_test->printErrorList(cout, m_test->compilerErrors(), " ", true, true, m_formatted); + cout << endl; + return Result::ParserError; + } + else + { + cout << outputMessages.str() << endl; + return Result::Failure; + } + } +} + +SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _parserError) +{ + if (_parserError) + cout << "(e)dit/(s)kip/(q)uit? "; + else + cout << "(e)dit/(u)pdate expectations/(s)kip/(q)uit? "; + cout.flush(); + + while (true) + { + switch(readStandardInputChar()) + { + case 's': + cout << endl; + return Request::Skip; + case 'u': + if (_parserError) + break; + else + { + cout << endl; + ofstream file(m_path.string(), ios::trunc); + file << m_test->source(); + file << "// ----" << endl; + if (!m_test->errorList().empty()) + m_test->printErrorList(file, m_test->errorList(), "// ", false, false, false); + return Request::Rerun; + } + case 'e': + cout << endl << endl; + if (system((editor + " \"" + m_path.string() + "\"").c_str())) + cerr << "Error running editor command." << endl << endl; + return Request::Rerun; + case 'q': + cout << endl; + return Request::Quit; + default: + break; + } + } +} + + +SyntaxTestStats SyntaxTestTool::processPath( + fs::path const& _basepath, + fs::path const& _path, + bool const _formatted +) +{ + std::queue paths; + paths.push(_path); + int successCount = 0; + int runCount = 0; + + while (!paths.empty()) + { + auto currentPath = paths.front(); + + fs::path fullpath = _basepath / currentPath; + if (fs::is_directory(fullpath)) + { + paths.pop(); + for (auto const& entry: boost::iterator_range( + fs::directory_iterator(fullpath), + fs::directory_iterator() + )) + paths.push(currentPath / entry.path().filename()); + } + else + { + SyntaxTestTool testTool(currentPath.string(), fullpath, _formatted); + ++runCount; + auto result = testTool.process(); + + switch(result) + { + case Result::Failure: + case Result::ParserError: + switch(testTool.handleResponse(result == Result::ParserError)) + { + case Request::Quit: + return { successCount, runCount }; + case Request::Rerun: + cout << "Re-running test case..." << endl; + --runCount; + break; + case Request::Skip: + paths.pop(); + break; + } + break; + case Result::Success: + paths.pop(); + ++successCount; + break; + default: + // non-recoverable error; continue with next test case + paths.pop(); + break; + } + } + } + + return { successCount, runCount }; + +} + +int main(int argc, char *argv[]) +{ + if (getenv("EDITOR")) + SyntaxTestTool::editor = getenv("EDITOR"); + + fs::path testPath; + bool formatted = true; + po::options_description options( + R"(isoltest, tool for interactively managing test contracts. +Usage: isoltest [Options] --testpath path +Interactively validates test contracts. + +Allowed options)", + po::options_description::m_default_line_length, + po::options_description::m_default_line_length - 23); + options.add_options() + ("help", "Show this help screen.") + ("testpath", po::value(&testPath), "path to test files") + ("no-color", "don't use colors") + ("editor", po::value(&SyntaxTestTool::editor), "editor for opening contracts"); + + po::variables_map arguments; + try + { + po::command_line_parser cmdLineParser(argc, argv); + cmdLineParser.options(options); + po::store(cmdLineParser.run(), arguments); + + if (arguments.count("help")) + { + cout << options << endl; + return 0; + } + + if (arguments.count("no-color")) + formatted = false; + + po::notify(arguments); + } + catch (po::error const& _exception) + { + cerr << _exception.what() << endl; + return 1; + } + + if (testPath.empty()) + { + auto const searchPath = + { + fs::current_path() / ".." / ".." / ".." / "test", + fs::current_path() / ".." / ".." / "test", + fs::current_path() / ".." / "test", + fs::current_path() / "test", + fs::current_path() + }; + for (auto const& basePath : searchPath) + { + fs::path syntaxTestPath = basePath / "libsolidity" / "syntaxTests"; + if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath)) + { + testPath = basePath; + break; + } + } + } + + fs::path syntaxTestPath = testPath / "libsolidity" / "syntaxTests"; + + if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath)) + { + auto stats = SyntaxTestTool::processPath(testPath / "libsolidity", "syntaxTests", formatted); + + cout << endl << "Summary: "; + FormattedScope(cout, formatted, {BOLD, stats ? GREEN : RED}) << + stats.successCount << "/" << stats.runCount; + cout << " tests successful." << endl; + + return stats ? 0 : 1; + } + else + { + cerr << "Test path not found. Use the --testpath argument." << endl; + return 1; + } +} -- cgit v1.2.3 From ea8d5f8afc8cdd886be70efc8f49163af0675014 Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 15 Mar 2018 18:20:06 +0100 Subject: Use /usr/bin/editor if exists. --- test/tools/isoltest.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test/tools/isoltest.cpp') diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index 668481cf..b71f14b9 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -263,6 +263,8 @@ int main(int argc, char *argv[]) { if (getenv("EDITOR")) SyntaxTestTool::editor = getenv("EDITOR"); + else if (fs::exists("/usr/bin/editor")) + SyntaxTestTool::editor = "/usr/bin/editor"; fs::path testPath; bool formatted = true; -- cgit v1.2.3 From e68c19c47b03eebb9af528bf2b03bb0086484c63 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 16 Mar 2018 11:40:03 +0100 Subject: Only consider files ending with .sol and not starting with ~ in syntax tests. --- test/tools/isoltest.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'test/tools/isoltest.cpp') diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index 668481cf..5ad3bfb5 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -218,7 +219,8 @@ SyntaxTestStats SyntaxTestTool::processPath( fs::directory_iterator(fullpath), fs::directory_iterator() )) - paths.push(currentPath / entry.path().filename()); + if (fs::is_directory(entry.path()) || SyntaxTest::isTestFilename(entry.path().filename())) + paths.push(currentPath / entry.path().filename()); } else { -- cgit v1.2.3 From 6f9644add18b27363a423e2c7ccb0e578ba800bc Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Tue, 3 Apr 2018 12:05:26 +0200 Subject: SyntaxTests: extend syntax tests and isoltest to support parser errors and compiler exceptions. --- test/tools/isoltest.cpp | 65 +++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 26 deletions(-) (limited to 'test/tools/isoltest.cpp') diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index 5efec421..07df8f60 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -55,8 +55,8 @@ public: { Success, Failure, - ParserError, - InputOutputError + InputOutputError, + Exception }; Result process(); @@ -76,7 +76,7 @@ private: Quit }; - Request handleResponse(bool const _parserError); + Request handleResponse(bool const _exception); void printContract() const; @@ -100,7 +100,6 @@ void SyntaxTestTool::printContract() const SyntaxTestTool::Result SyntaxTestTool::process() { bool success; - bool parserError = false; std::stringstream outputMessages; (FormattedScope(cout, m_formatted, {BOLD}) << m_name << ": ").flush(); @@ -119,10 +118,35 @@ SyntaxTestTool::Result SyntaxTestTool::process() { success = m_test->run(outputMessages, " ", m_formatted); } - catch (...) + catch(CompilerError const& _e) { - success = false; - parserError = true; + FormattedScope(cout, m_formatted, {BOLD, RED}) << + "Exception: " << SyntaxTest::errorMessage(_e) << endl; + return Result::Exception; + } + catch(InternalCompilerError const& _e) + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << + "InternalCompilerError: " << SyntaxTest::errorMessage(_e) << endl; + return Result::Exception; + } + catch(FatalError const& _e) + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << + "FatalError: " << SyntaxTest::errorMessage(_e) << endl; + return Result::Exception; + } + catch(UnimplementedFeatureError const& _e) + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << + "UnimplementedFeatureError: " << SyntaxTest::errorMessage(_e) << endl; + return Result::Exception; + } + catch(...) + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << + "Unknown Exception" << endl; + return Result::Exception; } if (success) @@ -137,25 +161,14 @@ SyntaxTestTool::Result SyntaxTestTool::process() FormattedScope(cout, m_formatted, {BOLD, CYAN}) << " Contract:" << endl; printContract(); - if (parserError) - { - cout << " "; - FormattedScope(cout, m_formatted, {INVERSE, RED}) << "Parsing failed:" << endl; - m_test->printErrorList(cout, m_test->compilerErrors(), " ", true, true, m_formatted); - cout << endl; - return Result::ParserError; - } - else - { - cout << outputMessages.str() << endl; - return Result::Failure; - } + cout << outputMessages.str() << endl; + return Result::Failure; } } -SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _parserError) +SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception) { - if (_parserError) + if (_exception) cout << "(e)dit/(s)kip/(q)uit? "; else cout << "(e)dit/(u)pdate expectations/(s)kip/(q)uit? "; @@ -169,7 +182,7 @@ SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _parserError) cout << endl; return Request::Skip; case 'u': - if (_parserError) + if (_exception) break; else { @@ -178,7 +191,7 @@ SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _parserError) file << m_test->source(); file << "// ----" << endl; if (!m_test->errorList().empty()) - m_test->printErrorList(file, m_test->errorList(), "// ", false, false, false); + m_test->printErrorList(file, m_test->errorList(), "// ", false); return Request::Rerun; } case 'e': @@ -231,8 +244,8 @@ SyntaxTestStats SyntaxTestTool::processPath( switch(result) { case Result::Failure: - case Result::ParserError: - switch(testTool.handleResponse(result == Result::ParserError)) + case Result::Exception: + switch(testTool.handleResponse(result == Result::Exception)) { case Request::Quit: return { successCount, runCount }; -- cgit v1.2.3 From f03695731b9b36cd4014620b7b63556a69b4e952 Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 6 Apr 2018 18:37:01 +0200 Subject: Add source locations to syntax test expectations. --- test/tools/isoltest.cpp | 71 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 19 deletions(-) (limited to 'test/tools/isoltest.cpp') diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index 07df8f60..7a147bd0 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -55,7 +55,6 @@ public: { Success, Failure, - InputOutputError, Exception }; @@ -90,11 +89,53 @@ string SyntaxTestTool::editor; void SyntaxTestTool::printContract() const { - stringstream stream(m_test->source()); - string line; - while (getline(stream, line)) - cout << " " << line << endl; - cout << endl; + if (m_formatted) + { + string const& source = m_test->source(); + if (source.empty()) + return; + + std::vector sourceFormatting(source.length(), formatting::RESET); + for (auto const& error: m_test->errorList()) + if (error.locationStart >= 0 && error.locationEnd >= 0) + { + assert(static_cast(error.locationStart) < source.length()); + assert(static_cast(error.locationEnd) < source.length()); + bool isWarning = error.type == "Warning"; + for (int i = error.locationStart; i < error.locationEnd; i++) + if (isWarning) + { + if (sourceFormatting[i] == formatting::RESET) + sourceFormatting[i] = formatting::ORANGE_BACKGROUND; + } + else + sourceFormatting[i] = formatting::RED_BACKGROUND; + } + + cout << " " << sourceFormatting.front() << source.front(); + for (size_t i = 1; i < source.length(); i++) + { + if (sourceFormatting[i] != sourceFormatting[i - 1]) + cout << sourceFormatting[i]; + if (source[i] != '\n') + cout << source[i]; + else + { + cout << formatting::RESET << endl; + if (i + 1 < source.length()) + cout << " " << sourceFormatting[i]; + } + } + cout << formatting::RESET << endl; + } + else + { + stringstream stream(m_test->source()); + string line; + while (getline(stream, line)) + cout << " " << line << endl; + cout << endl; + } } SyntaxTestTool::Result SyntaxTestTool::process() @@ -107,15 +148,6 @@ SyntaxTestTool::Result SyntaxTestTool::process() try { m_test = unique_ptr(new SyntaxTest(m_path.string())); - } - catch (std::exception const& _e) - { - FormattedScope(cout, m_formatted, {BOLD, RED}) << "cannot read test: " << _e.what() << endl; - return Result::InputOutputError; - } - - try - { success = m_test->run(outputMessages, " ", m_formatted); } catch(CompilerError const& _e) @@ -142,6 +174,11 @@ SyntaxTestTool::Result SyntaxTestTool::process() "UnimplementedFeatureError: " << SyntaxTest::errorMessage(_e) << endl; return Result::Exception; } + catch (std::exception const& _e) + { + FormattedScope(cout, m_formatted, {BOLD, RED}) << "Exception: " << _e.what() << endl; + return Result::Exception; + } catch(...) { FormattedScope(cout, m_formatted, {BOLD, RED}) << @@ -262,10 +299,6 @@ SyntaxTestStats SyntaxTestTool::processPath( paths.pop(); ++successCount; break; - default: - // non-recoverable error; continue with next test case - paths.pop(); - break; } } } -- cgit v1.2.3