Merge pull request #5605 from ethereum/cmdlineTests

Extend capabilities of the generic commandline test.

19 files changed, 25 insertions, 13 deletions

diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh
index fdc9fbe3..7b2b528b 100755
--- a/test/cmdlineTests.sh
+++ b/test/cmdlineTests.sh
@@ -110,13 +110,15 @@ printTask "Testing unknown options..."
 
 # General helper function for testing SOLC behaviour, based on file name, compile opts, exit code, stdout and stderr.
 # An failure is expected.
-test_solc_file_input_failures() {
+test_solc_behaviour() {
     local filename="${1}"
     local solc_args="${2}"
     local stdout_expected="${3}"
-    local stderr_expected="${4}"
+    local exit_code_expected="${4}"
+    local stderr_expected="${5}"
     local stdout_path=`mktemp`
     local stderr_path=`mktemp`
+    if [[ "$exit_code_expected" = "" ]]; then exit_code_expected="0"; fi
 
     set +e
     "$SOLC" "${filename}" ${solc_args} 1>$stdout_path 2>$stderr_path
@@ -126,14 +128,14 @@ test_solc_file_input_failures() {
     sed -i -e '/^Warning: This is a pre-release compiler version, please do not use it in production./d' "$stderr_path"
     sed -i -e 's/ Consider adding "pragma .*$//' "$stderr_path"
 
-    if [[ $exitCode -eq 0 ]]; then
-        printError "Incorrect exit code. Expected failure (non-zero) but got success (0)."
+    if [[ $exitCode -ne "$exit_code_expected" ]]; then
+        printError "Incorrect exit code. Expected $exit_code_expected but got $exitCode."
         rm -f $stdout_path $stderr_path
         exit 1
     fi
 
     if [[ "$(cat $stdout_path)" != "${stdout_expected}" ]]; then
-        printError "Incorrect output on stderr received. Expected:"
+        printError "Incorrect output on stdout received. Expected:"
         echo -e "${stdout_expected}"
 
         printError "But got:"
@@ -156,22 +158,26 @@ test_solc_file_input_failures() {
 }
 
 printTask "Testing passing files that are not found..."
-test_solc_file_input_failures "file_not_found.sol" "" "" "\"file_not_found.sol\" is not found."
+test_solc_behaviour "file_not_found.sol" "" "" 1 "\"file_not_found.sol\" is not found."
 
 printTask "Testing passing files that are not files..."
-test_solc_file_input_failures "." "" "" "\".\" is not a valid file."
+test_solc_behaviour "." "" "" 1 "\".\" is not a valid file."
 
 printTask "Testing passing empty remappings..."
-test_solc_file_input_failures "${0}" "=/some/remapping/target" "" "Invalid remapping: \"=/some/remapping/target\"."
-test_solc_file_input_failures "${0}" "ctx:=/some/remapping/target" "" "Invalid remapping: \"ctx:=/some/remapping/target\"."
+test_solc_behaviour "${0}" "=/some/remapping/target" "" 1 "Invalid remapping: \"=/some/remapping/target\"."
+test_solc_behaviour "${0}" "ctx:=/some/remapping/target" "" 1 "Invalid remapping: \"ctx:=/some/remapping/target\"."
 
-printTask "Testing passing location printing..."
+printTask "Running general commandline tests..."
 (
-cd "$REPO_ROOT"/test/cmdlineErrorReports/
+cd "$REPO_ROOT"/test/cmdlineTests/
 for file in *.sol
 do
-    ret=`cat $file.ref`
-    test_solc_file_input_failures "$file" "" "" "$ret"
+    args=$(cat $file.args 2>/dev/null || true)
+    stdout=$(cat $file.stdout 2>/dev/null || true)
+    exitCode=$(cat $file.exit 2>/dev/null || true)
+    err=$(cat $file.err 2>/dev/null || true)
+    printTask " - $file"
+    test_solc_behaviour "$file" "$args" "$stdout" "$exitCode" "$err"
 done
 )
 
diff --git a/test/cmdlineErrorReports/too_long_line.sol b/test/cmdlineTests/too_long_line.sol
index 7df1057a..7df1057a 100644
--- a/test/cmdlineErrorReports/too_long_line.sol
+++ b/test/cmdlineTests/too_long_line.sol
diff --git a/test/cmdlineErrorReports/too_long_line.sol.ref b/test/cmdlineTests/too_long_line.sol.err
index 55cd1935..55cd1935 100644
--- a/test/cmdlineErrorReports/too_long_line.sol.ref
+++ b/test/cmdlineTests/too_long_line.sol.err
diff --git a/test/cmdlineTests/too_long_line.sol.exit b/test/cmdlineTests/too_long_line.sol.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/test/cmdlineTests/too_long_line.sol.exit
@@ -0,0 +1 @@
+1
diff --git a/test/cmdlineErrorReports/too_long_line_both_sides_short.sol b/test/cmdlineTests/too_long_line_both_sides_short.sol
index 062f0292..062f0292 100644
--- a/test/cmdlineErrorReports/too_long_line_both_sides_short.sol
+++ b/test/cmdlineTests/too_long_line_both_sides_short.sol
diff --git a/test/cmdlineErrorReports/too_long_line_both_sides_short.sol.ref b/test/cmdlineTests/too_long_line_both_sides_short.sol.err
index 9a5ebfba..9a5ebfba 100644
--- a/test/cmdlineErrorReports/too_long_line_both_sides_short.sol.ref
+++ b/test/cmdlineTests/too_long_line_both_sides_short.sol.err
diff --git a/test/cmdlineTests/too_long_line_both_sides_short.sol.exit b/test/cmdlineTests/too_long_line_both_sides_short.sol.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/test/cmdlineTests/too_long_line_both_sides_short.sol.exit
@@ -0,0 +1 @@
+1
diff --git a/test/cmdlineErrorReports/too_long_line_edge_in.sol b/test/cmdlineTests/too_long_line_edge_in.sol
index 6f181c83..6f181c83 100644
--- a/test/cmdlineErrorReports/too_long_line_edge_in.sol
+++ b/test/cmdlineTests/too_long_line_edge_in.sol
diff --git a/test/cmdlineErrorReports/too_long_line_edge_in.sol.ref b/test/cmdlineTests/too_long_line_edge_in.sol.err
index ad3b7805..ad3b7805 100644
--- a/test/cmdlineErrorReports/too_long_line_edge_in.sol.ref
+++ b/test/cmdlineTests/too_long_line_edge_in.sol.err
diff --git a/test/cmdlineTests/too_long_line_edge_in.sol.exit b/test/cmdlineTests/too_long_line_edge_in.sol.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/test/cmdlineTests/too_long_line_edge_in.sol.exit
@@ -0,0 +1 @@
+1
diff --git a/test/cmdlineErrorReports/too_long_line_edge_out.sol b/test/cmdlineTests/too_long_line_edge_out.sol
index 29d3cee6..29d3cee6 100644
--- a/test/cmdlineErrorReports/too_long_line_edge_out.sol
+++ b/test/cmdlineTests/too_long_line_edge_out.sol
diff --git a/test/cmdlineErrorReports/too_long_line_edge_out.sol.ref b/test/cmdlineTests/too_long_line_edge_out.sol.err
index d8495c11..d8495c11 100644
--- a/test/cmdlineErrorReports/too_long_line_edge_out.sol.ref
+++ b/test/cmdlineTests/too_long_line_edge_out.sol.err
diff --git a/test/cmdlineTests/too_long_line_edge_out.sol.exit b/test/cmdlineTests/too_long_line_edge_out.sol.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/test/cmdlineTests/too_long_line_edge_out.sol.exit
@@ -0,0 +1 @@
+1
diff --git a/test/cmdlineErrorReports/too_long_line_left_short.sol b/test/cmdlineTests/too_long_line_left_short.sol
index 2accfcce..2accfcce 100644
--- a/test/cmdlineErrorReports/too_long_line_left_short.sol
+++ b/test/cmdlineTests/too_long_line_left_short.sol
diff --git a/test/cmdlineErrorReports/too_long_line_left_short.sol.ref b/test/cmdlineTests/too_long_line_left_short.sol.err
index 00b6be5c..00b6be5c 100644
--- a/test/cmdlineErrorReports/too_long_line_left_short.sol.ref
+++ b/test/cmdlineTests/too_long_line_left_short.sol.err
diff --git a/test/cmdlineTests/too_long_line_left_short.sol.exit b/test/cmdlineTests/too_long_line_left_short.sol.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/test/cmdlineTests/too_long_line_left_short.sol.exit
@@ -0,0 +1 @@
+1
diff --git a/test/cmdlineErrorReports/too_long_line_right_short.sol b/test/cmdlineTests/too_long_line_right_short.sol
index 936b3961..936b3961 100644
--- a/test/cmdlineErrorReports/too_long_line_right_short.sol
+++ b/test/cmdlineTests/too_long_line_right_short.sol
diff --git a/test/cmdlineErrorReports/too_long_line_right_short.sol.ref b/test/cmdlineTests/too_long_line_right_short.sol.err
index 88072d95..88072d95 100644
--- a/test/cmdlineErrorReports/too_long_line_right_short.sol.ref
+++ b/test/cmdlineTests/too_long_line_right_short.sol.err
diff --git a/test/cmdlineTests/too_long_line_right_short.sol.exit b/test/cmdlineTests/too_long_line_right_short.sol.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/test/cmdlineTests/too_long_line_right_short.sol.exit
@@ -0,0 +1 @@
+1