Merge pull request #2405 from ewanmellor/testability-and-build-improv…

…ements

Testability and build improvements

.travis.yml

@@ -202,4 +202,9 @@ before_install:
   - f="./.travis/before-install-$TRAVIS_OS_NAME-$TARGET.sh"; ! [ -x "$f" ] || "$f"
 
 script:
-  - cd runtime-testsuite; travis_wait 40 ../.travis/run-tests-$TARGET.sh
+  - |
+    cd runtime-testsuite;
+    travis_wait 40 ../.travis/run-tests-$TARGET.sh;
+    rc=$?;
+    cat target/surefire-reports/*.dumpstream || true;
+    exit $rc

.travis/run-tests-cpp.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euo pipefail
+
 if [ $GROUP == "LEXER" ]; then
     mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test 
 elif [ $GROUP == "PARSER" ]; then

.travis/run-tests-csharp.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+set -euo pipefail
+
 mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* test

.travis/run-tests-dotnet.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 
+set -euo pipefail
+
 # we need to build the runtime before test run, since we used "--no-dependencies"
 # when we call dotnet cli for restore and build, in order to speed up
 
 dotnet restore ../runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
-dotnet build -c Release ../runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
+dotnet build -c Release -f netstandard1.3 ../runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
 
 # call test
 

.travis/run-tests-go.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+set -euo pipefail
+
 mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=go.* test

.travis/run-tests-java.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euo pipefail
+
 mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=java.* test
 cd ../tool-testsuite
 mvn test

.travis/run-tests-javascript.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+set -euo pipefail
+
 mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=node.* test

.travis/run-tests-python2.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+set -euo pipefail
+
 mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=python2.* test

.travis/run-tests-python3.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+set -euo pipefail
+
 mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=python3.* test

pom.xml

@@ -138,6 +138,8 @@
 					<filesets>
 						<fileset>
 							<directory>runtime/Swift/.build</directory>
+						</fileset>
+						<fileset>
 							<directory>runtime/Swift/Tests/Antlr4Tests/gen</directory>
 						</fileset>
 					</filesets>

runtime-testsuite/pom.xml

@@ -92,6 +92,11 @@
 			</resource>
 			<resource>
 				<directory>../runtime</directory>
+				<excludes>
+					<exclude>**/.build/**</exclude>
+					<exclude>**/target/**</exclude>
+					<exclude>Swift/*.xcodeproj/**</exclude>
+				</excludes>
 			</resource>
 		</resources>
 		<plugins>

runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java

@@ -31,6 +31,8 @@
 import java.util.List;
 
 import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.fail;
+import static junit.framework.TestCase.failNotEquals;
 import static org.junit.Assume.assumeFalse;
 
 /** This class represents a single runtime test. It pulls data from
@@ -161,14 +163,7 @@ public void testParser(RuntimeTestDescriptor descriptor) throws Exception {
 		                                   descriptor.getInput(),
 		                                   descriptor.showDiagnosticErrors()
 		                                  );
-		if ( delegate instanceof SpecialRuntimeTestAssert ) {
-			((SpecialRuntimeTestAssert)delegate).assertEqualStrings(descriptor.getErrors(), delegate.getParseErrors());
-			((SpecialRuntimeTestAssert)delegate).assertEqualStrings(descriptor.getOutput(), found);
-		}
-		else {
-			assertEquals(descriptor.getErrors(), delegate.getParseErrors());
-			assertEquals(descriptor.getOutput(), found);
-		}
+		assertCorrectOutput(descriptor, delegate, found);
 	}
 
 	public void testLexer(RuntimeTestDescriptor descriptor) throws Exception {
@@ -202,16 +197,7 @@ public void testLexer(RuntimeTestDescriptor descriptor) throws Exception {
 		grammar = grammarST.render();
 
 		String found = delegate.execLexer(grammarName+".g4", grammar, grammarName, descriptor.getInput(), descriptor.showDFA());
-		if ( delegate instanceof SpecialRuntimeTestAssert ) {
-			((SpecialRuntimeTestAssert)delegate).assertEqualStrings(descriptor.getOutput(), found);
-			((SpecialRuntimeTestAssert)delegate).assertEqualStrings(descriptor.getANTLRToolErrors(), delegate.getANTLRToolErrors());
-			((SpecialRuntimeTestAssert)delegate).assertEqualStrings(descriptor.getErrors(), delegate.getParseErrors());
-		}
-		else {
-			assertEquals(descriptor.getOutput(), found);
-			assertEquals(descriptor.getANTLRToolErrors(), delegate.getANTLRToolErrors());
-			assertEquals(descriptor.getErrors(), delegate.getParseErrors());
-		}
+		assertCorrectOutput(descriptor, delegate, found);
 	}
 
 	/** Write a grammar to tmpdir and run antlr */
@@ -313,4 +299,66 @@ public static void writeFile(String dir, String fileName, String content) {
 			ioe.printStackTrace(System.err);
 		}
 	}
+
+
+	protected static void assertCorrectOutput(RuntimeTestDescriptor descriptor, RuntimeTestSupport delegate, String actualOutput) {
+		String actualParseErrors = delegate.getParseErrors();
+		String actualToolErrors = delegate.getANTLRToolErrors();
+		String expectedOutput = descriptor.getOutput();
+		String expectedParseErrors = descriptor.getErrors();
+		String expectedToolErrors = descriptor.getANTLRToolErrors();
+
+		if (actualOutput == null) {
+			actualOutput = "";
+		}
+		if (actualParseErrors == null) {
+			actualParseErrors = "";
+		}
+		if (actualToolErrors == null) {
+			actualToolErrors = "";
+		}
+		if (expectedOutput == null) {
+			expectedOutput = "";
+		}
+		if (expectedParseErrors == null) {
+			expectedParseErrors = "";
+		}
+		if (expectedToolErrors == null) {
+			expectedToolErrors = "";
+		}
+
+		if (actualOutput.equals(expectedOutput) &&
+				actualParseErrors.equals(expectedParseErrors) &&
+				actualToolErrors.equals(expectedToolErrors)) {
+			return;
+		}
+
+		if (actualOutput.equals(expectedOutput)) {
+			if (actualParseErrors.equals(expectedParseErrors)) {
+				failNotEquals("[" + descriptor.getTarget() + ":" + descriptor.getTestName() + "] " +
+								"Parse output and parse errors are as expected, but tool errors are incorrect",
+						expectedToolErrors, actualToolErrors);
+			}
+			else {
+				fail("[" + descriptor.getTarget() + ":" + descriptor.getTestName() + "] " +
+						"Parse output is as expected, but errors are not: " +
+						"expectedParseErrors:<" + expectedParseErrors +
+						">; actualParseErrors:<" + actualParseErrors +
+						">; expectedToolErrors:<" + expectedToolErrors +
+						">; actualToolErrors:<" + actualToolErrors +
+						">.");
+			}
+		}
+		else {
+			fail("[" + descriptor.getTarget() + ":" + descriptor.getTestName() + "] " +
+					"Parse output is incorrect: " +
+					"expectedOutput:<" + expectedOutput +
+					">; actualOutput:<" + actualOutput +
+					">; expectedParseErrors:<" + expectedParseErrors +
+					">; actualParseErrors:<" + actualParseErrors +
+					">; expectedToolErrors:<" + expectedToolErrors +
+					">; actualToolErrors:<" + actualToolErrors +
+					">.");
+		}
+	}
 }

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java

@@ -56,7 +56,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
-public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTestAssert*/ {
+public class BaseCSharpTest implements RuntimeTestSupport {
 	public static final String newline = System.getProperty("line.separator");
 	public static final String pathSep = System.getProperty("path.separator");
 

runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java

@@ -279,7 +279,6 @@ private void writeParserTestFile(String parserName,
 						"    }\n" +
 						"}\n" +
 						"\n" +
-						"do {\n" +
 						"let args = CommandLine.arguments\n" +
 						"let input = try ANTLRFileStream(args[1])\n" +
 						"let lex = <lexerName>(input)\n" +
@@ -289,12 +288,7 @@ private void writeParserTestFile(String parserName,
 						"<profile>\n" +
 						"let tree = try parser.<parserStartRuleName>()\n" +
 						"<if(profile)>print(profiler.getDecisionInfo().description)<endif>\n" +
-						"try ParseTreeWalker.DEFAULT.walk(TreeShapeListener(), tree)\n" +
-						"}catch ANTLRException.recognition(let e )   {\n" +
-						"    print(\"error occur\\(e)\")\n" +
-						"}catch {\n" +
-						"    print(\"error occur\")\n" +
-						"}\n"
+						"try ParseTreeWalker.DEFAULT.walk(TreeShapeListener(), tree)\n"
 		);
 		ST createParserST = new ST("       let parser = try <parserName>(tokens)\n");
 		if (debug) {
@@ -329,13 +323,7 @@ private void writeLexerTestFile(String lexerName, boolean showDFA) {
 						"let lex = <lexerName>(input)\n" +
 						"let tokens = CommonTokenStream(lex)\n" +
 
-						"do {\n" +
-						"	try tokens.fill()\n" +
-						"} catch ANTLRException.recognition(let e )   {\n" +
-						"	print(\"error occur\\(e)\")\n" +
-						"} catch {\n" +
-						"	print(\"error occur\")\n" +
-						"}\n" +
+						"try tokens.fill()\n" +
 
 						"for t in tokens.getTokens() {\n" +
 						"	print(t)\n" +

runtime/Swift/boot.py

@@ -16,8 +16,10 @@
 import argparse
 import fnmatch
 import os.path
+import subprocess
+import sys
 import time
-from subprocess import call
+from subprocess import check_call
 
 
 # ANTLR Version, here we only care about major version.
@@ -83,17 +85,17 @@ def gen_parser(grammar, a4):
         antlr_complains("Cannot find java. Check your JAVA_HOME setting.")
         return
 
-    call([java, "-jar", a4,
-          "-Dlanguage=Swift", grammar, "-visitor",
-          "-o", grammar_folder + "/gen"])
+    check_call([java, "-jar", a4,
+               "-Dlanguage=Swift", grammar, "-visitor",
+               "-o", grammar_folder + "/gen"])
 
 
 def swift_test():
     """
     Run unit tests.
     """
     generate_parser()
-    call(["swift", "test"])
+    check_call(["swift", "test"])
 
 
 def get_argument_parser():
@@ -141,10 +143,10 @@ def generate_spm_module(in_folder=TMP_FOLDER):
     shutil.copy("Package.swift", tmp_antlr_folder)
 
     os.chdir(tmp_antlr_folder)
-    call(["git", "init"])
-    call(["git", "add", "*"])
-    call(["git", "commit", "-m", "Initial commit."])
-    call(["git", "tag", "{}.0.0".format(MAJOR_VERSION)])
+    check_call(["git", "init"])
+    check_call(["git", "add", "*"])
+    check_call(["git", "commit", "-m", "Initial commit."])
+    check_call(["git", "tag", "{}.0.0".format(MAJOR_VERSION)])
 
     antlr_says("Created local repository.")
     antlr_says("(swift-tools-version:3.0) " 
@@ -163,7 +165,7 @@ def generate_xcodeproj():
     :return:
     """
     generate_parser()
-    call(["swift", "package", "generate-xcodeproj"])
+    check_call(["swift", "package", "generate-xcodeproj"])
 
 
 def generate_parser():
@@ -186,11 +188,16 @@ def antlr_complains(msg):
 if __name__ == "__main__":
     parser = get_argument_parser()
     args = parser.parse_args()
-    if args.gen_spm_module:
-        generate_spm_module()
-    elif args.gen_xcodeproj:
-        generate_xcodeproj()
-    elif args.test:
-        swift_test()
-    else:
-        parser.print_help()
+    try:
+        if args.gen_spm_module:
+            generate_spm_module()
+        elif args.gen_xcodeproj:
+            generate_xcodeproj()
+        elif args.test:
+            swift_test()
+        else:
+            parser.print_help()
+    except subprocess.CalledProcessError as err:
+        print >>sys.stderr, ("Error: command '%s' exited with status %d" %
+                             (' '.join(err.cmd), err.returncode))
+        sys.exit(err.returncode)