jtreg: add failed results as discrete attachments

For each test that fails add the corresponding .jtr (jtreg result) for
easier observation from the LAVA dashboard.  Also attach a zip of the
JTwork and JTreport directory sans the class files.  This zip is now
attached directly to the overall LAVA test using lava-test-run-attach.
The overall execution is now run using jtreg and the results are then
parsed and analysed for success/fail.

Signed-off-by: Andrew McDermott <andrew.mcdermott@linaro.org>
Change-Id: Id5e918e196892e6935b3361f822f6f129f59772c

17 files changed, 90 insertions, 65 deletions

diff --git a/openembedded/jtreg-hotspot-compiler.yaml b/openembedded/jtreg-hotspot-compiler.yaml
index ce821d1..d29b3c7 100644
--- a/openembedded/jtreg-hotspot-compiler.yaml
+++ b/openembedded/jtreg-hotspot-compiler.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t hotspot-compiler -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/compiler'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-hotspot-gc.yaml b/openembedded/jtreg-hotspot-gc.yaml
index 42a8043..3ba1584 100644
--- a/openembedded/jtreg-hotspot-gc.yaml
+++ b/openembedded/jtreg-hotspot-gc.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t hotspot-gc -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/gc'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-hotspot-runtime.yaml b/openembedded/jtreg-hotspot-runtime.yaml
index ab51dde..4459ab2 100644
--- a/openembedded/jtreg-hotspot-runtime.yaml
+++ b/openembedded/jtreg-hotspot-runtime.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t hotspot-runtime -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/runtime'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-hotspot-sanity.yaml b/openembedded/jtreg-hotspot-sanity.yaml
index 9514d02..a6b861a 100644
--- a/openembedded/jtreg-hotspot-sanity.yaml
+++ b/openembedded/jtreg-hotspot-sanity.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t hotspot-sanity -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/sanity'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-hotspot-serviceability.yaml b/openembedded/jtreg-hotspot-serviceability.yaml
index ed6a221..f96b6ca 100644
--- a/openembedded/jtreg-hotspot-serviceability.yaml
+++ b/openembedded/jtreg-hotspot-serviceability.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t hotspot-serviceability -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/serviceability'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-hotspot-testlibrary.yaml b/openembedded/jtreg-hotspot-testlibrary.yaml
index 03f5d38..c5b7673 100644
--- a/openembedded/jtreg-hotspot-testlibrary.yaml
+++ b/openembedded/jtreg-hotspot-testlibrary.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t hotspot-testlibrary -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/testlibrary'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_beans.yaml b/openembedded/jtreg-jdk_beans.yaml
index 8027e5a..a26a224 100644
--- a/openembedded/jtreg-jdk_beans.yaml
+++ b/openembedded/jtreg-jdk_beans.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_beans -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/beans'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_io.yaml b/openembedded/jtreg-jdk_io.yaml
index a56145c..3ea069d 100644
--- a/openembedded/jtreg-jdk_io.yaml
+++ b/openembedded/jtreg-jdk_io.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_io -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/io'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_lang.yaml b/openembedded/jtreg-jdk_lang.yaml
index 9441eb1..9bf99c7 100644
--- a/openembedded/jtreg-jdk_lang.yaml
+++ b/openembedded/jtreg-jdk_lang.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_lang -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/lang'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_math.yaml b/openembedded/jtreg-jdk_math.yaml
index 61b2760..f5247e0 100644
--- a/openembedded/jtreg-jdk_math.yaml
+++ b/openembedded/jtreg-jdk_math.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_math -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/math'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_net.yaml b/openembedded/jtreg-jdk_net.yaml
index ad554c2..0e1e136 100644
--- a/openembedded/jtreg-jdk_net.yaml
+++ b/openembedded/jtreg-jdk_net.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_net -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/net'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_other.yaml b/openembedded/jtreg-jdk_other.yaml
index bb830c4..27ebcbf 100644
--- a/openembedded/jtreg-jdk_other.yaml
+++ b/openembedded/jtreg-jdk_other.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_other -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/other'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_security1.yaml b/openembedded/jtreg-jdk_security1.yaml
index 839ac83..bca59e7 100644
--- a/openembedded/jtreg-jdk_security1.yaml
+++ b/openembedded/jtreg-jdk_security1.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_security1 -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/security1'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_text.yaml b/openembedded/jtreg-jdk_text.yaml
index 3ff85d0..5db963a 100644
--- a/openembedded/jtreg-jdk_text.yaml
+++ b/openembedded/jtreg-jdk_text.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_text -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/text'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_time.yaml b/openembedded/jtreg-jdk_time.yaml
index 7e828f4..664e4f7 100644
--- a/openembedded/jtreg-jdk_time.yaml
+++ b/openembedded/jtreg-jdk_time.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_time -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/time'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/jtreg-jdk_util.yaml b/openembedded/jtreg-jdk_util.yaml
index 428bff1..e7b33f8 100644
--- a/openembedded/jtreg-jdk_util.yaml
+++ b/openembedded/jtreg-jdk_util.yaml
@@ -42,7 +42,7 @@ run:
     - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
     - './jtreg-test -t jdk_util -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/util'
 parse:
-  pattern: "(?P<result>^(Passed|FAILED)):\\s+(?P<test_case_id>(.*))"
+  pattern: "(?P<result>^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P<test_case_id>(.*))"
   fixupdict:
-    Passed: pass
-    FAILED: fail
+    JTREG-SUCCESS: pass
+    JTREG-FAIL: fail
diff --git a/openembedded/scripts/jtreg-test b/openembedded/scripts/jtreg-test
index 7f3b82b..261e1fa 100755
--- a/openembedded/scripts/jtreg-test
+++ b/openembedded/scripts/jtreg-test
@@ -58,14 +58,13 @@ shift $((OPTIND-1))
 output_dir=/tmp/jtreg/$test_case.$$
 rm -rf $output_dir
 
-lava-test-case $test_case --shell		\
-    jtreg					\
-      -J${java_vm}				\
-      -vmoption:${java_vm}			\
-      -jdk:"${product_home}"			\
-      -w:$output_dir/JTwork			\
-      -r:$output_dir/JTreport			\
-      "$@"
+jtreg						\
+    -J${java_vm}				\
+    -vmoption:${java_vm}			\
+    -jdk:"${product_home}"			\
+    -w:$output_dir/JTwork			\
+    -r:$output_dir/JTreport			\
+    "$@"
 
 exit_code=$?
 
@@ -75,26 +74,52 @@ shopt -s nullglob
 # By pushd'ing we make the attached filenames in the LAVA dashboard
 # much smaller.
 
+mkdir -p $output_dir/JTwork/failed
+
 if [ -d $output_dir/JTreport/text ]; then
     pushd $output_dir/JTreport/text
     for i in *.txt; do
+	lava-test-run-attach $i
 	if [ $i = "summary.txt" ]; then
 	    grep 'Failed\.' $i > failed.txt
-	    lava-test-case-attach $test_case failed.txt
 	    grep 'Passed\.' $i > success.txt
-	    lava-test-case-attach $test_case success.txt
-	    sed -i '/Not run\./d' $i
+	    lava-test-run-attach failed.txt
+	    lava-test-run-attach success.txt
+            while read -r line; do
+		tc=$(echo $line | awk '{print $1}')
+		y=$(dirname $tc)
+		z=$(basename $tc .java)
+		lava-test-case $z --result pass
+		echo "JTREG-SUCCESS $z"
+            done < success.txt
+            while read -r line; do
+		tc=$(echo $line | awk '{print $1}')
+		y=$(dirname $tc)
+		z=$(basename $tc .java)
+		test_result=$output_dir/JTwork/$y/${z}.jtr
+		if [ -e $test_result ]; then
+		    pushd $output_dir/JTwork
+		    mkdir -p failed-results/$y
+		    cp $test_result failed-results/$y/${z}.jtr
+		    pushd failed-results
+		    lava-test-case $z --result fail
+		    lava-test-case-attach $z $y/${z}.jtr text/plain
+		    popd
+		    echo "JTREG-FAIL $z"
+		    popd
+		fi
+	    done < failed.txt
 	fi
-	lava-test-case-attach $test_case $i
     done
     popd
 fi
 
-if [ -d $output_dir/JTwork/scratch ]; then
-    pushd $output_dir/JTwork/scratch
-    for i in hs*.log; do
-	lava-test-case-attach $test_case $i text/plain
-    done
+if [ -d $output_dir ]; then
+    pushd $output_dir
+    # Remove .class files as they are quite large.
+    find . -name \*.class -print | xargs rm
+    zip --quiet -r RESULTS-BUNDLE.zip *
+    lava-test-run-attach RESULTS-BUNDLE.zip
     popd
 fi