tcwg_gnu-build.sh: Compare results using validate_failures.py

Change-Id: Icd08eb2890c1a33d55267480d817e91a6800c017

1 files changed, 32 insertions, 23 deletions

diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
index cb108e9f..53886294 100755
--- a/tcwg_gnu-build.sh
+++ b/tcwg_gnu-build.sh
@@ -32,6 +32,8 @@ rr[top_artifacts]=$(abs_path "${rr[top_artifacts]}")
 IFS=- read -a ci_config <<EOF
 ${rr[ci_config]}
 EOF
+# Toolchain version -- master or release
+toolchain_ver=${toolchain_ver-${ci_config[1]}}
 # type_of_test contains the type of action to perform in this test
 # campaign: bootstrap, bootstrap_lto, check_binutils, ....
 type_of_test=${type_of_test-${ci_config[3]}}
@@ -118,7 +120,7 @@ no_regression_p ()
 	return 1
     fi
 
-    local res
+    local res res1 res2
 
     # We use our modified version of GCC's comparison script
     clone_or_update_repo gcc-compare-results master https://git.linaro.org/toolchain/gcc-compare-results.git
@@ -127,38 +129,45 @@ no_regression_p ()
     # shellcheck disable=SC2154
     gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
 				      $sumfiles_base $sumfiles_new \
-	| tee $run_step_artifacts/results.compare &
-    res=0 && wait $! || res=$?
+	| tee $run_step_artifacts/results.compare1 &
+    res1=0 && wait $! || res1=$?
+
+    local xfail="gcc-compare-results/contrib/testsuite-management/flaky"
+    if [ -f "$xfail-${rr[ci_config]}.xfail" ]; then
+	xfail="$xfail-${rr[ci_config]}"
+    fi
+
+    gcc-compare-results/contrib/testsuite-management/validate_failures.py \
+	--manifest=$xfail.xfail --clean_build=$sumfiles_base \
+	--build_dir=$sumfiles_new \
+	| tee $run_step_artifacts/results.compare2 &
+    res2=0 && wait $! || res2=$?
 
     # Handle return codes in the same way as in compare_jobs.sh from
     # gcc-compare-results
-    case $res in
+    case $res1:$res2 in
 	# No change or improvement
-	0|1)
+	0:0|1:0)
 	    res=0
 	    ;;
-	2|3|4|5)
-	    # Regression, No common logs, extra logs or build failed
-	    local reg_lines
-	    reg_lines=$(cat $run_step_artifacts/results.compare | wc -l)
-	    reg_lines=$(($reg_lines-100))
-	    cat $run_step_artifacts/results.compare | sed -e "s/^/# /" \
-		| head -n100 > $run_step_artifacts/results.regressions
-	    if [ $reg_lines -gt 0 ]; then
-		echo "# ... and $reg_lines more entries" \
-		     >> $run_step_artifacts/results.regressions
-	    fi
-	    ;;
+	# Regression, No common logs, extra logs or build failed
 	*)
-	    assert_with_msg "Unknown results comparison status: $res" false
+	    res=1
+	    local reg_lines
+	    for i in 1 2; do
+		reg_lines=$(cat $run_step_artifacts/results.compare$i | wc -l)
+		reg_lines=$(($reg_lines-100))
+		cat $run_step_artifacts/results.compare$i | sed -e "s/^/# /" \
+		    | head -n100 > $run_step_artifacts/results.regressions
+		if [ $reg_lines -gt 0 ]; then
+		    echo "# ... and $reg_lines more entries" \
+			 >> $run_step_artifacts/results.regressions
+		fi
+	    done
 	    ;;
     esac
 
-    if [ $res -ne 0 ]; then
-        return 1
-    fi
-
-    return 0
+    return $res
     )
 }