From dfd96ad3ef7c513ed55f4817218ee07487c58203 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Mon, 12 Jul 2021 20:06:58 +0000
Subject: tcwg_gnu-build.sh: Rely on validate_results.py to detect regressions

... because it ignore PASS/FAIL ratio, which can be abnormal in
bisection runs on a subset of the testsuite.

Change-Id: I6c862715e5e0d8fa5d63bd4207d7530894480300
---
 tcwg_gnu-build.sh | 77 +++++++++++++++++++++++++------------------------------
 1 file changed, 35 insertions(+), 42 deletions(-)

(limited to 'tcwg_gnu-build.sh')

diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
index c5389238..1b48ac4b 100755
--- a/tcwg_gnu-build.sh
+++ b/tcwg_gnu-build.sh
@@ -128,7 +128,7 @@ no_regression_p ()
 	return 1
     fi
 
-    local res res1 res2
+    local res
 
     # We use our modified version of GCC's comparison script
     clone_or_update_repo gcc-compare-results master https://git.linaro.org/toolchain/gcc-compare-results.git
@@ -138,7 +138,7 @@ no_regression_p ()
     gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
 				      $sumfiles_base $sumfiles_new \
 	| tee $run_step_artifacts/results.compare1 &
-    res1=0 && wait $! || res1=$?
+    res=0 && wait $! || res=$?
 
     local xfail="gcc-compare-results/contrib/testsuite-management/flaky"
     if [ -f "$xfail/${rr[ci_config]}.xfail" ]; then
@@ -149,46 +149,39 @@ no_regression_p ()
 	--manifest=$xfail.xfail --clean_build=$sumfiles_base \
 	--build_dir=$sumfiles_new \
 	| tee $run_step_artifacts/results.compare2 &
-    res2=0 && wait $! || res2=$?
-
-    # Handle return codes in the same way as in compare_jobs.sh from
-    # gcc-compare-results
-    case $res1:$res2 in
-	0:0) res=0 ;;	# No change
-	1:0) res=0 ;;	# Improvement
-	4:0) res=0 ;;	# Extra or missing logs
-	*)		# Regression, no common logs or build failed
-	    res=1
-	    local reg_lines
-	    for i in 1 2; do
-		reg_lines=$(cat $run_step_artifacts/results.compare$i | wc -l)
-		reg_lines=$(($reg_lines-100))
-		cat $run_step_artifacts/results.compare$i | sed -e "s/^/# /" \
-		    | (head -n100; cat >/dev/null) \
-			  > $run_step_artifacts/results.regressions
-		if [ $reg_lines -gt 0 ]; then
-		    echo "# ... and $reg_lines more entries" \
-			 >> $run_step_artifacts/results.regressions
-		fi
-	    done
-
-	    gcc-compare-results/contrib/testsuite-management/validate_failures.py \
-		--manifest=$xfail.xfail --clean_build=$sumfiles_base \
-		--build_dir=$sumfiles_new --verbosity=1 \
-		> $run_step_artifacts/fails.sum &
-	    res1=0 && wait $! || res1=$?
-	    assert_with_msg "Result comparison should have failed" \
-			    [ $res1 = $res2 ]
-
-	    printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
-	    local exp
-	    while read exp; do
-		printf "++testsuites %s " $exp >> $run_step_artifacts/extra-bisect-params
-	    done < <(cat $run_step_artifacts/fails.sum \
-			 | awk '/^Running .* \.\.\./ { print $2 }')
-	    printf "\n" >> $run_step_artifacts/extra-bisect-params
-	    ;;
-    esac
+    res=0 && wait $! || res=$?
+
+    if [ $res != 0 ]; then
+	local reg_lines
+	for i in 1 2; do
+	    reg_lines=$(cat $run_step_artifacts/results.compare$i | wc -l)
+	    reg_lines=$(($reg_lines-100))
+	    cat $run_step_artifacts/results.compare$i | sed -e "s/^/# /" \
+		| (head -n100; cat >/dev/null) \
+		      > $run_step_artifacts/results.regressions
+	    if [ $reg_lines -gt 0 ]; then
+		echo "# ... and $reg_lines more entries" \
+		     >> $run_step_artifacts/results.regressions
+	    fi
+	done
+
+	local res1
+	gcc-compare-results/contrib/testsuite-management/validate_failures.py \
+	    --manifest=$xfail.xfail --clean_build=$sumfiles_base \
+	    --build_dir=$sumfiles_new --verbosity=1 \
+	    > $run_step_artifacts/fails.sum &
+	res1=0 && wait $! || res1=$?
+	assert_with_msg "Result comparison should have failed" \
+			[ $res1 = $res ]
+
+	printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
+	local exp
+	while read exp; do
+	    printf "++testsuites %s " $exp >> $run_step_artifacts/extra-bisect-params
+	done < <(cat $run_step_artifacts/fails.sum \
+		     | awk '/^Running .* \.\.\./ { print $2 }')
+	printf "\n" >> $run_step_artifacts/extra-bisect-params
+    fi
 
     return $res
     )
-- 
cgit v1.2.3