summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-07-12 20:06:58 +0000
committerMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-07-12 20:06:58 +0000
commitdfd96ad3ef7c513ed55f4817218ee07487c58203 (patch)
treec9ad6d80b137ce596c493c8c49c1719dc3a3db29
parent68168d9d94ce297025610fff0f6b9486e921bcaa (diff)
downloadjenkins-scripts-dfd96ad3ef7c513ed55f4817218ee07487c58203.tar.gz
tcwg_gnu-build.sh: Rely on validate_results.py to detect regressions
... because it ignore PASS/FAIL ratio, which can be abnormal in bisection runs on a subset of the testsuite. Change-Id: I6c862715e5e0d8fa5d63bd4207d7530894480300
-rwxr-xr-xtcwg_gnu-build.sh77
1 files changed, 35 insertions, 42 deletions
diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
index c5389238..1b48ac4b 100755
--- a/tcwg_gnu-build.sh
+++ b/tcwg_gnu-build.sh
@@ -128,7 +128,7 @@ no_regression_p ()
return 1
fi
- local res res1 res2
+ local res
# We use our modified version of GCC's comparison script
clone_or_update_repo gcc-compare-results master https://git.linaro.org/toolchain/gcc-compare-results.git
@@ -138,7 +138,7 @@ no_regression_p ()
gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
$sumfiles_base $sumfiles_new \
| tee $run_step_artifacts/results.compare1 &
- res1=0 && wait $! || res1=$?
+ res=0 && wait $! || res=$?
local xfail="gcc-compare-results/contrib/testsuite-management/flaky"
if [ -f "$xfail/${rr[ci_config]}.xfail" ]; then
@@ -149,46 +149,39 @@ no_regression_p ()
--manifest=$xfail.xfail --clean_build=$sumfiles_base \
--build_dir=$sumfiles_new \
| tee $run_step_artifacts/results.compare2 &
- res2=0 && wait $! || res2=$?
-
- # Handle return codes in the same way as in compare_jobs.sh from
- # gcc-compare-results
- case $res1:$res2 in
- 0:0) res=0 ;; # No change
- 1:0) res=0 ;; # Improvement
- 4:0) res=0 ;; # Extra or missing logs
- *) # Regression, no common logs or build failed
- res=1
- local reg_lines
- for i in 1 2; do
- reg_lines=$(cat $run_step_artifacts/results.compare$i | wc -l)
- reg_lines=$(($reg_lines-100))
- cat $run_step_artifacts/results.compare$i | sed -e "s/^/# /" \
- | (head -n100; cat >/dev/null) \
- > $run_step_artifacts/results.regressions
- if [ $reg_lines -gt 0 ]; then
- echo "# ... and $reg_lines more entries" \
- >> $run_step_artifacts/results.regressions
- fi
- done
-
- gcc-compare-results/contrib/testsuite-management/validate_failures.py \
- --manifest=$xfail.xfail --clean_build=$sumfiles_base \
- --build_dir=$sumfiles_new --verbosity=1 \
- > $run_step_artifacts/fails.sum &
- res1=0 && wait $! || res1=$?
- assert_with_msg "Result comparison should have failed" \
- [ $res1 = $res2 ]
-
- printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
- local exp
- while read exp; do
- printf "++testsuites %s " $exp >> $run_step_artifacts/extra-bisect-params
- done < <(cat $run_step_artifacts/fails.sum \
- | awk '/^Running .* \.\.\./ { print $2 }')
- printf "\n" >> $run_step_artifacts/extra-bisect-params
- ;;
- esac
+ res=0 && wait $! || res=$?
+
+ if [ $res != 0 ]; then
+ local reg_lines
+ for i in 1 2; do
+ reg_lines=$(cat $run_step_artifacts/results.compare$i | wc -l)
+ reg_lines=$(($reg_lines-100))
+ cat $run_step_artifacts/results.compare$i | sed -e "s/^/# /" \
+ | (head -n100; cat >/dev/null) \
+ > $run_step_artifacts/results.regressions
+ if [ $reg_lines -gt 0 ]; then
+ echo "# ... and $reg_lines more entries" \
+ >> $run_step_artifacts/results.regressions
+ fi
+ done
+
+ local res1
+ gcc-compare-results/contrib/testsuite-management/validate_failures.py \
+ --manifest=$xfail.xfail --clean_build=$sumfiles_base \
+ --build_dir=$sumfiles_new --verbosity=1 \
+ > $run_step_artifacts/fails.sum &
+ res1=0 && wait $! || res1=$?
+ assert_with_msg "Result comparison should have failed" \
+ [ $res1 = $res ]
+
+ printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
+ local exp
+ while read exp; do
+ printf "++testsuites %s " $exp >> $run_step_artifacts/extra-bisect-params
+ done < <(cat $run_step_artifacts/fails.sum \
+ | awk '/^Running .* \.\.\./ { print $2 }')
+ printf "\n" >> $run_step_artifacts/extra-bisect-params
+ fi
return $res
)