abe-bisect.sh: Add a sanity check before starting bisection.

Make sure that the BAD revision has more failures than the GOOD one.

This is useful since sometimes the regression being tracked is
random, and bisection could fail or report the revision.

Change-Id: I752a921eb04c9cbb411a2e2baee94d521053fd33

1 files changed, 32 insertions, 0 deletions

diff --git a/abe-bisect.sh b/abe-bisect.sh
index 42c09091..1c7cee21 100755
--- a/abe-bisect.sh
+++ b/abe-bisect.sh
@@ -42,6 +42,10 @@ cd "${mydir}"
 mydir="$(pwd)"
 cd ${rundir}
 
+# Do not wait for the user in case of interactive use
+# (abe-bisect-helper.sh uses git show)
+PAGER=
+
 rm -rf abe
 git clone http://git.linaro.org/toolchain/abe.git
 ABE=${rundir}/abe
@@ -68,6 +72,34 @@ echo BAD: $BADSHA1
 [ x"$GOODSHA1" = x ] && echo "ERROR: Could not find good commit ($GOOD)" && exit 1
 [ x"$BADSHA1" = x ] && echo "ERROR: Could not find bad commit ($BAD)" && exit 1
 
+# Sanity check: make sure BAD has more failures than GOOD
+pushd ${rundir}
+${ABE}/abe.sh --target ${TARGET} gcc=gcc.git@${GOODSHA1} --set runtestflags="${EXP}=${TESTNAME}" --build all --check gcc --excludecheck gdb >& gcc-${GOODSHA1}.log
+if [ $? -ne 0 ]; then
+    echo "ERROR: failure while building/testing GOOD ($GOOD}"
+    exit 1
+fi
+
+sums=`find builds/*/${TARGET}/gcc.git~master_rev_${GOODSHA1}-stage2/ -name "*.sum"`
+good_nb_fail=$(grep ${TESTNAME} $sums | grep -c FAIL)
+good_nb_pass=$(grep ${TESTNAME} $sums | grep -c PASS)
+
+${ABE}/abe.sh --target ${TARGET} gcc=gcc.git@${BADSHA1} --set runtestflags="${EXP}=${TESTNAME}" --build all --check gcc --excludecheck gdb >& gcc-${BADSHA1}.log
+if [ $? -ne 0 ]; then
+    echo "ERROR: failure while building/testing BAD ($BAD}"
+    exit 1
+fi
+
+sums=`find builds/*/${TARGET}/gcc.git~master_rev_${BADSHA1}-stage2/ -name "*.sum"`
+bad_nb_fail=$(grep ${TESTNAME} $sums | grep -c FAIL)
+bad_nb_pass=$(grep ${TESTNAME} $sums | grep -c PASS)
+
+if [ $bad_nb_fail -le $good_nb_fail ]; then
+    echo "ERROR: BAD ($BAD) does not have more failures ($bad_nb_fail) than GOOD ($GOOD) ($good_nb_fail)"
+    exit 1
+fi
+popd
+
 git bisect good ${GOODSHA1}
 git bisect bad ${BADSHA1}