Merge branch 'master' into tcwg_bmk

Change-Id: I53fc3879608ee9654ba2b8b7da5b86c9b7099269

8 files changed, 327 insertions, 166 deletions

diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index a55cec14..65428d9a 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -64,10 +64,18 @@ print_nodes_in_labels ()
 
     local labels="$@"
     local label
+    local tmpfile=$(mktemp)
 
     for label in $labels; do
-	wget --retry-connrefused --waitretry=1 -O - https://ci.linaro.org/label/$label/api/json?pretty=true 2>/dev/null | grep nodeName | cut -d: -f 2 | sed -e 's/"//g'
+	# Handle gracefully the case where the label contains no
+	# machine: we don't want to generate an error, rather return
+	# an empty string. Use a tmp file so that we still generate an
+	# error in case wget fails.
+	wget --retry-connrefused --waitretry=1 -O - https://ci.linaro.org/label/$label/api/json?pretty=true 2>/dev/null > $tmpfile
+	grep nodeName $tmpfile | cut -d: -f 2 | sed -e 's/"//g' || true
     done
+
+    rm $tmpfile
     )
 }
 
@@ -675,9 +683,9 @@ print_volume_mounts ()
     local volume_id
 
     case "$job" in
-	tcwg_kernel-*|tcwg_bmk-*)
-	    # Add ccache volume for tcwg_kernel jobs.  These jobs
-	    # depend on ccache for fast rebuilds of LLVM and GCC with
+	tcwg_*-*)
+	    # Add ccache volume for tcwg_* jobs.
+	    # These jobs depend on ccache for fast rebuilds of LLVM and GCC with
 	    # the host compiler.
 	    job="$(echo $job | cut -d- -f 1)"
 	    volume_id=$(print_docker_name "$job$suffix")
diff --git a/round-robin.sh b/round-robin.sh
index 95ce74c9..0844c9de 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -91,14 +91,24 @@ reset_artifacts ()
 
     local single_branch=${rr[baseline_branch]}
 
-    # Clone base-artifacts so that run_step can rsync artifacts for skipped
-    # steps.
+    # Clone base-artifacts here so that bisect runs (which skip this step)
+    # don't overwrite it.
     # base-artifacts repo is big and changes all the time, so we
     # fetch only the $baseline_branch, instead of all branches.
-    rr[base-artifacts_rev]="${rr[base-artifacts_rev]-${rr[baseline_branch]}}"
     if ${rr[init_configuration]-false}; then
 	rr[base-artifacts_rev]=empty
 	single_branch=empty
+    elif [ x"${rr[base-artifacts_rev]+set}" = x"set" ] && ${rr[reset_baseline]}; then
+	# Since we re-write history of base-artifacts all the time we can't
+	# rely on SHA1 stored in manifests (aka ${rr[base-artifacts_rev]}) to
+	# exist.  The important case here is reproduction of regressions from
+	# email instructions: baseline run with reset_baseline set.  We could
+	# have tagged such SHA1s to keep them stored in the repo, but it is easier
+	# to just ignore base-artifacts_rev when reset_baseline is set.
+	rr[base-artifacts_rev]=empty
+	single_branch=empty
+    else
+	rr[base-artifacts_rev]="${rr[base-artifacts_rev]-${rr[baseline_branch]}}"
     fi
 
     clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch
@@ -201,7 +211,9 @@ build_abe ()
 
     local component="$1"
 
-    local project stage
+    local project stage action
+    action="build"
+
     case "$component" in
 	stage1)
 	    project=gcc
@@ -211,6 +223,30 @@ build_abe ()
 	    project=gcc
 	    stage="--stage 2"
 	    ;;
+	bootstrap)
+	    project=gcc
+	    stage="--enable bootstrap"
+	    ;;
+	bootstrap_ubsan)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-ubsan"
+	    ;;
+	bootstrap_O3)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-O3"
+	    ;;
+	bootstrap_O1)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-O1"
+	    ;;
+	bootstrap_lto)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-lto"
+	    ;;
+	bootstrap_debug)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-debug"
+	    ;;
 	*)
 	    project=$component
 	    stage=""
@@ -256,10 +292,15 @@ build_abe ()
     fi
 
     ccache -z
+    local target_opt=""
+    if [ x"${rr[target]}" != x"native" ]; then
+        target_opt="--target $gnu_target"
+    fi
+
     # Run "./abe.sh --build $project".
     ./abe.sh \
-	--build $project \
-	--target $gnu_target \
+	--$action $project \
+	$target_opt \
 	--extraconfigdir config/master \
 	$custom_abe_src_opt \
 	$stage
@@ -442,18 +483,6 @@ update_baseline ()
     git -C base-artifacts commit $amend -m "$msg_title
 
 $(cat ${rr[top_artifacts]}/results)"
-
-    # We saw strange behavior with base-artifacts/results not being updated
-    # in the rsync above.  This should be fixed by "-I" rsync option, but
-    # keep below asserts just in case.
-    if [ x"$(diff -up ${rr[top_artifacts]}/results base-artifacts/results)" != x"" ] \
-	   || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then
-	cd base-artifacts
-	git status
-	cat results
-	ls -la
-	assert false
-    fi
     )
 }
 
diff --git a/tcwg-benchmark.sh b/tcwg-benchmark.sh
index e36e5798..1791369b 100755
--- a/tcwg-benchmark.sh
+++ b/tcwg-benchmark.sh
@@ -25,6 +25,7 @@ NODE_NAME="$NODE_NAME"
 WORKSPACE="$WORKSPACE"
 reboot="$reboot"
 ignore_errors="$ignore_errors"
+clean_older_than="$clean_older_than"
 
 # Jenkins doesn't define variables when parameter value is empty (like cflags),
 # so enable "set -u" only after above binding of variables.
@@ -256,4 +257,5 @@ ssh -t -Snone "$boardname" bmk-scripts/run.sh \
     --resultsdest "bkp-01.tcwglab:/home/tcwg-benchmark/results-${results_id}/${NODE_NAME}" \
     --nodename "${NODE_NAME}" \
     --forceinstall "${forceinstall}" \
+    "${clean_older_than:+--clean_older_than "$clean_older_than"}" \
     --verbose true
diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh
index cc748c5c..d5d999d9 100755
--- a/tcwg-cleanup-stale-containers.sh
+++ b/tcwg-cleanup-stale-containers.sh
@@ -65,7 +65,11 @@ do_cleanup_containers ()
     local hours="$1"
     local docker_ps_opts="$2"
     local action="$3"
+    local action_msg="$4"
+    local msg="$5"
+
     local cleanup_containers=true
+    local dryrun_msg=""
     local only_jenkins_containers=true
 
     if [ "$hours" -eq "0" ]; then
@@ -73,8 +77,10 @@ do_cleanup_containers ()
     elif [ "$hours" -lt "0" ]; then
 	hours="$((0-$hours))"
 	cleanup_containers=false
+	dryrun_msg=" (DRYRUN)"
     fi
 
+    echo "$msg (more than ${hours}h)${dryrun_msg}"
     echo "Container report before:"
     $DOCKER ps $docker_ps_opts
 
@@ -109,10 +115,10 @@ do_cleanup_containers ()
     local res
     local status="0"
     if [ ${#rm_containers[@]} != 0 ]; then
-	echo "Removing containers: ${rm_containers[@]}"
+	echo "Containers to ${action_msg}: ${rm_containers[@]}"
 	if $cleanup_containers; then
             for container in "${rm_containers[@]}"; do
-		echo "Removing container $container"
+		echo "Container to ${action_msg}: $container"
 		$DOCKER $action $container &
 		res=0; wait $! || res=$?
 		if [ $res != 0 ]; then
@@ -121,7 +127,7 @@ do_cleanup_containers ()
 		fi
             done
 	else
-            echo "DRY_RUN: NOT REMOVING CONTAINERS"
+            echo "DRY_RUN: NOT ACTING ON CONTAINERS"
 	    echo "Increasing exit code to indicate stale containers"
 	    status="1"
 	fi
@@ -129,19 +135,19 @@ do_cleanup_containers ()
 	echo "Containers report after:"
 	$DOCKER ps $docker_ps_opts
     else
-	echo "Found no container to remove"
+	echo "Found no container to ${action_msg}"
     fi
 
     exit $status
 }
 
 res="0"
-do_cleanup_containers $cleanup_running_hours "" "stop" &
+do_cleanup_containers $cleanup_running_hours "" "stop" "stop" "Stopping long-running containers" &
 wait $! || res=$?
 status=$res
 
 res="0"
-do_cleanup_containers $cleanup_stopped_hours "-a" "rm -fv" &
+do_cleanup_containers $cleanup_stopped_hours "-a" "rm -fv" "remove" "Removing containers stopped long ago" &
 wait $! || res=$?
 status=$(($status|(2*$res)))
 
@@ -200,9 +206,24 @@ fi
 if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then
     res=0; killall --older-than ${cleanup_ssh_agent_hours}h -u $USER ssh-agent &
     wait $! || res=$?
+    # Killall can fail for several reasons:
+    # return-code 127: command not found
+    # return-code 1: in general means no ssh-agent process was found
+    # In the first case, we want the cleanup job to fail, so that we
+    # know we need to install killall
+    # The second case is OK, unless killall fails for another
+    # reason. Assume OK for now.
     if [ $res != 0 ]; then
-	echo "WARNING: could not kill stale ssh-agent processes"
-	status=$(($status|16))
+	case $res in
+	    127)
+		echo "WARNING: could not kill stale ssh-agent processes (killall command not found)"
+		echo "Increasing exit code to indicate killall is missing"
+		status=$(($status|16))
+		;;
+	    1)
+		echo "WARNING: could not kill stale ssh-agent processes or there was no stale ssh-agent older than ${cleanup_ssh_agent_hours}h"
+		;;
+	esac
     fi
 fi
 
diff --git a/tcwg-rsync-dir.sh b/tcwg-rsync-dir.sh
deleted file mode 100755
index 56372616..00000000
--- a/tcwg-rsync-dir.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/bin/bash
-
-snapshots_dir=
-verbose=false
-
-OPTS="`getopt -l dir:,verbose: -- "$@"`"
-while test $# -gt 0; do
-    case $1 in
-	--dir) snapshots_dir="$2"; shift ;;
-	--verbose) verbose="$2"; shift ;;
-	--) shift; break ;;
-    esac
-    shift
-done
-
-if $verbose; then
-    verbose="set -x"
-else
-    verbose=""
-fi
-$verbose
-
-if [ x"$snapshots_dir" = x"" ]; then
-    echo "ERROR: --dir DIR not specified"
-    exit 1
-fi
-
-declare -A pids
-declare -A results
-
-todo_machines="$@"
-
-for M in $todo_machines; do
-    (
-	$verbose
-	rsync -az --delete $snapshots_dir-new/ $M:$snapshots_dir-new/
-	ssh -fn $M "flock -x $snapshots_dir.lock -c \"rsync -a --delete ${snapshots_dir}-new/ $snapshots_dir/\""
-    ) > /tmp/update-snapshots-ref.$$.$M 2>&1 &
-    pids[$M]=$!
-done
-
-for M in $todo_machines; do
-    set +e
-    echo "Waiting completion for $M..."
-    wait ${pids[$M]}
-    results[$M]=$?
-    set -e
-
-    sed -e "s/^/$M: /" < /tmp/update-snapshots-ref.$$.$M
-    rm /tmp/update-snapshots-ref.$$.$M
-done
-
-all_ok="0"
-for M in $todo_machines; do
-    if [ ${results[$M]} = 0 ]; then
-	result="SUCCESS"
-    else
-	result="FAILED"
-	all_ok="1"
-    fi
-    echo "$result: $M"
-done
-
-exit $all_ok
diff --git a/tcwg-update-snapshots_ref.sh b/tcwg-update-snapshots_ref.sh
deleted file mode 100755
index 09d3991b..00000000
--- a/tcwg-update-snapshots_ref.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-. jenkins-helpers.sh
-convert_args_to_variables "$@"
-
-# Make shellcheck happy and workaround Jenkins not defining variables
-# for empty arguments.
-label="${label-tcwg-x86_64-dev-01}"
-generate_abe="${generate_abe-true}"
-generate_llvm="${generate_llvm-true}"
-generate_misc="${generate_misc-true}"
-update_abe_git="${update_abe_git-true}"
-update_llvm_git="${update_llvm_git-true}"
-verbose="${verbose-true}"
-abe_branch="${abe_branch-tested}"
-
-# Jenkins doesn't define variables when parameter value is empty,
-# so enable "set -u" only after above binding of variables.
-set -u
-
-case "$label" in
-    tcwg-x86_64-dev-01)
-        machines="ex40-01.tcwglab.linaro.org tcwg-d05-01 tcwg-thx1-01"
-        nodes_list=($(print_nodes_in_labels tcwg-apm_64-build tcwg-tk1 tcwg-tx1 tcwg-sq_64-build tcwg-x86_64-1-cam tcwg-x86_64-build tcwg-coordinator))
-        for node in "${nodes_list[@]}"; do
-            machines="$machines $(print_host_for_node $node)"
-        done
-        ;;
-    *) echo "ERROR: Unknown host: $label"; exit 1 ;;
-esac
-
-./tcwg-generate-source-cache.sh --dir-abe $HOME/snapshots-ref-new --dir-llvm $HOME/llvm-reference-new --generate-abe $generate_abe --generate-llvm $generate_llvm --generate-misc $generate_misc --update-abe-git $update_abe_git --update-llvm-git $update_llvm_git --verbose $verbose --abe-branch $abe_branch
-./tcwg-rsync-dir.sh --dir $HOME/snapshots-ref --verbose $verbose -- $machines
-./tcwg-rsync-dir.sh --dir $HOME/llvm-reference --verbose $verbose -- $machines
diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
new file mode 100755
index 00000000..1a546169
--- /dev/null
+++ b/tcwg_gnu-build.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+scripts=$(dirname $0)
+. $scripts/jenkins-helpers.sh
+. $scripts/round-robin.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables rr[ci_config]
+
+# Execution mode: baseline, bisect, jenkins-full
+rr[mode]="${rr[mode]-baseline}"
+
+# Set custom revision for one of the projects, and use baseline revisions
+# for all other projects.
+rr[ci_project]="${rr[ci_project]-tcwg_gnu}"
+rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
+rr[reset_baseline]="${rr[reset_baseline]-false}"
+rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
+
+# {toolchain_name}-{toolchain_ver}-{target}-{bootstrap_config}
+IFS=- read -a ci_config <<EOF
+${rr[ci_config]}
+EOF
+rr[target]="native"
+bootstrap_config=${bootstrap_config-${ci_config[3]}}
+rr[components]="gcc"
+
+# Use baseline branches by default.
+for c in ${rr[components]}; do
+    rr[${c}_branch]=${rr[${c}_branch]-baseline}
+    obligatory_variables rr[${c}_url]
+done
+
+start_at="${start_at-default}"
+finish_at="${finish_at-default}"
+verbose="${verbose-true}"
+verbose2="${verbose2-false}"
+
+if $verbose2; then set -x; fi
+
+trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT
+
+# Set start and finish steps for different modes.
+default_start_at=""
+default_finish_at=""
+case "${rr[mode]}" in
+    "baseline")
+	default_finish_at="update_baseline"
+	;;
+    "bisect")
+	case "$(print_single_updated_component)" in
+	    gcc) default_start_at="build_abe-$bootstrap_config" ;;
+	    *) assert false ;;
+	esac
+	default_finish_at="check_regression"
+	;;
+    "jenkins-full") ;;
+esac
+if [ x"$start_at" = x"default" ]; then
+    start_at="$default_start_at"
+fi
+if [ x"$finish_at" = x"default" ]; then
+    finish_at="$default_finish_at"
+fi
+
+run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"
+
+# Exit with code 0 if no regression compared to base-artifacts/results.
+no_regression_p ()
+{
+    (
+    set -euf -o pipefail
+
+    # The following check tests for an empty branch, and we return 0 if results
+    # are not present, since that's an inherently better state. If base-artifacts
+    # were not existing, then it would fail in reset_artifacts() stage.
+
+    if ! [ -f base-artifacts/results ]; then
+	return 0
+    fi
+
+    local build_result_ref build_result_new
+    build_result_ref=$(tail -n1 base-artifacts/results)
+    build_result_new=$(tail -n1 ${rr[top_artifacts]}/results)
+
+    if [ $build_result_new -lt $build_result_ref ]; then
+	return 1
+    fi
+
+    return 0
+    )
+}
+
+run_step stop_on_fail -10 reset_artifacts
+run_step stop_on_fail x prepare_abe
+run_step skip_on_fail -5 build_abe ${bootstrap_config}
+run_step reset_on_fail x check_regression
+run_step stop_on_fail x update_baseline
+run_step stop_on_fail x push_baseline
+
+trap "" EXIT
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 136dc24c..f6169710 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -187,21 +187,112 @@ ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh"
 
 git bisect start $bad_rev $baseline_rev 2>&1 | tee $artifacts/bisect.log
 
-# "git bisect run" can fail (exit with non-zero) in a number of cases:
-# - on trivial bisects (e.g., between HEAD^ and HEAD),
-# - when merge-base between baseline and bad is worse than baseline,
-# - something else?
-# In all these cases we want to reset baseline to HEAD, so that we catch
-# most of the commits that introduced change in the result metric.
-git bisect run $artifacts/test.sh 2>&1 | tee -a $artifacts/bisect.log &
-res=0 && wait $! || res=$?
+if ! git bisect log > /dev/null; then
+    # Bisect ended before it could start: $bad_rev^ == $baseline_rev
+    first_bad=$bad_rev
+    last_good=$baseline_rev
+else
+    first_bad=""
+    last_good=""
+fi
 
-if [ x"$res" = x"0" ]; then
-    first_bad=$(cat .git/BISECT_RUN | head -n 1 | grep "is the first bad commit" | cut -d" " -f 1)
-    assert [ x"$first_bad" != x"" ]
+# Clone interesting-commits.git repo, which contains a list of SHA1s
+# that might cut down bisection time.  Mostly, these are first_bad and
+# last_good commits.
+interesting_commits_rev=${interesting_commits_rev-${rr[ci_project]}}
+clone_or_update_repo ../interesting-commits $interesting_commits_rev https://git-us.linaro.org/toolchain/ci/interesting-commits.git auto $interesting_commits_rev
+cat <<EOF | manifest_out
+declare -g interesting_commits_rev=$interesting_commits_rev
+EOF
+
+# Add revisions in $@ to interesting-commits and push the repo.
+# Ignore failures (since this is cache handling).
+push_interesting_commits ()
+{
+    (
+    set -euf -o pipefail
 
+    local sha1
+
+    for sha1 in "$@"; do
+	if ! grep -q "$sha1" ../interesting-commits/$current_project; then
+	    echo "$sha1" >> ../interesting-commits/$current_project
+	fi
+    done
+
+    git -C ../interesting-commits add .
+    git -C ../interesting-commits commit -m "Add commits from $BUILD_URL:
+
+$*" &
+    local res=0 && wait $! || res=$?
+
+    if [ x"$res" = x"0" ]; then
+	git_init_linaro_local_remote ../interesting-commits baseline false
+	git_push ../interesting-commits baseline ${rr[ci_project]}
+    fi
+    ) || true
+}
+
+# Print first_bad revision (if detected)
+get_first_bad ()
+{
+    (
+    set -euf -o pipefail
+
+    if [ x"$first_bad" != x"" ]; then
+	echo "$first_bad"
+	return
+    fi
+
+    git bisect log | tail -n1 | grep "^# first bad commit:" \
+	| sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/"
+    )
+}
+
+# Try to reduce bisection range by testing regressions (and their parents)
+# identified in other configurations.
+sha1=""
+touch ../interesting-commits/$current_project
+while [ x"$(get_first_bad </dev/null)" = x"" ] && read sha1; do
+    (
+    set -euf -o pipefail
+
+    # Skip $bad_rev and $baseline_rev, these were already tested.
+    if [ x"$sha1" = x"$bad_rev" -o x"$sha1" = x"$baseline_rev" ]; then
+	continue
+    fi
+    # Skip revisions outside of bisection range.
+    if ! git merge-base --is-ancestor $baseline_rev $sha1 \
+	    || ! git merge-base --is-ancestor $sha1 $bad_rev; then
+	continue
+    fi
+
+    git checkout --detach $sha1
+    $artifacts/test.sh &
+    res=0 && wait $! || res=$?
+    if [ x"$res" = x"0" ]; then
+	git bisect good
+    else
+	git bisect bad
+    fi
+    ) </dev/null
+done < ../interesting-commits/$current_project
+
+if [ x"$(get_first_bad)" = x"" ]; then
+    git bisect run $artifacts/test.sh 2>&1 | tee -a $artifacts/bisect.log &
+    res=0 && wait $! || res=$?
+
+    if [ x"$res" = x"0" ]; then
+	assert [ x"$(get_first_bad)" != x"" ]
+    fi
+fi
+
+first_bad=$(get_first_bad)
+if [ x"$first_bad" != x"" ]; then
+    # "git bisect run" succeeded.  Check whether this is an actual regression
+    # or bisection artifact.
     res=0
-    for last_good in $(git rev-parse $first_bad^@); do
+    for last_good in $last_good $(git rev-parse $first_bad^@); do
 	# It seems that git-bisect assumes parent commit as "good" on
 	# the basis of one of its children being "good".  Therefore we
 	# can have a situation when we have parent P with children C1 and C2,
@@ -218,37 +309,43 @@ if [ x"$res" = x"0" ]; then
 	    break
 	fi
     done
-    if [ x"$res" != x"0" ]; then
+    # Add both $last_good and $first_bad to interesting commits.
+    push_interesting_commits $last_good $first_bad
+    if [ x"$res" = x"0" ]; then
+	# Success!  Touch $artifacts/first-bad as a marker of successful bisect.
+	echo $first_bad > $artifacts/first-bad
+    else
 	# It seems $last_good was on a path that tested good, even though
-	# it itself is bad.  We re-trigger the bisection job with updated
-	# parameters.
+	# it itself is bad.
+	#
 	# We need to be careful to avoid re-trigger loops, so verify that
 	# last_good is an ancestor of bad_rev.
 	assert git merge-base --is-ancestor $last_good $bad_rev
-	cat > $artifacts/trigger-bisect <<EOF
+	if git merge-base --is-ancestor $baseline_rev $last_good; then
+	    # $last_good is a child of $baseline_rev, so we can re-trigger
+	    # bisection with reduced bisection range.
+	    cat > $artifacts/trigger-bisect <<EOF
 current_project=$current_project
 baseline_branch=$baseline_rev
 bad_branch=$last_good
 EOF
-	# Don't send any emails.
-	echo > $artifacts/jenkins/mail-recipients.txt
-	touch $artifacts/jenkins/build-name
-	sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name
-	trap "" EXIT
-	exit 0
+	    touch $artifacts/jenkins/build-name
+	    sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name
+	    # Don't send any emails.
+	    echo > $artifacts/jenkins/mail-recipients.txt
+	    trap "" EXIT
+	    exit 0
+	fi
+	# This case will be handled similar to "git bisect run" failure below.
+	# We are going to reset baseline to $first_bad.
     fi
-
-    echo $first_bad > $artifacts/first-bad
 else
+    # When "git bisect run" fails, e.g., due to merge-base of $baseline_rev and
+    # $bad_rev is worse than $baseline_rev, we want to reset baseline to HEAD,
+    # so that we catch most of the commits that introduced change in the result
+    # metric.
     first_bad=$(git rev-parse HEAD)
-    if ! [ -f .git/BISECT_LOG ]; then
-	# It seems this was a trivial bisect with $bad_rev^ == $baseline_rev.
-	first_bad=$bad_rev
-	last_good=$(git rev-parse $first_bad^)
-	assert [ x"$last_good" = x"$baseline_rev" ]
-
-	echo $first_bad > $artifacts/first-bad
-    fi
+    push_interesting_commits $first_bad
 fi
 cd ..
 
@@ -293,7 +390,7 @@ EOF
 fi
 
 # In log scan for errors below
-#   - ": error:" detects compiler errors from GCC and Clang
+#   - " error:" detects compiler errors from GCC and Clang (including GCC ICEs)
 #   - "^ERROR:" detects linker errors
 #   - "] Error " detects GNU make errors
 cat >> $artifacts/jenkins/mail-body.txt <<EOF
@@ -307,7 +404,7 @@ to (for $bad_name == $bad_sha1)
 $(cat $artifacts/build-$bad_sha1/results)
 
 First few errors in logs of $bad_name:
-$(grep ": error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head)
+$(grep " error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head)
 
 Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/
 Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/
@@ -364,9 +461,9 @@ fi
 CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org"
 case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
     tcwg_kernel/gnu-*:linux) ;;
-    tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;;
+    tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, tcwg-gcc@linaro.org" ;;
     tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
-    tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;;
+    tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, llvm@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
 esac
 cat > $artifacts/jenkins/mail-recipients.txt <<EOF
 $CI_MAIL_RECIPIENTS