From a7205ca33e25f8763fa6ccbc365ba9b370698da3 Mon Sep 17 00:00:00 2001
From: Christophe Lyon <christophe.lyon@linaro.org>
Date: Wed, 6 Mar 2019 20:48:37 +0000
Subject: tcwg-cleanup-stale-containers.sh: Do not fail if there is no
 ssh-agent process to kill.

Handle two cases where killall can fail:
- command not found, in which case we want an error so that we know we have to install killall
- no ssh-agent process running, which is OK

Change-Id: I0d6d7e40eed6db7c6250f2231f05b8cca73c8355
---
 tcwg-cleanup-stale-containers.sh | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh
index cc748c5c..df723dba 100755
--- a/tcwg-cleanup-stale-containers.sh
+++ b/tcwg-cleanup-stale-containers.sh
@@ -200,9 +200,24 @@ fi
 if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then
     res=0; killall --older-than ${cleanup_ssh_agent_hours}h -u $USER ssh-agent &
     wait $! || res=$?
+    # Killall can fail for several reasons:
+    # return-code 127: command not found
+    # return-code 1: in general means no ssh-agent process was found
+    # In the first case, we want the cleanup job to fail, so that we
+    # know we need to install killall
+    # The second case is OK, unless killall fails for another
+    # reason. Assume OK for now.
     if [ $res != 0 ]; then
-	echo "WARNING: could not kill stale ssh-agent processes"
-	status=$(($status|16))
+	case $res in
+	    127)
+		echo "WARNING: could not kill stale ssh-agent processes (killall command not found)"
+		echo "Increasing exit code to indicate killall is missing"
+		status=$(($status|16))
+		;;
+	    1)
+		echo "WARNING: could not kill stale ssh-agent processes"
+		;;
+	esac
     fi
 fi
 
-- 
cgit v1.2.3


From f91cadd371b18475234861422db4ef347f16b2ad Mon Sep 17 00:00:00 2001
From: Christophe Lyon <christophe.lyon@linaro.org>
Date: Thu, 7 Mar 2019 15:58:22 +0000
Subject: tcwg-cleanup-stale-containers.sh: Make output easier to understand.

Add a few more messages, to make the Jenkins console easier to understand.

Change-Id: I2bab8aaf5333c4e7f8d8d8681986c961336f1fd6
---
 tcwg-cleanup-stale-containers.sh | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh
index df723dba..d5d999d9 100755
--- a/tcwg-cleanup-stale-containers.sh
+++ b/tcwg-cleanup-stale-containers.sh
@@ -65,7 +65,11 @@ do_cleanup_containers ()
     local hours="$1"
     local docker_ps_opts="$2"
     local action="$3"
+    local action_msg="$4"
+    local msg="$5"
+
     local cleanup_containers=true
+    local dryrun_msg=""
     local only_jenkins_containers=true
 
     if [ "$hours" -eq "0" ]; then
@@ -73,8 +77,10 @@ do_cleanup_containers ()
     elif [ "$hours" -lt "0" ]; then
 	hours="$((0-$hours))"
 	cleanup_containers=false
+	dryrun_msg=" (DRYRUN)"
     fi
 
+    echo "$msg (more than ${hours}h)${dryrun_msg}"
     echo "Container report before:"
     $DOCKER ps $docker_ps_opts
 
@@ -109,10 +115,10 @@ do_cleanup_containers ()
     local res
     local status="0"
     if [ ${#rm_containers[@]} != 0 ]; then
-	echo "Removing containers: ${rm_containers[@]}"
+	echo "Containers to ${action_msg}: ${rm_containers[@]}"
 	if $cleanup_containers; then
             for container in "${rm_containers[@]}"; do
-		echo "Removing container $container"
+		echo "Container to ${action_msg}: $container"
 		$DOCKER $action $container &
 		res=0; wait $! || res=$?
 		if [ $res != 0 ]; then
@@ -121,7 +127,7 @@ do_cleanup_containers ()
 		fi
             done
 	else
-            echo "DRY_RUN: NOT REMOVING CONTAINERS"
+            echo "DRY_RUN: NOT ACTING ON CONTAINERS"
 	    echo "Increasing exit code to indicate stale containers"
 	    status="1"
 	fi
@@ -129,19 +135,19 @@ do_cleanup_containers ()
 	echo "Containers report after:"
 	$DOCKER ps $docker_ps_opts
     else
-	echo "Found no container to remove"
+	echo "Found no container to ${action_msg}"
     fi
 
     exit $status
 }
 
 res="0"
-do_cleanup_containers $cleanup_running_hours "" "stop" &
+do_cleanup_containers $cleanup_running_hours "" "stop" "stop" "Stopping long-running containers" &
 wait $! || res=$?
 status=$res
 
 res="0"
-do_cleanup_containers $cleanup_stopped_hours "-a" "rm -fv" &
+do_cleanup_containers $cleanup_stopped_hours "-a" "rm -fv" "remove" "Removing containers stopped long ago" &
 wait $! || res=$?
 status=$(($status|(2*$res)))
 
@@ -215,7 +221,7 @@ if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then
 		status=$(($status|16))
 		;;
 	    1)
-		echo "WARNING: could not kill stale ssh-agent processes"
+		echo "WARNING: could not kill stale ssh-agent processes or there was no stale ssh-agent older than ${cleanup_ssh_agent_hours}h"
 		;;
 	esac
     fi
-- 
cgit v1.2.3


From 04ed7f8fed2024731f52c315b527f6e7257863a9 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Mon, 11 Mar 2019 09:57:49 +0000
Subject: tcwg_kernel-bisect.sh: Use llvm@ and tcwg-gcc@ groups in email
 settings.

Change-Id: I4c5ccf6fed723c2661cefb5a650eb19e5be0667b
---
 tcwg_kernel-bisect.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 136dc24c..6e1334fc 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -364,9 +364,9 @@ fi
 CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org"
 case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
     tcwg_kernel/gnu-*:linux) ;;
-    tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;;
+    tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, tcwg-gcc@linaro.org" ;;
     tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
-    tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;;
+    tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, llvm@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
 esac
 cat > $artifacts/jenkins/mail-recipients.txt <<EOF
 $CI_MAIL_RECIPIENTS
-- 
cgit v1.2.3


From d7f15cdb0d3d1cc41946142d4deddb96d795c480 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Mon, 11 Mar 2019 10:59:28 +0000
Subject: tcwg_kernel-bisect.sh: Match GCC ICEs in the logs

Change-Id: Ibd291b23a3c4228f548cdf270fb5cb719b7033b5
---
 tcwg_kernel-bisect.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 6e1334fc..06aeaadc 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -293,7 +293,7 @@ EOF
 fi
 
 # In log scan for errors below
-#   - ": error:" detects compiler errors from GCC and Clang
+#   - " error:" detects compiler errors from GCC and Clang (including GCC ICEs)
 #   - "^ERROR:" detects linker errors
 #   - "] Error " detects GNU make errors
 cat >> $artifacts/jenkins/mail-body.txt <<EOF
@@ -307,7 +307,7 @@ to (for $bad_name == $bad_sha1)
 $(cat $artifacts/build-$bad_sha1/results)
 
 First few errors in logs of $bad_name:
-$(grep ": error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head)
+$(grep " error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head)
 
 Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/
 Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/
-- 
cgit v1.2.3


From 51e051a167baa7c572647432c3aa03222f783010 Mon Sep 17 00:00:00 2001
From: Christophe Lyon <christophe.lyon@linaro.org>
Date: Wed, 13 Mar 2019 13:37:17 +0000
Subject: print_nodes_in_labels: Handle empty labels

Return an empty list instead of an error in case the label contains no
slave.

Change-Id: I5b1561dd86d62347b1db59b3204c1c955d181000
---
 jenkins-helpers.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index c9c90560..b43d131f 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -64,10 +64,18 @@ print_nodes_in_labels ()
 
     local labels="$@"
     local label
+    local tmpfile=$(mktemp)
 
     for label in $labels; do
-	wget --retry-connrefused --waitretry=1 -O - https://ci.linaro.org/label/$label/api/json?pretty=true 2>/dev/null | grep nodeName | cut -d: -f 2 | sed -e 's/"//g'
+	# Handle gracefully the case where the label contains no
+	# machine: we don't want to generate an error, rather return
+	# an empty string. Use a tmp file so that we still generate an
+	# error in case wget fails.
+	wget --retry-connrefused --waitretry=1 -O - https://ci.linaro.org/label/$label/api/json?pretty=true 2>/dev/null > $tmpfile
+	grep nodeName $tmpfile | cut -d: -f 2 | sed -e 's/"//g' || true
     done
+
+    rm $tmpfile
     )
 }
 
-- 
cgit v1.2.3


From e53831da288d46eb2288f7251dcc8092d6088ef1 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Tue, 12 Mar 2019 11:12:48 +0000
Subject: tcwg_kernel-bisect.sh: Add support for regression cache

Change-Id: I4d88a4ed947c2bd6f23f216957dc1a141a29cf5b
---
 tcwg_kernel-bisect.sh | 128 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 108 insertions(+), 20 deletions(-)

diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 06aeaadc..468b4bc6 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -187,21 +187,110 @@ ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh"
 
 git bisect start $bad_rev $baseline_rev 2>&1 | tee $artifacts/bisect.log
 
-# "git bisect run" can fail (exit with non-zero) in a number of cases:
-# - on trivial bisects (e.g., between HEAD^ and HEAD),
-# - when merge-base between baseline and bad is worse than baseline,
-# - something else?
-# In all these cases we want to reset baseline to HEAD, so that we catch
-# most of the commits that introduced change in the result metric.
-git bisect run $artifacts/test.sh 2>&1 | tee -a $artifacts/bisect.log &
-res=0 && wait $! || res=$?
+if ! git bisect log > /dev/null; then
+    # Bisect ended before it could start: $bad_rev^ == $baseline_rev
+    first_bad=$bad_rev
+    last_good=$baseline_rev
+else
+    first_bad=""
+    last_good=""
+fi
 
-if [ x"$res" = x"0" ]; then
-    first_bad=$(cat .git/BISECT_RUN | head -n 1 | grep "is the first bad commit" | cut -d" " -f 1)
-    assert [ x"$first_bad" != x"" ]
+# Clone interesting-commits.git repo, which contains a list of SHA1s
+# that might cut down bisection time.  Mostly, these are first_bad and
+# last_good commits.
+interesting_commits_rev=${interesting_commits_rev-${rr[ci_project]}}
+clone_or_update_repo ../interesting-commits $interesting_commits_rev https://git-us.linaro.org/toolchain/ci/interesting-commits.git auto $interesting_commits_rev
+cat <<EOF | manifest_out
+declare -g interesting_commits_rev=$interesting_commits_rev
+EOF
+
+# Add revisions in $@ to interesting-commits and push the repo.
+# Ignore failures (since this is cache handling).
+push_interesting_commits ()
+{
+    (
+    set -euf -o pipefail
+
+    local sha1
+
+    for sha1 in "$@"; do
+	if ! grep -q "$sha1" ../interesting-commits/$current_project; then
+	    echo "$sha1" >> ../interesting-commits/$current_project
+	fi
+    done
+
+    git -C ../interesting-commits add .
+    git -C ../interesting-commits commit -m "Add commits from $BUILD_URL:
+
+$@" &
+    local res=0 && wait $! || res=$?
+
+    if [ x"$res" = x"0" ]; then
+	git_init_linaro_local_remote ../interesting-commits baseline false
+	git_push ../interesting-commits baseline ${rr[ci_project]}
+    fi
+    ) || true
+}
+
+# Print first_bad revision (if detected)
+get_first_bad ()
+{
+    (
+    set -euf -o pipefail
+
+    if [ x"$first_bad" != x"" ]; then
+	echo "$first_bad"
+	return
+    fi
+
+    git bisect log | tail -n1 | grep "^# first bad commit:" \
+	| sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/"
+    )
+}
+
+# Try to reduce bisection range by testing regressions (and their parents)
+# identified in other configurations.
+sha1=""
+touch ../interesting-commits/$current_project
+while [ x"$(get_first_bad </dev/null)" = x"" ] && read sha1; do
+    (
+    set -euf -o pipefail
+
+    # Skip $bad_rev and $baseline_rev, these were already tested.
+    if [ x"$sha1" = x"$bad_rev" -o x"$sha1" = x"$baseline_rev" ]; then
+	continue
+    fi
+    # Skip revisions outside of bisection range.
+    if ! git merge-base --is-ancestor $baseline_rev $sha1 \
+	    || ! git merge-base --is-ancestor $sha1 $bad_rev; then
+	continue
+    fi
+
+    git checkout --detach $sha1
+    $artifacts/test.sh &
+    res=0 && wait $! || res=$?
+    if [ x"$res" = x"0" ]; then
+	git bisect good
+    else
+	git bisect bad
+    fi
+    ) </dev/null
+done < ../interesting-commits/$current_project
+
+if [ x"$(get_first_bad)" = x"" ]; then
+    git bisect run $artifacts/test.sh 2>&1 | tee -a $artifacts/bisect.log &
+    res=0 && wait $! || res=$?
 
+    if [ x"$res" = x"0" ]; then
+	assert [ x"$(get_first_bad)" != x"" ]
+    fi
+fi
+
+first_bad=$(get_first_bad)
+if [ x"$first_bad" != x"" ]; then
     res=0
-    for last_good in $(git rev-parse $first_bad^@); do
+    for last_good in $last_good $(git rev-parse $first_bad^@); do
 	# It seems that git-bisect assumes parent commit as "good" on
 	# the basis of one of its children being "good".  Therefore we
 	# can have a situation when we have parent P with children C1 and C2,
@@ -234,21 +323,20 @@ EOF
 	echo > $artifacts/jenkins/mail-recipients.txt
 	touch $artifacts/jenkins/build-name
 	sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name
+	push_interesting_commits $last_good
 	trap "" EXIT
 	exit 0
     fi
 
+    push_interesting_commits $first_bad $last_good
     echo $first_bad > $artifacts/first-bad
 else
+    # When "git bisect run" fails, e.g., due to merge-base of $baseline_rev and
+    # $bad_rev is worse than $baseline_rev, we want to reset baseline to HEAD,
+    # so that we catch most of the commits that introduced change in the result
+    # metric.
     first_bad=$(git rev-parse HEAD)
-    if ! [ -f .git/BISECT_LOG ]; then
-	# It seems this was a trivial bisect with $bad_rev^ == $baseline_rev.
-	first_bad=$bad_rev
-	last_good=$(git rev-parse $first_bad^)
-	assert [ x"$last_good" = x"$baseline_rev" ]
-
-	echo $first_bad > $artifacts/first-bad
-    fi
+    push_interesting_commits $first_bad
 fi
 cd ..
 
-- 
cgit v1.2.3


From 54f2d735bc3fab394b7b28e5fbffe26261a46603 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Thu, 14 Mar 2019 10:43:31 +0000
Subject: tcwg_kernel-bisect.sh: Fix re-triggering bisection with invalid range

When re-triggering bisections due to bad $last_good commit, we
need to check that baseline_rev is a parent of last_good -- otherwise
we will end up with invalid bisection interval.

Change-Id: Ic9f9e07783c967f2aef777105a26f02e526a8505
---
 tcwg_kernel-bisect.sh | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 468b4bc6..12d48f68 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -289,6 +289,8 @@ fi
 
 first_bad=$(get_first_bad)
 if [ x"$first_bad" != x"" ]; then
+    # "git bisect run" succeeded.  Check whether this is an actual regression
+    # or bisection artifact.
     res=0
     for last_good in $last_good $(git rev-parse $first_bad^@); do
 	# It seems that git-bisect assumes parent commit as "good" on
@@ -307,29 +309,38 @@ if [ x"$first_bad" != x"" ]; then
 	    break
 	fi
     done
-    if [ x"$res" != x"0" ]; then
+    if [ x"$res" = x"0" ]; then
+	# Success!
+	push_interesting_commits $first_bad $last_good
+	# Touch $artifacts/first-bad as a marker of successful bisect.
+	echo $first_bad > $artifacts/first-bad
+    else
 	# It seems $last_good was on a path that tested good, even though
 	# it itself is bad.  We re-trigger the bisection job with updated
 	# parameters.
+	push_interesting_commits $last_good
+
 	# We need to be careful to avoid re-trigger loops, so verify that
 	# last_good is an ancestor of bad_rev.
 	assert git merge-base --is-ancestor $last_good $bad_rev
-	cat > $artifacts/trigger-bisect <<EOF
+	if git merge-base --is-ancestor $baseline_rev $last_good; then
+	    # $last_good is a child of $baseline_rev, so we can re-trigger
+	    # bisection with reduced bisection range.
+	    cat > $artifacts/trigger-bisect <<EOF
 current_project=$current_project
 baseline_branch=$baseline_rev
 bad_branch=$last_good
 EOF
-	# Don't send any emails.
-	echo > $artifacts/jenkins/mail-recipients.txt
-	touch $artifacts/jenkins/build-name
-	sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name
-	push_interesting_commits $last_good
-	trap "" EXIT
-	exit 0
+	    touch $artifacts/jenkins/build-name
+	    sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name
+	    # Don't send any emails.
+	    echo > $artifacts/jenkins/mail-recipients.txt
+	    trap "" EXIT
+	    exit 0
+	fi
+	# This case will be handled similar to "git bisect run" failure below.
+	# We are going to reset baseline to $first_bad.
     fi
-
-    push_interesting_commits $first_bad $last_good
-    echo $first_bad > $artifacts/first-bad
 else
     # When "git bisect run" fails, e.g., due to merge-base of $baseline_rev and
     # $bad_rev is worse than $baseline_rev, we want to reset baseline to HEAD,
-- 
cgit v1.2.3


From 7b44d0602565d7a6da313506637610318ce2bed4 Mon Sep 17 00:00:00 2001
From: Christophe Lyon <christophe.lyon@linaro.org>
Date: Thu, 14 Mar 2019 10:24:58 +0000
Subject: tcwg-update-snapshots_ref.sh: Use tcwg-tk1_32-build label instead of
 tcwg-tk1

Our new tcwg-tk1-10 board has tcwg-tk1_32-build only.

Change-Id: Ifaa68fbadc83f6e1b0cca0cdf520587ef210d77c
---
 tcwg-update-snapshots_ref.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tcwg-update-snapshots_ref.sh b/tcwg-update-snapshots_ref.sh
index 09d3991b..83a5ac69 100755
--- a/tcwg-update-snapshots_ref.sh
+++ b/tcwg-update-snapshots_ref.sh
@@ -23,7 +23,7 @@ set -u
 case "$label" in
     tcwg-x86_64-dev-01)
         machines="ex40-01.tcwglab.linaro.org tcwg-d05-01 tcwg-thx1-01"
-        nodes_list=($(print_nodes_in_labels tcwg-apm_64-build tcwg-tk1 tcwg-tx1 tcwg-sq_64-build tcwg-x86_64-1-cam tcwg-x86_64-build tcwg-coordinator))
+        nodes_list=($(print_nodes_in_labels tcwg-apm_64-build tcwg-tk1_32-build tcwg-tx1 tcwg-sq_64-build tcwg-x86_64-1-cam tcwg-x86_64-build tcwg-coordinator))
         for node in "${nodes_list[@]}"; do
             machines="$machines $(print_host_for_node $node)"
         done
-- 
cgit v1.2.3


From 82cbd12c5395b01d1a4ba3b7b62611d3065f7122 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Fri, 15 Mar 2019 06:19:22 +0000
Subject: tcwg_kernel-bisect.sh: Always add $first_bad to
 interesting-commits.git

In case of last_good being bad we often have first_bad just on
the boundary of bisection range, and last_good outside of bisection
range, so we end up ignoring last_good commit in bisections of similar
configurations.  Adding first_bad commit will reduce bisection time in
such cases.

Change-Id: I33ab875ec751a161452634c8b989a39b4e8eb05a
---
 tcwg_kernel-bisect.sh | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 12d48f68..5129d1f5 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -309,17 +309,15 @@ if [ x"$first_bad" != x"" ]; then
 	    break
 	fi
     done
+    # Add both $last_good and $first_bad to interesting commits.
+    push_interesting_commits $last_good $first_bad
     if [ x"$res" = x"0" ]; then
-	# Success!
-	push_interesting_commits $first_bad $last_good
-	# Touch $artifacts/first-bad as a marker of successful bisect.
+	# Success!  Touch $artifacts/first-bad as a marker of successful bisect.
 	echo $first_bad > $artifacts/first-bad
     else
 	# It seems $last_good was on a path that tested good, even though
-	# it itself is bad.  We re-trigger the bisection job with updated
-	# parameters.
-	push_interesting_commits $last_good
-
+	# it itself is bad.
+	#
 	# We need to be careful to avoid re-trigger loops, so verify that
 	# last_good is an ancestor of bad_rev.
 	assert git merge-base --is-ancestor $last_good $bad_rev
-- 
cgit v1.2.3


From 258dc2d14d1374840daa7cb41154dfcdb0b41d55 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Mon, 11 Mar 2019 09:39:50 +0000
Subject: round-robin.sh: Workaround re-writes of base-artifacts.git history

Change-Id: I946de0b6d4d4ee62f1bae1532ad9bff52131bcb5
---
 round-robin.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/round-robin.sh b/round-robin.sh
index 81b04e95..21d7862a 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -91,14 +91,23 @@ reset_artifacts ()
 
     local single_branch=${rr[baseline_branch]}
 
-    # Clone base-artifacts so that run_step can rsync artifacts for skipped
-    # steps.
+    # Clone base-artifacts here so that bisect runs (which skip this step)
+    # don't overwrite it.
     # base-artifacts repo is big and changes all the time, so we
     # fetch only the $baseline_branch, instead of all branches.
-    rr[base-artifacts_rev]="${rr[base-artifacts_rev]-${rr[baseline_branch]}}"
-    if ${rr[init_configuration]-false}; then
+    #
+    # Since we re-write history of base-artifacts all the time we can't
+    # rely on SHA1 stored in manifests (aka ${rr[base-artifacts_rev]}) to
+    # exist.  The important case here is reproduction of regressions from
+    # email instructions: baseline run with reset_baseline set.  We could
+    # have tagged such SHA1s to keep them stored in the repo, but it is easier
+    # to just ignore base-artifacts_rev when reset_baseline is set.
+    if ${rr[init_configuration]-false} \
+	    || ${rr[base-artifacts_rev]+${rr[reset_baseline]}}; then
 	rr[base-artifacts_rev]=empty
 	single_branch=empty
+    else
+	rr[base-artifacts_rev]="${rr[base-artifacts_rev]-${rr[baseline_branch]}}"
     fi
 
     clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch
-- 
cgit v1.2.3


From 7404092ade6be57c39e0dbf50b4375d29e69d569 Mon Sep 17 00:00:00 2001
From: Christophe Lyon <christophe.lyon@linaro.org>
Date: Thu, 14 Mar 2019 14:37:52 +0000
Subject: tcwg-benchmark.sh: Add support for clean_older_than

... to remove old results and avoid filling up the disk.

Change-Id: Ife87756fc92f48717baa0f84c7c3fe16b49b1899
---
 tcwg-benchmark.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tcwg-benchmark.sh b/tcwg-benchmark.sh
index fbb49111..6fa63c22 100755
--- a/tcwg-benchmark.sh
+++ b/tcwg-benchmark.sh
@@ -25,6 +25,7 @@ NODE_NAME="$NODE_NAME"
 WORKSPACE="$WORKSPACE"
 reboot="$reboot"
 ignore_errors="$ignore_errors"
+clean_older_than="$clean_older_than"
 
 # Jenkins doesn't define variables when parameter value is empty (like cflags),
 # so enable "set -u" only after above binding of variables.
@@ -254,4 +255,5 @@ ssh -t -Snone "$boardname" bmk-scripts/run.sh \
     --resultsdest "dev-01.tcwglab:/home/tcwg-benchmark/results-${results_id}/${NODE_NAME}" \
     --nodename "${NODE_NAME}" \
     --forceinstall "${forceinstall}" \
+    "${clean_older_than:+--clean_older_than "$clean_older_than"}" \
     --verbose true
-- 
cgit v1.2.3


From 04a47e7244b7eeeb54952fdc5dba4962f9a372a5 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Fri, 15 Mar 2019 11:31:03 +0000
Subject: tcwg_kernel-bisect.sh: Fix shell expansion

"$@" expands to several strings, which become extra git command-line
arguments, and "$*" expands to a single string.

Change-Id: I95799d258ffb8dc3e4f79777894a0425eafeceb9
---
 tcwg_kernel-bisect.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 5129d1f5..f6169710 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -223,7 +223,7 @@ push_interesting_commits ()
     git -C ../interesting-commits add .
     git -C ../interesting-commits commit -m "Add commits from $BUILD_URL:
 
-$@" &
+$*" &
     local res=0 && wait $! || res=$?
 
     if [ x"$res" = x"0" ]; then
-- 
cgit v1.2.3


From 97580e57b3099e1bfd7257bbda47eb1050184b91 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Fri, 15 Mar 2019 11:59:24 +0000
Subject: round-robin.sh: Remove assert in update_baseline

... which no longer holds.  After we started keeping first-bad
artifacts (which are very useful), we can have no change in result
metric compared to previous [first-bad] build.

Change-Id: I9a3651226ca12c20ae4c06a2b2bd00bd8a63f518
---
 round-robin.sh | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/round-robin.sh b/round-robin.sh
index 21d7862a..129e6fd5 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -444,18 +444,6 @@ update_baseline ()
     git -C base-artifacts commit $amend -m "$msg_title
 
 $(cat ${rr[top_artifacts]}/results)"
-
-    # We saw strange behavior with base-artifacts/results not being updated
-    # in the rsync above.  This should be fixed by "-I" rsync option, but
-    # keep below asserts just in case.
-    if [ x"$(diff -up ${rr[top_artifacts]}/results base-artifacts/results)" != x"" ] \
-	   || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then
-	cd base-artifacts
-	git status
-	cat results
-	ls -la
-	assert false
-    fi
     )
 }
 
-- 
cgit v1.2.3


From f70292585aabfa29006398e941d2f76512d4d962 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Fri, 15 Mar 2019 12:18:24 +0000
Subject: round-robin.sh: Fix un-intutive bash behavior

Turns out that condition in "if ${undefined_var+false}; then ... fi"
evaluated to "true" when $undefined_var is undefined -- i.e., empty
condition is a "true" condition, as long as it is not physically
empty ("if ; then ... fi" will produce a syntatic error).

This perculiarity of bash caused unexpected behavior and we used
"empty" branch of base-artifacts for several builds -- which,
unfortunately, caused us to loose history regressions in several
configurations.  We caught this quickly, so no serious damage done.

Fix by adding verbose conditionals.

Change-Id: Ib6f1a624a9ea41f7223bc3e85e2e4eddc3fad9a9
---
 round-robin.sh | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/round-robin.sh b/round-robin.sh
index 129e6fd5..6e669e11 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -95,15 +95,16 @@ reset_artifacts ()
     # don't overwrite it.
     # base-artifacts repo is big and changes all the time, so we
     # fetch only the $baseline_branch, instead of all branches.
-    #
-    # Since we re-write history of base-artifacts all the time we can't
-    # rely on SHA1 stored in manifests (aka ${rr[base-artifacts_rev]}) to
-    # exist.  The important case here is reproduction of regressions from
-    # email instructions: baseline run with reset_baseline set.  We could
-    # have tagged such SHA1s to keep them stored in the repo, but it is easier
-    # to just ignore base-artifacts_rev when reset_baseline is set.
-    if ${rr[init_configuration]-false} \
-	    || ${rr[base-artifacts_rev]+${rr[reset_baseline]}}; then
+    if ${rr[init_configuration]-false}; then
+	rr[base-artifacts_rev]=empty
+	single_branch=empty
+    elif [ x"${rr[base-artifacts_rev]+set}" = x"set" ] && ${rr[reset_baseline]}; then
+	# Since we re-write history of base-artifacts all the time we can't
+	# rely on SHA1 stored in manifests (aka ${rr[base-artifacts_rev]}) to
+	# exist.  The important case here is reproduction of regressions from
+	# email instructions: baseline run with reset_baseline set.  We could
+	# have tagged such SHA1s to keep them stored in the repo, but it is easier
+	# to just ignore base-artifacts_rev when reset_baseline is set.
 	rr[base-artifacts_rev]=empty
 	single_branch=empty
     else
-- 
cgit v1.2.3


From 9bc981246a158a6a688a6c733a2958e755ddf7b9 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Mon, 18 Mar 2019 10:08:56 +0000
Subject: tcwg-update-snapshots_ref.sh, tcwg-rsync-dir.sh: Delete

... these are no longer used after refactor of tcwg-update-snapshots_ref job

Change-Id: I07e16c2c60e00cc629e5eef9fcd6798aaab50cc3
---
 tcwg-rsync-dir.sh            | 64 --------------------------------------------
 tcwg-update-snapshots_ref.sh | 36 -------------------------
 2 files changed, 100 deletions(-)
 delete mode 100755 tcwg-rsync-dir.sh
 delete mode 100755 tcwg-update-snapshots_ref.sh

diff --git a/tcwg-rsync-dir.sh b/tcwg-rsync-dir.sh
deleted file mode 100755
index 56372616..00000000
--- a/tcwg-rsync-dir.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/bin/bash
-
-snapshots_dir=
-verbose=false
-
-OPTS="`getopt -l dir:,verbose: -- "$@"`"
-while test $# -gt 0; do
-    case $1 in
-	--dir) snapshots_dir="$2"; shift ;;
-	--verbose) verbose="$2"; shift ;;
-	--) shift; break ;;
-    esac
-    shift
-done
-
-if $verbose; then
-    verbose="set -x"
-else
-    verbose=""
-fi
-$verbose
-
-if [ x"$snapshots_dir" = x"" ]; then
-    echo "ERROR: --dir DIR not specified"
-    exit 1
-fi
-
-declare -A pids
-declare -A results
-
-todo_machines="$@"
-
-for M in $todo_machines; do
-    (
-	$verbose
-	rsync -az --delete $snapshots_dir-new/ $M:$snapshots_dir-new/
-	ssh -fn $M "flock -x $snapshots_dir.lock -c \"rsync -a --delete ${snapshots_dir}-new/ $snapshots_dir/\""
-    ) > /tmp/update-snapshots-ref.$$.$M 2>&1 &
-    pids[$M]=$!
-done
-
-for M in $todo_machines; do
-    set +e
-    echo "Waiting completion for $M..."
-    wait ${pids[$M]}
-    results[$M]=$?
-    set -e
-
-    sed -e "s/^/$M: /" < /tmp/update-snapshots-ref.$$.$M
-    rm /tmp/update-snapshots-ref.$$.$M
-done
-
-all_ok="0"
-for M in $todo_machines; do
-    if [ ${results[$M]} = 0 ]; then
-	result="SUCCESS"
-    else
-	result="FAILED"
-	all_ok="1"
-    fi
-    echo "$result: $M"
-done
-
-exit $all_ok
diff --git a/tcwg-update-snapshots_ref.sh b/tcwg-update-snapshots_ref.sh
deleted file mode 100755
index 83a5ac69..00000000
--- a/tcwg-update-snapshots_ref.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-. jenkins-helpers.sh
-convert_args_to_variables "$@"
-
-# Make shellcheck happy and workaround Jenkins not defining variables
-# for empty arguments.
-label="${label-tcwg-x86_64-dev-01}"
-generate_abe="${generate_abe-true}"
-generate_llvm="${generate_llvm-true}"
-generate_misc="${generate_misc-true}"
-update_abe_git="${update_abe_git-true}"
-update_llvm_git="${update_llvm_git-true}"
-verbose="${verbose-true}"
-abe_branch="${abe_branch-tested}"
-
-# Jenkins doesn't define variables when parameter value is empty,
-# so enable "set -u" only after above binding of variables.
-set -u
-
-case "$label" in
-    tcwg-x86_64-dev-01)
-        machines="ex40-01.tcwglab.linaro.org tcwg-d05-01 tcwg-thx1-01"
-        nodes_list=($(print_nodes_in_labels tcwg-apm_64-build tcwg-tk1_32-build tcwg-tx1 tcwg-sq_64-build tcwg-x86_64-1-cam tcwg-x86_64-build tcwg-coordinator))
-        for node in "${nodes_list[@]}"; do
-            machines="$machines $(print_host_for_node $node)"
-        done
-        ;;
-    *) echo "ERROR: Unknown host: $label"; exit 1 ;;
-esac
-
-./tcwg-generate-source-cache.sh --dir-abe $HOME/snapshots-ref-new --dir-llvm $HOME/llvm-reference-new --generate-abe $generate_abe --generate-llvm $generate_llvm --generate-misc $generate_misc --update-abe-git $update_abe_git --update-llvm-git $update_llvm_git --verbose $verbose --abe-branch $abe_branch
-./tcwg-rsync-dir.sh --dir $HOME/snapshots-ref --verbose $verbose -- $machines
-./tcwg-rsync-dir.sh --dir $HOME/llvm-reference --verbose $verbose -- $machines
-- 
cgit v1.2.3


From 4f5214da5e215b67567e6e5ac39397c210dc0ab6 Mon Sep 17 00:00:00 2001
From: Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org>
Date: Fri, 15 Mar 2019 15:32:57 +0530
Subject: Patch [1/3] Merge tcwg_gnu: jenkins-helpers.sh -
 s/tcwg_kernel-*/tcwg_*-*.

Change-Id: Ibc6dcd085f684eb6f09ec9a512aa5abd6b9d8200
---
 jenkins-helpers.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index b43d131f..5291873e 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -679,11 +679,11 @@ print_volume_mounts ()
     local volume_id
 
     case "$job" in
-	tcwg_kernel-*)
-	    # Add ccache volume for tcwg_kernel jobs.  These jobs
-	    # depend on ccache for fast rebuilds of LLVM and GCC with
+	tcwg_*-*)
+	    # Add ccache volume for tcwg_* jobs.
+	    # These jobs depend on ccache for fast rebuilds of LLVM and GCC with
 	    # the host compiler.
-	    job="tcwg_kernel"
+	    job="$(echo $job | cut -d- -f 1)"
 	    volume_id=$(print_docker_name "$job$suffix")
 	    mounts+=(ccache-$volume_id:$HOME/.ccache)
 	    ;;
-- 
cgit v1.2.3


From aca75e6b808dd130d940d7b0ad806b0d9e12c936 Mon Sep 17 00:00:00 2001
From: Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org>
Date: Fri, 15 Mar 2019 15:34:33 +0530
Subject: Patch [2/3] Merge tcwg_gnu: Create tcwg_gnu-build.sh as copy of
 tcwg_kernel-build.sh.

Change-Id: I4c47f486d78f6e9a20732e06a387bb0db77c59ed
---
 tcwg_gnu-build.sh | 263 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 263 insertions(+)
 create mode 100755 tcwg_gnu-build.sh

diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
new file mode 100755
index 00000000..9485e0ba
--- /dev/null
+++ b/tcwg_gnu-build.sh
@@ -0,0 +1,263 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+scripts=$(dirname $0)
+. $scripts/jenkins-helpers.sh
+. $scripts/round-robin.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables rr[ci_config]
+
+# Execution mode: baseline, bisect, jenkins-full
+rr[mode]="${rr[mode]-baseline}"
+
+# Set custom revision for one of the projects, and use baseline revisions
+# for all other projects.
+rr[ci_project]="${rr[ci_project]-tcwg_kernel}"
+rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
+rr[reset_baseline]="${rr[reset_baseline]-false}"
+rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
+
+# {toolchain_name}-{toolchain_ver}-{target}-{linux}-{linux_config}
+IFS=- read -a ci_config <<EOF
+${rr[ci_config]}
+EOF
+rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
+rr[release]=${rr[release]-${ci_config[1]}}
+rr[target]=${rr[target]-${ci_config[2]}}
+rr[linux_config]=${rr[linux_config]-${ci_config[4]}}
+
+case "${rr[toolchain]}" in
+    llvm) rr[components]="llvm linux" ;;
+    gnu) rr[components]="binutils gcc linux" ;;
+    *) assert false ;;
+esac
+
+# Use baseline branches by default.
+for c in ${rr[components]}; do
+    rr[${c}_branch]=${rr[${c}_branch]-baseline}
+    obligatory_variables rr[${c}_url]
+done
+
+start_at="${start_at-default}"
+finish_at="${finish_at-default}"
+verbose="${verbose-true}"
+verbose2="${verbose2-false}"
+
+if $verbose2; then set -x; fi
+
+trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT
+
+# Set start and finish steps for different modes.
+default_start_at=""
+default_finish_at=""
+case "${rr[mode]}" in
+    "baseline")
+	default_finish_at="update_baseline"
+	;;
+    "bisect")
+	case "${rr[toolchain]}:$(print_single_updated_component)" in
+	    gnu:binutils) default_start_at="build_abe-binutils" ;;
+	    gnu:gcc) default_start_at="build_abe-stage1" ;;
+	    gnu:linux) default_start_at="build_linux" ;;
+	    llvm:llvm) default_start_at="build_llvm" ;;
+	    llvm:linux) default_start_at="build_linux" ;;
+	    *) assert false ;;
+	esac
+	default_finish_at="check_regression"
+	;;
+    "jenkins-full") ;;
+esac
+if [ x"$start_at" = x"default" ]; then
+    start_at="$default_start_at"
+fi
+if [ x"$finish_at" = x"default" ]; then
+    finish_at="$default_finish_at"
+fi
+
+run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"
+
+# Build Linux kernel
+build_linux_1 ()
+{
+    (
+    set -euf -o pipefail
+
+    clone_repo linux
+
+    rm -rf $(pwd)/bin
+    mkdir $(pwd)/bin
+
+    local bin cc ld_opt
+
+    case "${rr[toolchain]}" in
+	llvm)
+	    bin="$(pwd)/llvm-install/bin"
+	    cc="$bin/clang"
+	    ;;
+	gnu)
+	    bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin"
+	    cc="$bin/$(print_gnu_target ${rr[target]})-gcc"
+	    ;;
+    esac
+
+    case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in
+        llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;;
+        llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;;
+        *) ld_opt="" ;;
+    esac
+
+    # Use binutils, etc from $bin
+    export PATH="$bin:$PATH"
+
+    # Use ccache only when bisecting linux (or preparing to).
+    # Otherwise the compiler is new in every build and we would
+    # only clobber ccache volume.
+    local ccache=""
+    if [ x"${rr[mode]}" != x"jenkins-full" \
+	  -a x"$(print_single_updated_component)" = x"linux" ]; then
+	ccache="ccache"
+    fi
+    cat > $(pwd)/bin/${rr[target]}-cc <<EOF
+#!/bin/sh
+exec $ccache $cc "\$@"
+EOF
+    chmod +x $(pwd)/bin/${rr[target]}-cc
+
+    # Define make variables.
+    local opts="CC=$(pwd)/bin/${rr[target]}-cc $ld_opt SUBLEVEL=0 EXTRAVERSION=-bisect"
+    if [ x"${rr[target]}" != x"$(uname -m)" ]; then
+	opts="$opts ARCH=$(print_kernel_target ${rr[target]})"
+	opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-"
+	opts="$opts HOSTCC=gcc"
+    fi
+
+    cd linux
+
+    make $opts distclean
+
+    make $opts ${rr[linux_config]}
+    sed -i -e 's:CONFIG_LOCALVERSION_AUTO=y:# CONFIG_LOCALVERSION_AUTO is not set:' .config
+    set +f; rm -f localversion*; set -f
+    make $opts oldconfig
+
+    ccache -z
+    KBUILD_BUILD_TIMESTAMP=0 make $opts -j$(nproc --all) -s -k &
+    local res=0 && wait $! || res=$?
+    ccache -s
+    return $res
+    )
+}
+
+# Build linux and count number successfully built .o files in linux
+build_linux ()
+{
+    (
+    set -euf -o pipefail
+
+    build_linux_1 &
+    local res=0 && wait $! || res=$?
+
+    # Number of .o files created is the main success metric.
+    local linux_n_obj
+    linux_n_obj=$(find linux -name "*.o" | wc -l)
+    echo "linux_n_obj:" >> ${rr[top_artifacts]}/results
+    echo "$linux_n_obj" >> ${rr[top_artifacts]}/results
+
+    if [ $res = 0 ]; then
+	echo "linux build successful:" >> ${rr[top_artifacts]}/results
+	echo "all" >> ${rr[top_artifacts]}/results
+    fi
+
+    return $res
+    )
+}
+
+# Boot linux kernel
+boot_linux ()
+{
+    (
+    set -euf -o pipefail
+
+    local image cpu
+    case ${rr[target]} in
+	aarch64)
+	    image=linux/arch/arm64/boot/Image.gz
+	    cpu="-cpu cortex-a53"
+	    ;;
+	arm)
+	    image=linux/arch/arm/boot/zImage
+	    cpu=""
+	    ;;
+	*) assert false ;;
+    esac
+    timeout 10s qemu-system-${rr[target]} \
+	    -kernel $image -machine virt $cpu -m 512 \
+	    -serial mon:stdio -display none \
+	    -append "console=ttyAMA0 panic=-1" -no-reboot
+
+    echo "linux boot successful:" >> ${rr[top_artifacts]}/results
+    echo "boot" >> ${rr[top_artifacts]}/results
+    )
+}
+
+# Exit with code 0 if no regression compared to base-artifacts/results.
+no_regression_p ()
+{
+    (
+    set -euf -o pipefail
+
+    local linux_n_obj
+    linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results)
+
+    # Assume worst for non-existent baseline.
+    local base_linux_n_obj="-10"
+    if [ -f base-artifacts/results ]; then
+	base_linux_n_obj=$(tail -n1 base-artifacts/results)
+    fi
+
+    case "$linux_n_obj:$base_linux_n_obj" in
+	boot:*) return 0 ;;
+	*:boot) return 1 ;;
+	all:*) return 0 ;;
+	*:all) return 1 ;;
+	*)
+	    if ! [ "$linux_n_obj" -ge "-10" ]; then
+		# Something is very wrong with result (e.g., it's not a number).
+		return 1
+	    fi
+	    if ! [ "$base_linux_n_obj" -ge "-10" ]; then
+		# Something is very wrong with result (e.g., it's not a number).
+		return 0
+	    fi
+
+	    if [ $linux_n_obj -ge $base_linux_n_obj ]; then
+		return 0
+	    else
+		return 1
+	    fi
+	    ;;
+    esac
+    )
+}
+
+run_step stop_on_fail -10 reset_artifacts
+case "${rr[toolchain]}" in
+    gnu)
+	run_step stop_on_fail x prepare_abe
+	run_step skip_on_fail -9 build_abe binutils
+	run_step skip_on_fail -8 build_abe stage1
+	;;
+    llvm)
+	run_step skip_on_fail -1 build_llvm
+	;;
+esac
+run_step skip_on_fail x build_linux
+run_step skip_on_fail x boot_linux
+run_step reset_on_fail x check_regression
+run_step stop_on_fail x update_baseline
+run_step stop_on_fail x push_baseline
+
+trap "" EXIT
-- 
cgit v1.2.3


From 278e12dee52779d31cc1b10d39f1dda5b814cf48 Mon Sep 17 00:00:00 2001
From: Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org>
Date: Fri, 15 Mar 2019 15:37:34 +0530
Subject: Patch [3/3] Merge tcwg_gnu: Add support for various bootstrap
 configs.

Change-Id: I21fad361d7b12c9a15617069073daf44ca56894b
---
 round-robin.sh    |  37 +++++++++-
 tcwg_gnu-build.sh | 203 ++++++------------------------------------------------
 2 files changed, 56 insertions(+), 184 deletions(-)

diff --git a/round-robin.sh b/round-robin.sh
index 6e669e11..f207d4a0 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -211,7 +211,9 @@ build_abe ()
 
     local component="$1"
 
-    local project stage
+    local project stage action
+    action="build"
+
     case "$component" in
 	stage1)
 	    project=gcc
@@ -221,6 +223,30 @@ build_abe ()
 	    project=gcc
 	    stage="--stage 2"
 	    ;;
+	bootstrap)
+	    project=gcc
+	    stage="--enable bootstrap"
+	    ;;
+	bootstrap_ubsan)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-ubsan"
+	    ;;
+	bootstrap_O3)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-O3"
+	    ;;
+	bootstrap_O1)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-O1"
+	    ;;
+	bootstrap_lto)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-lto"
+	    ;;
+	bootstrap_debug)
+	    project=gcc
+	    stage="--set buildconfig=bootstrap-debug"
+	    ;;
 	*)
 	    project=$component
 	    stage=""
@@ -266,10 +292,15 @@ build_abe ()
     fi
 
     ccache -z
+    local target_opt=""
+    if [ x"${rr[target]}" != x"native" ]; then
+        target_opt="--target $gnu_target"
+    fi
+
     # Run "./abe.sh --build $project".
     ./abe.sh \
-	--build $project \
-	--target $gnu_target \
+	--$action $project \
+	$target_opt \
 	--extraconfigdir config/master \
 	$custom_abe_src_opt \
 	$stage
diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
index 9485e0ba..1a546169 100755
--- a/tcwg_gnu-build.sh
+++ b/tcwg_gnu-build.sh
@@ -15,25 +15,18 @@ rr[mode]="${rr[mode]-baseline}"
 
 # Set custom revision for one of the projects, and use baseline revisions
 # for all other projects.
-rr[ci_project]="${rr[ci_project]-tcwg_kernel}"
+rr[ci_project]="${rr[ci_project]-tcwg_gnu}"
 rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
 rr[reset_baseline]="${rr[reset_baseline]-false}"
 rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
 
-# {toolchain_name}-{toolchain_ver}-{target}-{linux}-{linux_config}
+# {toolchain_name}-{toolchain_ver}-{target}-{bootstrap_config}
 IFS=- read -a ci_config <<EOF
 ${rr[ci_config]}
 EOF
-rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
-rr[release]=${rr[release]-${ci_config[1]}}
-rr[target]=${rr[target]-${ci_config[2]}}
-rr[linux_config]=${rr[linux_config]-${ci_config[4]}}
-
-case "${rr[toolchain]}" in
-    llvm) rr[components]="llvm linux" ;;
-    gnu) rr[components]="binutils gcc linux" ;;
-    *) assert false ;;
-esac
+rr[target]="native"
+bootstrap_config=${bootstrap_config-${ci_config[3]}}
+rr[components]="gcc"
 
 # Use baseline branches by default.
 for c in ${rr[components]}; do
@@ -58,12 +51,8 @@ case "${rr[mode]}" in
 	default_finish_at="update_baseline"
 	;;
     "bisect")
-	case "${rr[toolchain]}:$(print_single_updated_component)" in
-	    gnu:binutils) default_start_at="build_abe-binutils" ;;
-	    gnu:gcc) default_start_at="build_abe-stage1" ;;
-	    gnu:linux) default_start_at="build_linux" ;;
-	    llvm:llvm) default_start_at="build_llvm" ;;
-	    llvm:linux) default_start_at="build_linux" ;;
+	case "$(print_single_updated_component)" in
+	    gcc) default_start_at="build_abe-$bootstrap_config" ;;
 	    *) assert false ;;
 	esac
 	default_finish_at="check_regression"
@@ -79,183 +68,35 @@ fi
 
 run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"
 
-# Build Linux kernel
-build_linux_1 ()
-{
-    (
-    set -euf -o pipefail
-
-    clone_repo linux
-
-    rm -rf $(pwd)/bin
-    mkdir $(pwd)/bin
-
-    local bin cc ld_opt
-
-    case "${rr[toolchain]}" in
-	llvm)
-	    bin="$(pwd)/llvm-install/bin"
-	    cc="$bin/clang"
-	    ;;
-	gnu)
-	    bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin"
-	    cc="$bin/$(print_gnu_target ${rr[target]})-gcc"
-	    ;;
-    esac
-
-    case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in
-        llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;;
-        llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;;
-        *) ld_opt="" ;;
-    esac
-
-    # Use binutils, etc from $bin
-    export PATH="$bin:$PATH"
-
-    # Use ccache only when bisecting linux (or preparing to).
-    # Otherwise the compiler is new in every build and we would
-    # only clobber ccache volume.
-    local ccache=""
-    if [ x"${rr[mode]}" != x"jenkins-full" \
-	  -a x"$(print_single_updated_component)" = x"linux" ]; then
-	ccache="ccache"
-    fi
-    cat > $(pwd)/bin/${rr[target]}-cc <<EOF
-#!/bin/sh
-exec $ccache $cc "\$@"
-EOF
-    chmod +x $(pwd)/bin/${rr[target]}-cc
-
-    # Define make variables.
-    local opts="CC=$(pwd)/bin/${rr[target]}-cc $ld_opt SUBLEVEL=0 EXTRAVERSION=-bisect"
-    if [ x"${rr[target]}" != x"$(uname -m)" ]; then
-	opts="$opts ARCH=$(print_kernel_target ${rr[target]})"
-	opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-"
-	opts="$opts HOSTCC=gcc"
-    fi
-
-    cd linux
-
-    make $opts distclean
-
-    make $opts ${rr[linux_config]}
-    sed -i -e 's:CONFIG_LOCALVERSION_AUTO=y:# CONFIG_LOCALVERSION_AUTO is not set:' .config
-    set +f; rm -f localversion*; set -f
-    make $opts oldconfig
-
-    ccache -z
-    KBUILD_BUILD_TIMESTAMP=0 make $opts -j$(nproc --all) -s -k &
-    local res=0 && wait $! || res=$?
-    ccache -s
-    return $res
-    )
-}
-
-# Build linux and count number successfully built .o files in linux
-build_linux ()
-{
-    (
-    set -euf -o pipefail
-
-    build_linux_1 &
-    local res=0 && wait $! || res=$?
-
-    # Number of .o files created is the main success metric.
-    local linux_n_obj
-    linux_n_obj=$(find linux -name "*.o" | wc -l)
-    echo "linux_n_obj:" >> ${rr[top_artifacts]}/results
-    echo "$linux_n_obj" >> ${rr[top_artifacts]}/results
-
-    if [ $res = 0 ]; then
-	echo "linux build successful:" >> ${rr[top_artifacts]}/results
-	echo "all" >> ${rr[top_artifacts]}/results
-    fi
-
-    return $res
-    )
-}
-
-# Boot linux kernel
-boot_linux ()
-{
-    (
-    set -euf -o pipefail
-
-    local image cpu
-    case ${rr[target]} in
-	aarch64)
-	    image=linux/arch/arm64/boot/Image.gz
-	    cpu="-cpu cortex-a53"
-	    ;;
-	arm)
-	    image=linux/arch/arm/boot/zImage
-	    cpu=""
-	    ;;
-	*) assert false ;;
-    esac
-    timeout 10s qemu-system-${rr[target]} \
-	    -kernel $image -machine virt $cpu -m 512 \
-	    -serial mon:stdio -display none \
-	    -append "console=ttyAMA0 panic=-1" -no-reboot
-
-    echo "linux boot successful:" >> ${rr[top_artifacts]}/results
-    echo "boot" >> ${rr[top_artifacts]}/results
-    )
-}
-
 # Exit with code 0 if no regression compared to base-artifacts/results.
 no_regression_p ()
 {
     (
     set -euf -o pipefail
 
-    local linux_n_obj
-    linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results)
+    # The following check tests for an empty branch, and we return 0 if results
+    # are not present, since that's an inherently better state. If base-artifacts
+    # were not existing, then it would fail in reset_artifacts() stage.
 
-    # Assume worst for non-existent baseline.
-    local base_linux_n_obj="-10"
-    if [ -f base-artifacts/results ]; then
-	base_linux_n_obj=$(tail -n1 base-artifacts/results)
+    if ! [ -f base-artifacts/results ]; then
+	return 0
     fi
 
-    case "$linux_n_obj:$base_linux_n_obj" in
-	boot:*) return 0 ;;
-	*:boot) return 1 ;;
-	all:*) return 0 ;;
-	*:all) return 1 ;;
-	*)
-	    if ! [ "$linux_n_obj" -ge "-10" ]; then
-		# Something is very wrong with result (e.g., it's not a number).
-		return 1
-	    fi
-	    if ! [ "$base_linux_n_obj" -ge "-10" ]; then
-		# Something is very wrong with result (e.g., it's not a number).
-		return 0
-	    fi
+    local build_result_ref build_result_new
+    build_result_ref=$(tail -n1 base-artifacts/results)
+    build_result_new=$(tail -n1 ${rr[top_artifacts]}/results)
 
-	    if [ $linux_n_obj -ge $base_linux_n_obj ]; then
-		return 0
-	    else
-		return 1
-	    fi
-	    ;;
-    esac
+    if [ $build_result_new -lt $build_result_ref ]; then
+	return 1
+    fi
+
+    return 0
     )
 }
 
 run_step stop_on_fail -10 reset_artifacts
-case "${rr[toolchain]}" in
-    gnu)
-	run_step stop_on_fail x prepare_abe
-	run_step skip_on_fail -9 build_abe binutils
-	run_step skip_on_fail -8 build_abe stage1
-	;;
-    llvm)
-	run_step skip_on_fail -1 build_llvm
-	;;
-esac
-run_step skip_on_fail x build_linux
-run_step skip_on_fail x boot_linux
+run_step stop_on_fail x prepare_abe
+run_step skip_on_fail -5 build_abe ${bootstrap_config}
 run_step reset_on_fail x check_regression
 run_step stop_on_fail x update_baseline
 run_step stop_on_fail x push_baseline
-- 
cgit v1.2.3