Merge remote-tracking branch 'gerrit/master' into tcwg_bmk

Change-Id: I6bb8e3121ebae9af22c4b9b0593d7a35a2ad35fb

7 files changed, 86 insertions, 28 deletions

diff --git a/MakeRelease.job b/MakeRelease.job
index 83d80f20..6ca48bc3 100755
--- a/MakeRelease.job
+++ b/MakeRelease.job
@@ -99,10 +99,6 @@ else
     rsh="ssh $fileserver"
 fi
 
-if test x"${target}" != x"native" -a x"${target}" != x; then
-    platform="--target ${target}"
-fi
-
 shared="/home/buildslave/workspace/shared/"
 user_snapshots="${user_workspace}/snapshots"
 
@@ -156,6 +152,13 @@ gcc=${gcc_src:+gcc=${gcc_src}}
 srcs="${gcc} ${binutils} ${glibc} ${manifest}"
 logfile=${user_workspace}/MakeRelease-${buildnumber}-$(uname -m).log
 
+# Set ABE's --target setting.  No setting means native.
+# If manifest file is set, then ABE must use its setting.
+target_opt=
+if [ x"${target}" != x"native" -a x"${target}" != x -a x"$manifest" = x"" ]; then
+    target_opt="--target ${target}"
+fi
+
 # Build a binary release tarball
 # Remove logfile if present (for some unknown reason)
 rm -f ${logfile}
@@ -164,7 +167,7 @@ manifests=()
 tarballs=()
 # Canadian cross builds require a Linux hosted cross compiler first
 if test x"${canadian}" = x"true"; then
-    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} ${platform} --build all ${libc} ${extra} --tarbin >> ${logfile}
+    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} $target_opt --build all ${libc} ${extra} --tarbin >> ${logfile}
     abe_ret=$?
     host="--host i686-w64-mingw32"
     manifests+=($(read_var ${user_workspace}/artifacts1.txt manifest))
@@ -177,7 +180,7 @@ fi
 # build the mingw32 compiler only if the previous cross-compiler build was
 # successful.
 if test ${abe_ret} -eq 0; then
-    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} ${platform} ${host} --build all ${libc} ${extra} >> ${logfile}
+    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} $target_opt ${host} --build all ${libc} ${extra} >> ${logfile}
     abe_ret=$?
     manifests+=($(read_var ${user_workspace}/artifacts2.txt manifest))
 fi
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index 0303cfb5..a55cec14 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -1,5 +1,16 @@
 #!/bin/bash
 
+# Print absolute path to a file or directory
+# $1: Path (must exist)
+abs_path ()
+{
+    (
+    set -euf -o pipefail
+
+    echo "$(cd $(dirname "$1") && pwd)/$(basename "$1")"
+    )
+}
+
 # Assert that $@ returns success.
 assert ()
 {
@@ -216,9 +227,12 @@ print_tester_label_for_target ()
 	    # doesn't support KVM.  Test on APM builders for now.
 	    echo "tcwg-apm_64-build"
 	    ;;
-        aarch64-linux*) echo "tcwg-tx1_64-test" ;;
-        armv8l-linux*) echo "tcwg-tx1_32-test" ;;
-        arm-linux*) echo "tcwg-tk1_32-test" ;;
+	# We allocate all TK1/TX1 boards to benchmarking, so use APMs
+	# for cross-testing. This means we no longer test on armv7
+	# hardware.
+        aarch64-linux*) echo "tcwg-apm_64-test" ;;
+        armv8l-linux*) echo "tcwg-apm_32-test" ;;
+        arm-linux*) echo "tcwg-apm_32-test" ;;
     esac
     )
 }
diff --git a/round-robin.sh b/round-robin.sh
index fe0c1a69..95ce74c9 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -278,8 +278,9 @@ build_llvm ()
     clone_repo llvm
 
     # Copy only components from the monorepo that are required for kernel build
-    rsync -a --del --exclude /tools/clang llvm/llvm/ llvm-src/
+    rsync -a --del --exclude /tools/clang --exclude /tools/lld llvm/llvm/ llvm-src/
     rsync -a --del llvm/clang/ llvm-src/tools/clang/
+    rsync -a --del llvm/lld/ llvm-src/tools/lld/
 
     # Setup ccache and ninja wrappers.
     rm -rf $(pwd)/bin
@@ -355,8 +356,8 @@ check_regression ()
 
 	    cat > ${rr[top_artifacts]}/trigger-bisect <<EOF
 current_project=$single_component
-baseline_rev=$baseline_rev
-bad_rev=$cur_rev
+baseline_branch=$baseline_rev
+bad_branch=$cur_rev
 EOF
 	fi
 
diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh
index 86fc04b8..cc748c5c 100755
--- a/tcwg-cleanup-stale-containers.sh
+++ b/tcwg-cleanup-stale-containers.sh
@@ -148,7 +148,7 @@ status=$(($status|(2*$res)))
 rm_volumes=($($DOCKER volume ls -q -f dangling=true))
 # Filter-out named volumes like host-home and home-$USER.  Leave only volumes
 # named like a sha1 hash.
-rm_volumes=($(echo "${rm_volumes[@]}" | grep "^[a-f0-9]\{64\}\$" | cat))
+rm_volumes=($(echo "${rm_volumes[@]}" | tr " " "\n" | grep "^[a-f0-9]\{64\}\$" | cat))
 
 if [ ${#rm_volumes[@]} != 0 ]; then
     echo "Removing dangling volumes"
@@ -200,6 +200,10 @@ fi
 if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then
     res=0; killall --older-than ${cleanup_ssh_agent_hours}h -u $USER ssh-agent &
     wait $! || res=$?
+    if [ $res != 0 ]; then
+	echo "WARNING: could not kill stale ssh-agent processes"
+	status=$(($status|16))
+    fi
 fi
 
 # Check if we have more containers than max_containers
@@ -207,7 +211,7 @@ nb_containers=$($DOCKER ps -a | wc -l)
 
 if [ ${max_containers} -gt 0 -a ${nb_containers} -gt ${max_containers} ]; then
     echo "ERROR: Too many containers left after cleanup: ${nb_containers} (max: ${max_containers})"
-    status=$(($status|16))
+    status=$(($status|32))
 fi
 
 exit $status
diff --git a/tcwg-dev-build.sh b/tcwg-dev-build.sh
index cf24ab60..ff4afacc 100755
--- a/tcwg-dev-build.sh
+++ b/tcwg-dev-build.sh
@@ -8,6 +8,7 @@ convert_args_to_variables "$@"
 
 abe_branch="${abe_branch-tested}"
 dry_run="${dry_run-false}"
+manifest="${manifest-}"
 release_name="${release_name-default}"
 target="${target-aarch64-linux-gnu}"
 version="${version-default}"
@@ -23,6 +24,11 @@ if [ x"$release_name" = x"default" ]; then
     release_name="$(date +%Y%m%d-%H_%M_%S)"
 fi
 
+manifest_opt=
+if [ -f "$manifest" ]; then
+    manifest_opt="--manifest $(abs_path "$manifest")"
+fi
+
 manifest_validation_opt=
 if [ x"$target" = x"native" ]; then
     manifest_validation_opt="--manifest_validation false"
@@ -35,6 +41,7 @@ $scripts/MakeRelease.job \
     --target $target \
     --toolchainconfig $version \
     --workspace `pwd` \
+    $manifest_opt \
     $manifest_validation_opt \
     ${binutils:+--binutils "$binutils"} \
     ${gcc:+--gcc "$gcc"}
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 7fb28b17..136dc24c 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -16,7 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"
 convert_args_to_variables "$@"
 shift "$SHIFT_CONVERTED_ARGS"
 
-obligatory_variables bad_rev baseline_rev build_script current_project
+obligatory_variables bad_branch baseline_branch build_script current_project
 
 BUILD_URL="${BUILD_URL:-$(pwd)}"
 reproduce_bisect="${reproduce_bisect:-false}"
@@ -43,7 +43,8 @@ case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
 	# Search for regressions against linux-mainline:master (aka linux-next:stable).
 	clone_or_update_repo $current_project stable ${rr[linux_url]}
 	# Just in case linux-next:stable has advanced between the build and bisect jobs,
-	# use merge base between linux-next:stable and $bad_rev.
+	# use merge base between linux-next:stable and $bad_branch.
+	bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}"
 	linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}"
 	cat <<EOF | manifest_out
 declare -g linux_next_stable=$linux_next_stable
@@ -56,20 +57,21 @@ esac
 
 # Build baseline that we are going to re-use to speed-up bisection.
 # (This also confirms that infrastructure is OK.)
-echo "Testing baseline_rev $baseline_rev (should be success)"
+echo "Testing baseline_branch $baseline_branch (should be success)"
 $build_script \
     ^^ $reproduce_bisect \
     %% $rel_artifacts/manifests/build-baseline.sh \
     @@ $rel_artifacts/manifests/build-parameters.sh \
     ==rr[mode] "baseline" \
-    ==rr[${current_project}_branch] "$baseline_rev" \
+    ==rr[${current_project}_branch] "$baseline_branch" \
     ==rr[reset_baseline] true \
     ==rr[top_artifacts] "$rel_artifacts/build-baseline" \
     --verbose "$verbose"
 
-assert ! [ -f $artifacts/failures ]
-
-cd $current_project
+baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}"
+cat <<EOF | manifest_out
+declare -g baseline_rev=$baseline_rev
+EOF
 
 ln -f -s "build-baseline" "$artifacts/build-$baseline_rev"
 ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh"
@@ -134,6 +136,13 @@ fi
 EOF
 chmod +x $artifacts/test.sh
 
+bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}"
+cat <<EOF | manifest_out
+declare -g bad_rev=$bad_rev
+EOF
+
+cd $current_project
+
 git checkout --detach $bad_rev
 $artifacts/test.sh &
 res=0 && wait $! || res=$?
@@ -143,8 +152,21 @@ if [ x"$res" = x"0" ]; then
     if $rebase_workaround; then
 	echo "Rebase workaround: no regression between $baseline_rev and $bad_rev"
 	sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name
-	cat > $artifacts/trigger-build-rebase <<EOF
+	case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
+	    tcwg_kernel/llvm-*-next-*:linux)
+		cat > $artifacts/trigger-build-rebase <<EOF
 llvm_branch=baseline
+EOF
+		;;
+	    tcwg_kernel/gnu-*-next-*:linux)
+		cat > $artifacts/trigger-build-rebase <<EOF
+binutils_branch=baseline
+gcc_branch=baseline
+EOF
+		;;
+	    *) assert false ;;
+	esac
+	cat >> $artifacts/trigger-build-rebase <<EOF
 linux_branch=$baseline_rev
 reset_baseline=true
 EOF
@@ -205,8 +227,8 @@ if [ x"$res" = x"0" ]; then
 	assert git merge-base --is-ancestor $last_good $bad_rev
 	cat > $artifacts/trigger-bisect <<EOF
 current_project=$current_project
-baseline_rev=$baseline_rev
-bad_rev=$last_good
+baseline_branch=$baseline_rev
+bad_branch=$last_good
 EOF
 	# Don't send any emails.
 	echo > $artifacts/jenkins/mail-recipients.txt
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index 130e3403..9485e0ba 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -25,6 +25,7 @@ IFS=- read -a ci_config <<EOF
 ${rr[ci_config]}
 EOF
 rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
+rr[release]=${rr[release]-${ci_config[1]}}
 rr[target]=${rr[target]-${ci_config[2]}}
 rr[linux_config]=${rr[linux_config]-${ci_config[4]}}
 
@@ -60,9 +61,9 @@ case "${rr[mode]}" in
 	case "${rr[toolchain]}:$(print_single_updated_component)" in
 	    gnu:binutils) default_start_at="build_abe-binutils" ;;
 	    gnu:gcc) default_start_at="build_abe-stage1" ;;
-	    gnu:linux) default_start_at="count_linux_objs" ;;
+	    gnu:linux) default_start_at="build_linux" ;;
 	    llvm:llvm) default_start_at="build_llvm" ;;
-	    llvm:linux) default_start_at="count_linux_objs" ;;
+	    llvm:linux) default_start_at="build_linux" ;;
 	    *) assert false ;;
 	esac
 	default_finish_at="check_regression"
@@ -89,7 +90,7 @@ build_linux_1 ()
     rm -rf $(pwd)/bin
     mkdir $(pwd)/bin
 
-    local bin cc
+    local bin cc ld_opt
 
     case "${rr[toolchain]}" in
 	llvm)
@@ -102,6 +103,12 @@ build_linux_1 ()
 	    ;;
     esac
 
+    case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in
+        llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;;
+        llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;;
+        *) ld_opt="" ;;
+    esac
+
     # Use binutils, etc from $bin
     export PATH="$bin:$PATH"
 
@@ -120,7 +127,7 @@ EOF
     chmod +x $(pwd)/bin/${rr[target]}-cc
 
     # Define make variables.
-    local opts="CC=$(pwd)/bin/${rr[target]}-cc SUBLEVEL=0 EXTRAVERSION=-bisect"
+    local opts="CC=$(pwd)/bin/${rr[target]}-cc $ld_opt SUBLEVEL=0 EXTRAVERSION=-bisect"
     if [ x"${rr[target]}" != x"$(uname -m)" ]; then
 	opts="$opts ARCH=$(print_kernel_target ${rr[target]})"
 	opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-"