path: root/round-robin.sh

#!/bin/bash

# shellcheck source=jenkins-helpers.sh
. "$(dirname $0)"/jenkins-helpers.sh

# Round-Robin associative array.
# FIXME: This should be declared when starting a new manifest with
# %%rr[top_artifacts].
declare -gA rr

# Major and minor versions of the manifest.  These are used in
# round-robin-baseline.sh to determine when historic result in base-artifacts/
# needs to be regenerated.
# Mismatch in major numbers means that the historic result may not be
# compatible with the current scripts, and it is OK to drop it.
# Mismatch in minor numbers means that the historic result should be
# compatible with the current scripts, and it should be updated normally.
# In most cases we will be increasing minor numbers to trigger regeneration of
# reports in interesting-commits.git and updating jira cards.
# Mismatch in patch numbers means that the historic results are fully compatible
# with the current scripts. However, the generated notify files are not aligned
# anymore with one of the backend (ex: dashboard), and then need to be regenerated.
#rr[major]="0"
#rr[minor]="0"
#rr[patch]="0"

# PROJECT's git url#branch or url#SHA1 revision parsable by git rev-parse.
# A special value "baseline" means that PROJECT is not being updated
# in this build, and its baseline branch should be used.
# After a successful build round-robin-baseline.sh will update baseline
# branches of all PROJECTs to the current values, thus setting
# a baseline for the next build.
#rr[PROJECT_git]

# PROJECT's git SHA1 revision.  This is used in manifest and overrides
# any branch setting in ${rr[PROJECT_git]}.
#rr[PROJECT_rev]

# Baseline branch name for current configuration.
#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}"

# Run mode: build or bisect.  In bisect mode we do a couple things
# slightly differently (e.g., don't touch repo in clone_repo() ), and
# it is allowed to have only a single component updated.
#rr[mode]="$mode"

# How to handle baseline:
# - onsuccess: update baseline branches of base-artifacts and components' repos
#           on success (generate trigger files for bisection on failure).
# - force: ignore failures in check_regression(), which will make current
#          build successful.  Push our artifacts to the top of base-artifacts/.
# - init:  use "empty" results for base-artifacts, which will make current
#          build successful.  Push our artifacts as the one and only entry.
# - ignore: Do not affect baseline. Useful for developer testing.
#rr[update_baseline]=onsuccess/force/init/ignore


# Target architecture to build for: arm or aarch64.
#rr[target]="$target"

# Top-level artifacts directory.
#rr[top_artifacts]="$top_artifacts"

# Predicate function to determine whether there is regression between
# 2 sets of results.
# shellcheck disable=SC2154
rr[no_regression_p]=no_regression_p

# Hook to break up changed component (see print_changed_components) into
# smaller sets: print one set per line.  By default, breakup into singletons.
# shellcheck disable=SC2154
rr[breakup_changed_components]=breakup_changed_components

# Abe's repository and branch to use for the build.
rr[abe_repo]="https://git-us.linaro.org/toolchain/abe.git"
rr[abe_branch]="master"

# Host compiler defaults to /usr/bin/gcc and g++
rr[host_cc]="/usr/bin/gcc"
rr[host_c++]="/usr/bin/g++"

# Reset artifacts to an empty state.  ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
reset_artifacts ()
{
    (
    set -euf -o pipefail

    # Clean ${rr[top_artifacts]} but preserve
    # - ${rr[top_artifacts]}/console.log and $run_step_artifacts/console.log, which
    # are being written to by run_step().
    # - ${rr[top_artifacts]}/jenkins/*, which is cleaned by tcwg_kernel.yaml.
    # shellcheck disable=SC2154
    fresh_dir $run_step_top_artifacts \
	      $run_step_top_artifacts/console.log \
	      $run_step_artifacts/console.log \
	      $run_step_top_artifacts/manifest.sh \
	      "$run_step_top_artifacts/jenkins/*"

    local branch repo1 repo
    branch="${rr[baseline_branch]}"
    repo1="${branch#linaro-local/ci/}"
    repo="ssh://bkp.tcwglab/home/tcwg-buildslave/base-artifacts/$repo1.git"

    local git_result
    git_result=$(git ls-remote --heads "$repo" "refs/heads/$branch" || true)

    # FIXME: Remove transitional workaround.
    if [ "$git_result" = "" ]; then
	# Try to use old repo
	repo="https://git-us.linaro.org/toolchain/ci/base-artifacts/$repo1.git"
	git_result=$(git ls-remote --heads "$repo" "refs/heads/$branch" || true)
    fi

    if [ "$git_result" = "" ]; then
	echo "WARNING: BASELINE IS NOT FOUND; INITIALIZING AN EMPTY BASELINE"
	rr[update_baseline]="init"
	run_step_patch_env "==rr[update_baseline]" "init"
    fi

    if [ x"${rr[update_baseline]}" = x"init" ]; then
	branch="empty"
	# FIXME: Move empty.git to bkp.tcwglab.
	repo="https://git-us.linaro.org/toolchain/ci/base-artifacts/empty.git"
    fi

    # Clone base-artifacts here so that bisect runs (which skip this step)
    # don't overwrite it.
    # base-artifacts repo is big and changes all the time, so we
    # fetch only the $baseline_branch, instead of all branches.
    rr[base-artifacts_rev]="${rr[base-artifacts_rev]-$branch}"
    clone_or_update_repo base-artifacts "${rr[base-artifacts_rev]}" \
			 "$repo" auto "$branch"

    git_annex_download base-artifacts annex

    if [ -d base-artifacts/git/ ]; then
	# Copy baseline git_url/git_rev settings into the current build,
	# which will then be overwritten in due course by clone_repo()
	# of various components.
	# Note that we need to copy data for all components to correctly handle
	# builds that fail before all their components are checked out.
	# Note that we want to iterate over components (rather than rsync
	# the whole base-artifacts/git/ directory) to avoid copying data for
	# removed components.
	local c
	for c in ${rr[components]}; do
	    get_baseline_git ${c}_url | set_current_git ${c}_url
	    get_baseline_git ${c}_rev | set_current_git ${c}_rev
	done
    else
	# We are in "init" baseline build, apparently.  "Init" builds should
	# have a full set of git data for all components specified on
	# the command line, so that get_baseline_git() is not called.
	mkdir base-artifacts/git
    fi
    )
}

# Clone repository for $project
# $1: Repo / project name
clone_repo ()
{
    (
    set -euf -o pipefail
    local project="$1"

    local url branch

    case "${rr[${project}_git]}" in
	*"#"*)
	    # Fetch from specified remote repo.
	    url="${rr[${project}_git]%#*}"
	    branch="${rr[${project}_git]#*#}"
	    ;;
	"baseline")
	    # Fetch from remote repo specified in the baseline.
	    url=$(get_baseline_git ${project}_url)
	    branch=$(get_baseline_git ${project}_rev)
	    ;;
	*)
	    # Use revision in the existing local repo.
	    # Most likely it is "HEAD" in precommit testing.
	    url=""
	    branch="${rr[${project}_git]}"
	    ;;
    esac

    # Allow manifest override for $url
    url="${rr[${project}_url]-$url}"
    # Allow manifest override for $branch
    branch="${rr[${project}_rev]-$branch}"

    if [ x"${rr[mode]}" = x"bisect" ]; then
	# In bisect mode we rely on round-robin-bisect.sh to arrange
	# all source directories, and here we only clean them.
	# Note that in bisect mode round-robin-bisec.sh passes "_git" spec
	# as url#sha1 so that create_trigger_files() generates trigger-build-*
	# suitable for triggering last_good and first_bad builds.
	git_clean "$project"
    elif [ "$url" = "" ]; then
	# In local mode -- clean the project directory.
	git_clean "$project"
	# Don't use git_checkout(), which prefers remote resolution of refs.
	git -C "$project" checkout --detach "$branch"
    else
	clone_or_update_repo "$project" "$branch" "$url" > /dev/null
    fi

    local cur_rev
    cur_rev=$(git -C $project rev-parse HEAD)
    rr[debug_${project}_date]=$(git -C $project show --no-patch \
				    --pretty="%ct # %cr" HEAD)

    # Store git info in the manifest and git data into artifacts.
    # Git data in artifacts is then used by subsequent builds to fetch
    # baseline commits.
    if [ "$url" != "" ]; then
	echo "$url" | set_current_git ${project}_url
    fi
    echo "$cur_rev" | set_current_git ${project}_rev
    )
}

# Configure ccache wrappers in "$1".
setup_ccache ()
{
    (
    set -euf -o pipefail
    local bin="$1"

    local -a ccache_opts=("CCACHE_BASEDIR=$workspace")
    if [ -d "$HOME/.ccache" ] && ! touch "$HOME/.ccache" 2>/dev/null; then
	# Setup read-only ccache; this is for pre-commit testing.
	# Since this is for ephemeral pre-commit, do not bother about
	# cleaning up temp directory.
	#
	# Note that we use "touch" instead of "test -w" to check writability
	# of $HOME/.ccache.  This is because "test -w" documentation says
	# that "test -w" checks for "w" permission, which is not the same
	# as writability -- e.g., consider read-only filesystems.
	# In practice, "test -w" does seem to check for actual writability,
	# but "touch" is more robust.
	ccache_opts+=("CCACHE_READONLY=true" "CCACHE_NOSTATS=true"
		      "CCACHE_TEMPDIR=$(mktemp -d)")
    fi

    cat > "$bin/gcc" <<EOF
#!/bin/sh
${ccache_opts[@]} exec ccache ${rr[host_cc]} "\$@"
EOF
    chmod +x "$bin/gcc"
    cp "$bin/gcc" "$bin/cc"

    cat > "$bin/g++" <<EOF
#!/bin/sh
${ccache_opts[@]} exec ccache ${rr[host_c++]} "\$@"
EOF
    chmod +x "$bin/g++"
    cp "$bin/g++" "$bin/c++"
    )
}

# Prepare ABE tree for [partial] GNU builds
prepare_abe ()
{
    (
    set -euf -o pipefail

    clone_or_update_repo abe ${rr[abe_branch]} ${rr[abe_repo]} > /dev/null

    # We use our modified version of GCC's comparison script
    clone_or_update_repo gcc-compare-results master \
			 https://git.linaro.org/toolchain/gcc-compare-results.git

    local workspace
    workspace=$(pwd)

    cd abe

    # Add ccache wrappers.
    # shellcheck disable=SC2115
    rm -rf "$(pwd)/bin"
    mkdir "$(pwd)/bin"

    setup_ccache "$(pwd)/bin"

    # Disable building documention.  Apparently, this is one of
    # the most popular ways.
    cat > "$(pwd)/bin/makeinfo" <<EOF
#!/bin/sh
exec true
EOF
    chmod +x "$(pwd)/bin/makeinfo"

    PATH=$(pwd)/bin:$PATH
    export PATH

    ./configure --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref
    )
}

# Create GNU toolchain xfail files
# $1: file to store flaky xfails to.
# $2: file to store baseline xfails to.
build_abe_check_xfails ()
{
    (
    set -euf -o pipefail
    local flaky_tests="$1"
    local baseline_fails="$2"

    local sumfiles="$run_step_top_artifacts/sumfiles"
    if [ -f "$sumfiles/flaky.xfail" ]; then
	    # Add newly-detected flaky tests to the xfails to be used in
	    # tcwg_gnu-build.sh:no_regression_p().
	    # Strictly speaking, this is not necessary, since all tests
	    # detected as flaky in the current run will show up as PASSed
	    # test in the final merged .sum files.
	    echo "# New flaky tests" >> "$flaky_tests"
	    cat "$sumfiles/flaky.xfail" >> "$flaky_tests"
    fi

    # Fetch flaky tests from base-artifacts history.
    echo "# Known flaky tests" >> "$flaky_tests"
    local history_flaky history_root=""
    while read history_flaky; do
	if [ "$history_root" = "" ]; then
	    history_root="$history_flaky"
	    continue
	fi

	(echo; cat "$history_flaky") >> "$flaky_tests"
    done < <(get_git_history 0 base-artifacts sumfiles/flaky.xfail)
    rm -rf "$history_root"

    # Construct $baseline_fails from base-artifacts/sumfiles/.
    # These and $flaky_tests are passed to ABE to speed-up test convergence
    # and then to .sum comparison in tcwg_gnu-build.sh:no_regression_p().
    if [ -d base-artifacts/sumfiles ]; then
	gcc-compare-results/contrib/testsuite-management/validate_failures.py \
	    --build_dir=base-artifacts/sumfiles --produce_manifest \
	    --manifest "$baseline_fails"
    else
	touch "$baseline_fails"
    fi
    )
}

# Build ABE component. Arguments:
#
# build_abe <component> [--build_patch <patch_branch>] [--check_patch <patch_branch>] [--] [ABE arguments]*
#
# Where:
#
# <component>             ABE component to build.
# --build_patch <patch_branch>  Branch with patch to apply before build.
# --check_patch <patch_branch>  Branch with patch to apply before test.
# --                      Separates arguments for build_abe from arguments for
#                         other components.
#
# Any argument not mentioned above is carried over to ABE.
build_abe ()
{
    (
    set -euf -o pipefail

    local component="$1"
    shift

    if [ x"$component" = x"check_gdb" ]; then
	# Limit GDB testsuites to single-thread parallelism.
	# We've tried running GDB testsuites with 16-thread parallelism,
	# but could not shake out flaky tests in the course of several weeks.
	# Try stabilizing GDB testsuites with single-thread parallelism.
	# If this doesn't work, we'll have to look into dejagnu.
	local cpus
	cpus=$(cat abe/host.conf | grep "^cpus=" | sed -e "s/^cpus=\(.*\)/\1/")
	if [ "$cpus" -gt 1 ]; then
	    cp abe/host.conf abe/host.conf.orig
	    sed -i -e "s/^cpus=.*/cpus=1/" abe/host.conf
	fi
    elif [ -f abe/host.conf.orig ]; then
	mv abe/host.conf.orig abe/host.conf
    fi

    local project stage action check
    check=false

    # Check if component starts with an action (eg. "check")
    case "$component" in
	check_*)
	    component="${component#check_}"
	    check=true
	    ;;
    esac

    case "$component" in
	stage1)
	    project=gcc
	    stage="--stage 1"
	    ;;
	stage2)
	    project=gcc
	    stage="--stage 2"
	    ;;
	bootstrap)
	    project=gcc
	    stage="--set buildconfig=bootstrap"
	    ;;
	bootstrap_ubsan)
	    project=gcc
	    stage="--set buildconfig=bootstrap-ubsan"
	    ;;
	bootstrap_O3)
	    project=gcc
	    stage="--set buildconfig=bootstrap-O3"
	    ;;
	bootstrap_O1)
	    project=gcc
	    stage="--set buildconfig=bootstrap-O1"
	    ;;
	bootstrap_lto)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto"
	    ;;
	bootstrap_debug)
	    project=gcc
	    stage="--set buildconfig=bootstrap-debug"
	    ;;
	bootstrap_profiled)
	    project=gcc
	    stage="--set buildconfig=bootstrap --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	bootstrap_profiled_lto)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	bootstrap_profiled_lto_lean)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto-lean --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	*)
	    project=$component
	    stage=""
	    ;;
    esac

    local build_patch=""
    if [ $# -gt 0 ] && [ "$1" = "--build_patch" ]; then
	build_patch="$2"
	shift 2
    fi

    local check_patch=""
    if [ $# -gt 0 ] && [ "$1" = "--check_patch" ]; then
	check_patch="$2"
	shift 2
    fi

    # Finished processing arguments for build_abe. Now look for arguments meant
    # for various components.
    while [ $# -gt 0 ]; do
	if [ x"$1" = x"--" ]; then
	    shift
	    break
	fi
	shift
    done

    local -a rerun_failed_tests=()
    local -a send_results=()
    if $check; then
	# Clean testing results
	rm -rf "$run_step_top_artifacts/sumfiles" \
	   "$run_step_top_artifacts/00-sumfiles"
	mkdir "$run_step_top_artifacts/sumfiles" \
	      "$run_step_top_artifacts/00-sumfiles"

	local flaky_tests="$run_step_artifacts/flaky.xfail"
	local baseline_fails="$run_step_artifacts/baseline.xfail"
	build_abe_check_xfails "$flaky_tests" "$baseline_fails"

	rerun_failed_tests=("--rerun-failed-tests"
			    "--gcc-compare-results" "$PWD/gcc-compare-results"
			    "--flaky-failures" "$flaky_tests"
			    "--expected-failures" "$baseline_fails")

	if [ "${rr[update_baseline]}" != "ignore" ]; then
	    # If we have a chance to commit $new_flaky into base-artifacts.git
	    # then pretend it is 8 weeks into the future and ignore flaky entries
	    # that will be expired by then.  This, effectively, gives us 8 weeks
	    # to re-detect/confirm flaky tests without no_regression_p()
	    # noticing anything.
	    # We will then set expiration date to "now+12 weeks" (see below)
	    # for entries in $new_flaky/"sumfiles/flaky.xfail".
	    local week_from_now
	    week_from_now=$(date -d "now+8 weeks" +%Y%m%d)
	    rerun_failed_tests+=("--failures-expiration-date" "$week_from_now")
	fi

	case "${rr[ci_project]}" in
	    # Don't send results for partial 'make check'
	    *_fast_check_*) ;;
	    *)
		# Default recipient, overriden in round-robin-notify.sh
		send_results=("--send-results-to" "christophe.lyon@linaro.org"
			      "--send-results-filter" "$(pwd)/abe/scripts/testresults2jenkins.sh")
		;;
	esac
    fi

    if [ x"$project" = x"gcc" ]; then
	# Stage1 ignores "--set languages=" option, so this applies only
	# to stage2 builds.
	case "${rr[ci_project]}" in
	    *embed*)
		# Do not build fortran for bare-metal configurations.
		stage="$stage --set languages=c,c++,lto"
		;;
	    *_mingw_*)
		# FIXME: Only C is supported for aarch64-w64-mingw32.
		stage="$stage --set languages=c"
		;;
	    *)
		# Build upstream-default languages (not abe-default languages).
		stage="$stage --set languages=default"
		;;
	esac
    fi

    # Carry over any remaining arguments to ABE.
    if [ $# != 0 ]; then
	stage="$stage $*"
    fi

    action="--build $project"
    # When checking a component we still need to force its build,
    # otherwise ABE does nothing.
    if $check; then
	action="$action --check $project"
    fi

    local custom_abe_src_opt=""
    local git_dir="$project"
    local n_patches=0
    local patch_repo

    case "$project" in
	binutils|gdb)
	    patch_repo="binutils-gdb"
	    ;;
	*)
	    patch_repo="$project"
	    ;;
    esac

    case "$component" in
	# Use our custom sources for everything, but dejagnu.
	dejagnu) ;;
	*)
	    git_dir="$git_dir.git"
	    custom_abe_src_opt="$project=http://git.l.o/$git_dir~master --disable update"

	    if ! $check; then
		clone_repo $project

		if [ x"$build_patch" != x"" ]; then
		    git -C $project fetch \
			"https://git.linaro.org/toolchain/$patch_repo.git" \
			"refs/heads/$build_patch"
		    git -C $project cherry-pick FETCH_HEAD
		    n_patches=1
		fi

		(
		    cd $project

		    # Avoid rebuilding of auto-generated C files. Rather than
		    # try to determine which are auto-generated, touch all of
		    # them.  If a C file is not autogenerated, it does
		    # no harm to update its timestamp.
		    git ls-files -z '*.c' | xargs -r -0 touch

		    # Touch GCC's auto-generated files to avoid
		    # non-determenistic behavior.
		    if [ -x ./contrib/gcc_update ]; then
			./contrib/gcc_update --touch
		    fi
		)

		# Don't use ABE's repo clone functions and setup abe/snapshots/
		# directory to have the right entries.
		local git_path
		git_path="abe/snapshots/$git_dir"
		rm -rf $git_path $git_path~master
		ln -s "$(pwd)/$project" $git_path
		ln -s "$(pwd)/$project" $git_path~master
	    else
		if [ x"$check_patch" != x"" ]; then
		    git -C $project fetch \
			"https://git.linaro.org/toolchain/$patch_repo.git" \
			"refs/heads/$check_patch"
		    git -C $project cherry-pick FETCH_HEAD
		    n_patches=1
		fi
	    fi
	    ;;
    esac

    # FIXME remove debug traces
    set +f
    stat -c "%Y %n" * abe/snapshots/$git_dir~master abe/snapshots/$git_dir~master/ || true
    set -f


    # In precommit testing, enable maintainer_mode so that we
    # regenerate files as needed.  Also update $PATH to include the
    # right versions of autoconf and automake.
    # ${rr[update_baseline]} == "ignore" is an approximate detection
    # of precommit testing mode, since this can also be true while we
    # are bisecting. When bisecting, we want to keep the sources
    # exactly as they were committed, so we don't enable
    # maintainer_mode in this case.
    #
    # FIXME binutils, gdb and gcc are not ready for automatic
    # maintainer_mode.  Disable for all projects for the moment.
    local maintainer_mode=false
    if [ "${rr[update_baseline]}" = "ignore" ] \
	   && [ "${rr[mode]}" != "bisect" ] \
	   && [ "$project" != "binutils" ] \
	   && [ "$project" != "gcc" ] \
	   && [ "$project" != "gdb" ] \
	   && false; then
	maintainer_mode=true
    fi

    # FIXME Test only with fast_* projects for the moment (they have
    # no precommit mode, so enable maintainer_mode in the "normal"
    # jobs.
    if [ "${rr[mode]}" != "bisect" ]; then
	case "${rr[ci_project]}" in
	    *_fast_check_*)
		maintainer_mode=true
		;;
	esac
    fi

    if $maintainer_mode; then
	stage="$stage --enable maintainer_mode"
	# No need to export PATH, it is already exported by parent processes
	PATH=/usr/local/automake-1.15.1/bin:/usr/local/autoconf-2.69/bin:$PATH
	# Remove the fake makeinfo we created in prepare_abe(), so
	# that we can check that docs can be built.
	rm -f "$(pwd)/abe/bin/makeinfo"
    else
	stage="$stage --disable make_docs"
    fi

    cd abe

    # Remove previous build directories and .stamp files.
    # (we rely on ccache for fast rebuilds)
    # These stamp files tell ABE whether it needs to re-run
    # checkout/configure/build steps. Which we do not want
    # to do on a check step.
    if ! $check; then
	set +f; rm -rf builds/*/*/$project-*; set -f
    fi

    PATH=$(pwd)/bin:$PATH
    export PATH

    ccache -z
    local target_opt=""
    if [ x"${rr[target]}" != x"native" ]; then
        target_opt="--target $(print_gnu_target ${rr[target]})"
    fi

    # Run "./abe.sh --build $project".
    # shellcheck disable=SC2206
    local -a abe_cmd=(
	./abe.sh
	$action
	$target_opt
	--extraconfigdir config/master
	$custom_abe_src_opt
	"${rerun_failed_tests[@]}"
	"${send_results[@]}"
	$stage)
    TESTRESULTS_PREFIX=$run_step_artifacts/testresults- "${abe_cmd[@]}" &
    res=0 && wait $! || res=$?

    # FIXME remove debug traces
    set +f
    stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true
    find ../$project -newer builds/*/*/$project-*-configure.stamp || true
    find ../$project -newer builds/*/*/$project-*-build.stamp || true
    set -f

    # If a build without check failed, re-run it with --disable
    # parallel so that errors are easier to extract from the logs.
    if ! $check && [ $res -ne 0 ]; then
	# ABE skips the configure step if $builddir/config.status
	# exists. Remove this file when maintainer-mode is enabled, so
	# that files are regenerated in case something went wrong
	# during the parallel build. We cannot use ABE's --force
	# because it also implies --enable-update.
	if $maintainer_mode; then
	    rm -f builds/*/*/$project-*/config.status
	fi

	abe_cmd+=(--disable parallel)
	"${abe_cmd[@]}" | ts abe-debug-build: &
	res=0 && wait $! || res=$?

	# FIXME remove debug traces
	set +f
	stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true
	set -f

	if [ $res -eq 0 ]; then
	    # Parallel build failed, single-threaded one passed:
	    # consider this "OK" on aarch32, but a failure on other
	    # targets where we do not expect memory exhaustion.
	    if [ "$(getconf LONG_BIT)" = "32" ]; then
		echo "WARNNING: Parallel build failed, single-threaded one passed."
		echo "WARNNING: Considering this build as succesful (likely a transient memory exhaustion caused by a highly parallel build)"
	    else
		if $maintainer_mode; then
		    echo "WARNNING: Parallel build failed, single-threaded one passed. This error is ignored when maintainer-mode is enabled."
		else
		    echo "ERROR: Parallel build failed, single-threaded one passed."
		    res=1
		fi
	    fi
	fi
    fi

    # Revert patches if applied.
    if [ -d ../$project ]; then
	# FIXME remove debug traces
	set +f
	stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true
	git  -C ../$project status
	git  -C ../$project diff HEAD~$n_patches
	set -f

        git -C ../$project reset -q --hard HEAD~$n_patches

	# FIXME remove debug traces
	set +f
	stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true
	find ../$project -newer builds/*/*/$project-*-configure.stamp || true
	find ../$project -newer builds/*/*/$project-*-build.stamp || true
	set -f
    fi

    ccache -s

    # Save logs generated in the current step to artifacts.
    # Note that the logs generated in the previous steps will have .log.xz
    # extension, and would not match in "find".
    local log
    while IFS= read -r -d '' log; do
	rm -f "$log.xz"
	(
	    xz "$log"
	    cp "$log.xz" "$run_step_artifacts/"
	) &
    done < <(find builds/ \( -name "make-*.log" -o -name "check-*.log" \) \
		  -print0)

    # FIXME remove debug traces
    set +f
    stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true
    set -f

    if $check; then
	local sum sumfiles
	while IFS= read -r -d '' sum; do
	    case "$res:$sum" in
		0:*".sum")
		    # Only store sum files in definitive directory if abe
		    # succeeded.
		    sumfiles="$run_step_top_artifacts/sumfiles"
		    ;;
		*)
		    # Store *.sum.N files and .log.xz files in 00-sumfiles/,
		    # so that these non-essential big files are eventually
		    # removed from base-artifacts.git history by
		    # "git filter-repo" in round-robin-baseline.sh's
		    # trim_base_artifacts().
		    sumfiles="$run_step_top_artifacts/00-sumfiles"
		    ;;
	    esac

	    # Remove WORKSPACE prefix instead of making a plain copy
	    sed "s|Running .*/snapshots/|Running |" \
		< "$sum" > "$sumfiles/$(basename "$sum")"

	    log="${sum/.sum/.log}"
	    # Testsuite logs grow 50-400MB in size, so compress them to save
	    # disk space on ci.linaro.org and in base-artifacts.git.
	    # Delete previous XZ'ed log, which can occur when we are bisecting
	    # QEMU and not rebuilding compiler (therefore not cleaning compiler
	    # build/test directory).
	    # Process logs in parallel; "wait" below waits for all to
	    # finish.
	    rm -f "$log.xz"
	    (
		xz "$log"
		cp "$log.xz" "$run_step_top_artifacts/00-sumfiles/"
	    ) &
	done < <(find builds/ \( -name '*.sum' -o -name '*.sum.[0-9]*' \) \
		      -print0)

	# ABE re-wrote the $flaky_tests file to contain entries only
	# for the new flaky tests detected in this run.
	if [ -s "$flaky_tests" ]; then
	    sumfiles="$run_step_top_artifacts/sumfiles"
	    if [ "$res" != 0 ]; then
		sumfiles="$run_step_top_artifacts/00-sumfiles"
	    fi

	    # Mark new flaky entries to expire in 12 weeks.
	    # The upside of expiration is that we will have an up-to-date list
	    # of flaky tests, which we can address.
	    # The downside is that we will spend a bit more CPU cycles
	    # to re-detect flaky tests.
	    local expire
	    expire=$(date -d "now+12 weeks" +%Y%m%d)
	    # A lot of thought went into the following nugget:
	    expire="expire=$expire"
	    sed -i -e "s#^flaky | #flaky,$expire | #" "$flaky_tests"

	    # Move flaky fails to the sumfiles so that they will be
	    # fetched by next run's get_git_history().
	    echo "# From ${BUILD_URL-$(pwd)}:" > "$sumfiles/flaky.xfail"
	    cat "$flaky_tests" >> "$sumfiles/flaky.xfail"
	fi

	if [ $res -eq 0 ] \
	       && [ -f $run_step_artifacts/testresults-mail-body.txt ]; then
	    (
	    # Move testresults files, so that it's easier to find them
	    # later when we want to send them via email.
	    rm -rf $run_step_top_artifacts/testresults
	    mkdir $run_step_top_artifacts/testresults
	    mv $run_step_artifacts/testresults-mail-recipients.txt \
	       $run_step_artifacts/testresults-mail-subject.txt \
	       $run_step_top_artifacts/testresults/
	    # Add a pointer to the build origin, for easier tracking
	    echo "# From ${BUILD_URL-$(pwd)}:" \
		 > $run_step_top_artifacts/testresults/testresults-mail-body.txt
	    cat $run_step_artifacts/testresults-mail-body.txt \
		>> $run_step_top_artifacts/testresults/testresults-mail-body.txt
	    rm $run_step_artifacts/testresults-mail-body.txt
	    ) &
	    if ! wait $!; then
		echo "christophe.lyon@linaro.org" \
		     > artifacts/jenkins/error-mail-recipients.txt
		echo -e "${BUILD_URL-}\nERROR: failed to process testresults" \
		     >> artifacts/jenkins/error-mail-body.txt
	    fi
	fi
    fi

    # Wait for logs to compress.
    wait

    return $res
    )
}

# Print sysroot path under ABE's build tree.
print_abe_sysroot ()
{
    (
    set -euf -o pipefail

    local host target sysroot
    host=$(print_gnu_target native)
    target=$(print_gnu_target ${rr[target]})

    # FIXME: This is a copy of ugly code from abe/lib/globals.sh:
    # init_globals_and_PATH().  Don't ask my why we have different sysroot
    # paths for cross and native cases.
    sysroot="$(pwd)/abe/builds/destdir/$host"
    if [ "$host" != "$target" ]; then
	sysroot="$sysroot/$target"
    fi
    if [ "${rr[target]}" != "woa64" ]; then
	# FIXME: WoA toolchain uses mingw CRT, and this is a quick
	# fix to make it build.  At the moment ABE pretends to use
	# newlib library when building mingw GCC.  See settings of
	# stage2_flags in abe/config/gcc.conf.
	sysroot="$sysroot/libc"
    fi

    echo "$sysroot"
    )
}

# If we bisect a regression between different major versions of Glibc,
# then we might get a mixed sysroot with several versions of ld-M.N.so and
# other binaries installed side-by-side.  Such a sysroot will break
# benchmarking, which requires a single ld-*.so binary to be present.
# Similarly, weird problems can occur if we are bisecting linux
# and re-installing kernel headers one on top another.
# Forcefully delete sysroot before building C library or linux headers.
clean_sysroot ()
{
    (
    set -euf -o pipefail

    rm -rf "$(print_abe_sysroot)"
    )
}

# Build LLVM
build_llvm ()
{
    (
    set -euf -o pipefail

    local projects="${1-clang;lld}"
    local extra_targets="${2-}"
    local metric_id="${3-}"

    clone_repo llvm
    if [ x"$metric_id" = x"num_vect_loops" ]; then
        wget -O llvm-vect-metric.diff "https://git.linaro.org/toolchain/jenkins-scripts.git/plain/downstream_patches/llvm-vect-metric.diff"
        git -C llvm apply "$(pwd)/llvm-vect-metric.diff"
    fi
    sanity_check_pwd

    local workspace
    workspace=$(pwd)

    # Setup ccache and ninja wrappers.
    # shellcheck disable=SC2115
    rm -rf "$(pwd)/bin"
    mkdir "$(pwd)/bin"

    setup_ccache "$(pwd)/bin"

    if [ -f /usr/local/bin/ninja.bin ]; then
	# Use ninja configuration from llvm buildbots to avoid running out of RAM.
	cat > "$(pwd)/bin/ninja" <<EOF
#!/bin/sh
exec /usr/local/bin/ninja.bin -m 30 -M 50 -D 5000 "\$@"
EOF
	chmod +x "$(pwd)/bin/ninja"
    fi

    PATH=$(pwd)/bin:$PATH
    export PATH

    # Freshen up build and install directories.  We rely on ccache for fast
    # rebuilds.
    rm -rf llvm-build llvm-install
    mkdir -p llvm-build
    cd llvm-build

    local llvm_targets
    llvm_targets="$(print_llvm_target ${rr[target]})${extra_targets}"

    cmake -G Ninja  ../llvm/llvm "-DLLVM_ENABLE_PROJECTS=$projects" \
	  -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True \
	  -DCMAKE_INSTALL_PREFIX=../llvm-install \
	  "-DLLVM_TARGETS_TO_BUILD=$llvm_targets" -DCLANG_DEFAULT_LINKER=lld
    ccache -z
    ninja
    ninja install
    ccache -s
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_build_regression_p ()
{
    (
    set -euf -o pipefail

    local ref_artifacts=$1
    local new_artifacts=$2

    # The following check tests for an empty branch, and we return 0 if results
    # are not present, since that's an inherently better state. If base-artifacts
    # were not existing, then it would fail in reset_artifacts() stage.

    if ! [ -f $ref_artifacts/results ]; then
	return 0
    fi

    local build_result_ref build_result_new
    build_result_ref=$(grep -v -E "^#|^$" $ref_artifacts/results | tail -n1)
    build_result_new=$(grep -v -E "^#|^$" $new_artifacts/results | tail -n1)

    if [ $build_result_new -ge $build_result_ref ]; then
	return 0
    fi

    # If we are observing a regression with a negative score, consider results as invalid.
    # Setting EXTERNAL_FAIL will avoid any bisect, and the run will be marked as UNSTABLE.
    # A side-effect of this is lack of reduction to component when a failure to build
    # one of the components will just stop the job in UNSTABLE state and not attempt
    # to update other components.
    if [ $build_result_new -lt 0 ]; then
        return $EXTERNAL_FAIL
    fi

    local last_log
    last_log=$(find $new_artifacts/ -name console.log.xz | sort -g | tail -n1)
    # In log scan for errors below
    #   - sed -e 's/"[^"]*"//g' -- removes quoted "error: app diagnostics" strings
    #   - " error:" detects compiler errors from GCC and Clang (including GCC ICEs)
    #   - "^ERROR:" detects linker errors
    #   - ": undefined reference" detects missing symbols during linking
    #   - "] Error " detects GNU make errors
    # Then grep for "grep" to exclude other uses of this search.
    # Do this twice, once with the abe-debug-build: prefix (where we
    # disabled build parallelism to make debug easier), once without
    # this prefix if for some reason the failure occurred only with
    # parallelism enabled.
    local debug_log
    debug_log=$(mktemp)

    xzcat $last_log | \
	grep abe-debug-build: | \
	sed -e 's/abe-debug-build: //' | \
	sed -e 's/"[^"]*"//g' | \
	grep " error:\|^ERROR:\|: undefined reference\|\] Error " | \
	grep -v "grep" | \
	head | \
	sed -e "s/^/# /" > $debug_log

    cat > $run_step_artifacts/results.regressions <<EOF
# First few build errors in logs:
EOF

    if [ -s $debug_log ]; then
	cat $debug_log >> $run_step_artifacts/results.regressions
    else
	xzcat $last_log | \
	    grep -v abe-debug-build: | \
	    sed -e 's/"[^"]*"//g' | \
	    grep " error:\|^ERROR:\|: undefined reference\|\] Error " | \
	    grep -v "grep" | \
	    head | \
	    sed -e "s/^/# /" >> $run_step_artifacts/results.regressions
    fi
    rm -f $debug_log

    return 1
    )
}

# Generate trigger-build-* and trigger-bisect files to reduce the regression.
# $1: Directory for trigger-* files.
# $2: Score of the build.
create_trigger_files ()
{
    (
    set -euf -o pipefail
    local trigger_dest="$1"
    local score="$2"

    local -a changed_components
    IFS=" " read -r -a changed_components <<< "$(print_changed_components)"

    if [ ${#changed_components[@]} -gt 1 ] \
	   || { [ x"${rr[mode]}" = x"bisect" ] \
		    && ! [ "$score" -lt 0 ] 2>/dev/null; }; then
	# If we have several changed components, then trigger individual builds
	# for these components to narrow down the problem to a single component.
	# Also generate trigger-build-* files as a favor to
	# round-robin-bisect.sh script to distinguish "bad" from "skip" builds
	# and to provide templates for triggering follow-up builds.
	# Note that we can't use "[ "$score" -ge 0 ]" condition above and below
	# because it will give wrong answer for non-numeric scores like "all"
	# and "boot".  "-lt" comparison, however, will produce correct answer,
	# albeit, with an occasional harmless error message.
	local -a update_components

	while read -a update_components; do
	    local c update_components2
	    update_components2=$(echo "${update_components[@]}" | tr ' ' '-')

	    # find the list of components_to_update
	    local -a components_to_update=()
	    for c in ${rr[components]}; do
		if echo "${update_components[@]}" | tr ' ' '\n' \
			| grep "^$c\$" >/dev/null; then
		    assert_with_msg "Should never happen for precommit builds" \
				    [ "${rr[${c}_git]}" != "HEAD" ]
		    components_to_update+=("${c}")
		fi
	    done

	    # print components_to_update to the trigger file
	    if [ "${rr[dynamic_components_list]+abc}" ]; then
		echo "dynamic_components_list=${components_to_update[*]}"
	    else
		for c in "${components_to_update[@]}"; do
		    echo "${c}_git=${rr[${c}_git]}"
		done
	    fi > $trigger_dest/trigger-build-$update_components2

	    # Add update_baseline setting to the parameters, so that
	    # "ignore" builds will trigger "ignore" reduction builds.
	    # Do not set update_baseline for bisect builds, since these
	    # trigger-build-* files will then be used as templates to trigger
	    # last_good and first_bad builds.
	    if [ x"${rr[mode]}" != x"bisect" ]; then
		echo "update_baseline=${rr[update_baseline]}" \
		     >> $trigger_dest/trigger-build-$update_components2
	    fi
	done < <(${rr[breakup_changed_components]})
    elif [ ${#changed_components[@]} = 1 ] \
	     && ! [ "$score" -lt 0 ] 2>/dev/null; then
	local single_component="${changed_components[0]}"

	# Trigger bisect failures in a single changed component in all steps
	# with a positive result.  If $score is less-than 0, then
	# the regression is not very interesting, so don't bisect.

	# Rather than the current git commit sha1, use the original
	# specification which can be a branch name: in case the
	# regression has already been fxed in the branch, we won't
	# bother running a useless bisection.
	local bad_git
	bad_git=$(get_current_manifest "{rr[${single_component}_git]}")

	cat > $trigger_dest/trigger-bisect <<EOF
current_project=$single_component
bad_git=$bad_git
EOF
	if [ -f $run_step_artifacts/extra-bisect-params ]; then
	    cat $run_step_artifacts/extra-bisect-params >> $trigger_dest/trigger-bisect
	fi
    fi
    )
}

# Make sure manifest has all up-to-date rr[] fields that other
# round-robin-*.sh scripts require.  This is an easy way to update manifests
# saved in base-artifacts.git to the latest format by rewriting result history.
finalize_manifest ()
{
    (
    set -euf -o pipefail

    echo "# Saving rr[] in the manifest" | manifest_out
    local field value
    while read -r field; do
	value=$(get_current_manifest "{rr[$field]-}")
	if [ "$value" = "${rr[$field]}" ]; then
	    continue
	fi

	if [ "$value" != "" ]; then
	    echo "# WARNING: overriding previous rr[$field]=$value" \
		| manifest_out
	fi

	cat <<EOF | manifest_out
rr[$field]="${rr[$field]}"
EOF
    done < <(IFS=$'\n'; echo "${!rr[*]}" | sort | grep -v "^top_artifacts\$")
    )
}


# Check if current build regressed compared to the baseline.
#
# As inputs we have:
# $score -- last line of artifacts/results
# $res   -- exit code of no_regression_p()
#
# Additionally, ${rr[update_baseline]} and ${rr[mode]} affect how the rest
# of the build will proceed -- via create_trigger_files().
#
# Decision matrix:
#
# 1. score >= 0 && res == 0: return 0
#    OK; new results good for baseline;
#    update baseline if != ignore (e.g., not bisect test or precommit)
#      - round-robin-notify.sh compares results in artifacts/ vs
#        base-artifacts#HEAD^
#    don't update baseline if == ignore (e.g., bisect test or precommit)
#      - round-robin-notify.sh compares results in artifacts/ vs
#        base-artifacts#HEAD
#    send precommit feedback if precommit testing is active;
#    (see [R] below) send regression report if notify==onregression
#    push baseline if != ignore (e.g., not precommit build);
#    trigger pre-commit testing if SCM build;
#
# 2. score >= 0 && res == E: return E
#    sporadic fail; wait for next build; don't trigger anything;
#    don't update baseline even with force or init;
#    Can happen in, e.g., tcwg_native_build if git server fails while
#    cloning repo for one of the components;
#
# 3. score >= 0 && res != {0,E}: return I (or 0 if update_baseline==force)
#    regression (or expected regression if update_baseline==force);
#    if update_baseline == onsuccess -- trigger reduce or bisect;
#    [R] if update_baseline == force -- succeed, return 0, treat as (1).
#    if update_baseline == ignore -- nothing
#      - if mode==bisect, then create trigger-* files as favor for
#        round-robin-bisect.sh script.
#      - if pre-commit testing is active, then send notifications
#        similar to "update_baseline==force".  See [P] below.
#
# 4. score <  0 && res == 0: return E
#    should not happen; baseline is bad and current build is no better;
#    don't update the baseline or reduce or bisect; send error-mail
#
# 5. score <  0 && res == E: return E
#    sporadic fail; wait for next build; don't trigger anything;
#    don't update baseline even with force or init.
#
# 6. score <  0 && res != {0,E}: return I
#    uninteresting fail; trigger reduce, but no bisect
#    don't update baseline even with force or init.
#    This happens when the build failed before reaching "interesting"
#    phase -- e.g., glibc build failed in a configuration that checks
#    gcc testsuite results in a cross-toolchain.  In this case we want
#    to continue testing and updating baseline when advancing binutils and
#    gcc sources, while waiting for glibc build to get fixed.
#    For this reason we often have matching tcwg_*_build ci_projects for
#    most tcwg_*_check ci_projects.  Non-interesting (for tcwg_*_check) failures
#    are reduced, bisected and reported by tcwg_*_build projects.
#
# 7. score == -E: return E
#    special case indicating a sporadic failure; same as (5) but without
#    calling no_regression_p() to get "res".
#
# "E" -- EXTERNAL_FAIL; "I" -- INTERNAL_FAIL
#
# Notes:
# - If score < 0 then we never update baseline -- even with force or init.
#   This allows us to always have a reasonable "interesting" baseline,
#   which is necessary for pre-commit testing.  E.g., we don't want to
#   pre-commit test upstream patches against a baseline that doesn't build.
#
# - If we return E -- exit, do nothing, wait for the next timed trigger.
#   Hopefully by that time infra fixes itself.
#   In the .yaml files we have a choice on how to implement this:
#   a. Mark the build as FAILURE, which will prevent any subsequent steps
#      from running.  Pros: simple, doesn't require conditional-steps;
#      cons: the build is shown as "red" failure.
#   b. Mark the build as UNSTABLE, and then condition all subsequent steps
#      to run only for current-status==SUCCESS, which is what we are doing
#      now.  Pros: the build is shown as "yellow"; cons: all subsequent
#      steps need to be wrapped in conditional-step.
#
# - [P] It's not clear how to send notifications for failed pre-commit
#   builds.  At the moment we run trigger-followup-builds for failed
#   builds and exit, thus avoiding notify-and-push step for failed builds.
#   It seems we need to
#   a. Split notify-and-push into "notify" and "push"
#   b. Move "notify" before trigger-followup-builds and leave "push"
#      after trigger-followup-builds.
#   c. We need to make sure "notify" does not send anything when it is
#      running after a failed build.
#   d. Trigger of pre-commit testing needs to happen only after "push",
#      so we know that we have a good recent baseline.
#
# - If we return E during precommit testing, the check will be in "pending"
#   state, and testing of the patch will be retriggered on the next round.
#
# - If we return N != {0, I, E}, then the exit was due to script error,
#   so send error-mail and follow case of returning "E".
#
# - If we returned "0", then build should be marked as SUCCESS.
#
# - If we returned "I", then build should be marked FAILURE
#
# - If we returned "E" or anything else, then build should be marked
#   as UNSTABLE.
check_regression ()
{
    (
    set -euf -o pipefail

    local score
    score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)

    if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then
	echo "ERROR: We have encountered some infrastructure problem (e.g.,"
	echo "       benchmarking boards are offline), and we can't finish"
	echo "       the build."

	# Exit now and don't update baseline artifacts.
	# By not creating trigger-build-* files, we signal
	# round-robin-bisect.sh to skip this build/revision.
	return $EXTERNAL_FAIL
    fi

    local res

    # Generate comparison artifacts for update, reset, init and push modes.
    ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} &
    res=0 && wait $! || res=$?

    # Move extra artifacts that no_regression_p generated to $top_artifacts.
    if [ -d $run_step_artifacts/top-artifacts ]; then
	rsync -a $run_step_artifacts/top-artifacts/ ${rr[top_artifacts]}/
	rm -rf $run_step_artifacts/top-artifacts
    fi

    if [ -f $run_step_artifacts/results.regressions ]; then
	# Add regression info generated by no_regression_p to top-level
	# results file.
	cat $run_step_artifacts/results.regressions \
	    >> ${rr[top_artifacts]}/results
    fi

    rr[no_regression_result]="$res"
    finalize_manifest

    if [ $res = 0 ]; then
	if [ "$score" -lt 0 ] 2>/dev/null; then
	    # Case (4) in the comment above.
	    if [ -d ${rr[top_artifacts]}/jenkins ]; then
		echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
                     > artifacts/jenkins/error-mail-recipients.txt
                echo -e "${BUILD_URL-}\nERROR: case (4) in check_regression" \
                     >> artifacts/jenkins/error-mail-body.txt
	    fi
	    return $EXTERNAL_FAIL
	else
	    # All good, no regression
	    return 0
	fi
    elif [ $res = $EXTERNAL_FAIL ]; then
	# Comparison failed to produce a meaningful result
	return $EXTERNAL_FAIL
    fi

    assert_with_msg "no_regression_p should succeed in init baseline mode" \
		    [ x"${rr[update_baseline]}" != x"init" ]

    # We've got a regression.  Generate trigger-* files.
    local trigger_dest
    if [ "${rr[update_baseline]}" = "onsuccess" ] \
	   || [ "${rr[mode]}" = "bisect" ]; then
	# We are seeing a failure, so instead of updating baseline start
	# reducing/bisecting the failure.  Create trigger-* files at top level
	# where jenkins expects them -- and trigger the followup builds.
	trigger_dest="${rr[top_artifacts]}"
    else
	# We don't want to trigger follow up builds when forcing
	# the baseline.  So, for the record, place trigger-* files in
	# the step's artifacts directory.
	trigger_dest="$run_step_artifacts"
    fi

    create_trigger_files "$trigger_dest" "$score"

    if [ "$score" -lt "0" ]; then
	# Case (6) above.
	return $INTERNAL_FAIL
    elif [ "${rr[update_baseline]}" = "force" ]; then
	return 0
    fi

    echo "Detected a regression!"
    return $INTERNAL_FAIL
    )
}