path: root/round-robin.sh

#!/bin/bash

# shellcheck source=jenkins-helpers.sh
. "$(dirname $0)"/jenkins-helpers.sh

# Round-Robin associative array.
declare -gA rr

# PROJECT's git url#branch or url#SHA1 revision parsable by git rev-parse.
# A special value "baseline" means that PROJECT is not being updated
# in this build, and its baseline branch should be used.
# In a successful build "update_baseline" step will update baseline
# branches of all PROJECTs to the current values, thus setting
# a baseline for the next build.
#rr[PROJECT_git]

# PROJECT's git SHA1 revision.  These are mostly used in manifests.
#rr[PROJECT_rev]

# Baseline branch name for current configuration.  These branches should
# be present in all above git repos (if ${rr[init_configuration]} is false).
#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}"

# Run mode: bisect or non-bisect.  In bisect mode we do a couple things
# slightly differently (e.g., don't touch repo in clone_repo() ).
#rr[mode]="$mode"

# How to handle baseline:
# - init:  use "empty" results for base-artifacts, which will make current
#          build successful.  Push our artifacts as the one and only entry.
# - update: update baseline branches of base-artifacts and components' repos
#           on success (generate trigger files for bisection on failure).
# - reset: ignore failures in check_regression(), which will make current
#          build successful.  Push our artifacts to the top of base-artifacts/.
# - push:  push results from current build (whatever they are, regressions are
#          ignored) as new commit to base-artifacts, and update baseline
#          branches.  This is useful for user projects, e.g., to generate
#          historical results for several toolchain versions.
#rr[update_baseline]=update/reset/init/push

# Target architecture to build for: arm or aarch64.
#rr[target]="$target"

# Top-level artifacts directory.
#rr[top_artifacts]="$top_artifacts"

# Predicate function to determine whether there is regression between
# 2 sets of results.
# shellcheck disable=SC2154
rr[no_regression_p]=no_regression_p

# Hook to break up updated component (see print_updated_components) into
# smaller sets: print one set per line.  By default, breakup into singletons.
# shellcheck disable=SC2154
rr[breakup_updated_components]=breakup_updated_components

# Print round-robin components that are being updated in this build
# (the ones using non-baseline branches).
print_updated_components ()
{
    (
    set -euf -o pipefail

    local delim=""
    local c
    for c in ${rr[components]}; do
	if [ x"${rr[${c}_git]}" != x"baseline" ]; then
	    echo -ne "$delim$c"
	    delim=${1- }
	fi
    done
    echo
    )
}

# By default, print each component on its own line.
breakup_updated_components ()
{
    print_updated_components "\n"
}

# Print the single round-robin component being updated in this build.
# Print nothing if multiple components are being updated.
print_single_updated_component ()
{
    (
    set -euf -o pipefail

    local update_components
    IFS=" " read -r -a updated_components <<< "$(print_updated_components)"

    if [ ${#updated_components[@]} -eq 1 ]; then
	echo "${updated_components[0]}"
    fi
    )
}

# Reset artifacts to an empty state.  ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
reset_artifacts ()
{
    (
    set -euf -o pipefail

    # Clean ${rr[top_artifacts]} but preserve
    # - ${rr[top_artifacts]}/console.log and $run_step_artifacts/console.log, which
    # are being written to by run_step().
    # - ${rr[top_artifacts]}/jenkins/*, which is cleaned by tcwg_kernel.yaml.
    # shellcheck disable=SC2154
    fresh_dir $run_step_top_artifacts \
	      $run_step_top_artifacts/console.log \
	      $run_step_artifacts/console.log \
	      "$run_step_top_artifacts/jenkins/*"

    # Clone base-artifacts here so that bisect runs (which skip this step)
    # don't overwrite it.
    # base-artifacts repo is big and changes all the time, so we
    # fetch only the $baseline_branch, instead of all branches.
    local single_branch
    if [ x"${rr[update_baseline]}" = x"init" ]; then
	single_branch=empty
    else
	single_branch=${rr[baseline_branch]}
    fi
    rr[base-artifacts_rev]="${rr[base-artifacts_rev]-$single_branch}"

    clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch
    cat <<EOF | manifest_out
rr[base-artifacts_rev]=$(git -C base-artifacts rev-parse HEAD)
EOF
    )
}

# Clone repository for $project
# $1: Repo / project name
clone_repo ()
{
    (
    set -euf -o pipefail
    local project="$1"

    if [ x"${rr[mode]}" = x"bisect" ]; then
	# Cleanup current checkout in bisect mode.
	git_clean "$project"
	return 0
    fi

    local url branch

    if [ x"${rr[${project}_git]}" != x"baseline" ]; then
	# Fetch and checkout from the specified repo.
	url="${rr[${project}_git]%#*}"
	branch="${rr[${project}_git]#*#}"
    else
	# Fetch and checkout from baseline repo.
	url=$(print_baseline_repo "$project" true)
	branch="${rr[baseline_branch]}"
    fi

    clone_or_update_repo_no_checkout "$project" "$url" auto "" origin \
				     > /dev/null

    # Allow manifest override
    branch="${rr[${project}_rev]-$branch}"
    git_checkout "$project" "$branch" origin

    local cur_rev
    cur_rev=$(git -C $project rev-parse HEAD)

    cat <<EOF | manifest_out
rr[${project}_rev]=$cur_rev
EOF
    )
}

# Prepare ABE tree for [partial] GNU builds
prepare_abe ()
{
    (
    set -euf -o pipefail

    clone_or_update_repo abe tested https://git-us.linaro.org/toolchain/abe.git > /dev/null

    local workspace
    workspace=$(pwd)

    cd abe

    # Add ccache wrappers.
    # shellcheck disable=SC2115
    rm -rf "$(pwd)/bin"
    mkdir "$(pwd)/bin"

    cat > "$(pwd)/bin/gcc" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/gcc "\$@"
EOF
    chmod +x "$(pwd)/bin/gcc"
    cp "$(pwd)/bin/gcc" "$(pwd)/bin/cc"

    cat > "$(pwd)/bin/g++" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/g++ "\$@"
EOF
    chmod +x "$(pwd)/bin/g++"
    cp "$(pwd)/bin/g++" "$(pwd)/bin/c++"

    # Disable building documention.  Apparently, this is one of
    # the most popular ways.
    cat > "$(pwd)/bin/makeinfo" <<EOF
#!/bin/sh
exec true
EOF
    chmod +x "$(pwd)/bin/makeinfo"

    PATH=$(pwd)/bin:$PATH
    export PATH

    ./configure --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref
    )
}

# Build ABE component
# $1: Component -- ABE component to build.
build_abe ()
{
    (
    set -euf -o pipefail

    local component="$1"

    if [ x"$component" = x"check_gdb" ]; then
	# Limit GDB testsuites to single-thread parallelism.
	# We've tried running GDB testsuites with 16-thread parallelism,
	# but could not shake out flaky tests in the course of several weeks.
	# Try stabilizing GDB testsuites with single-thread parallelism.
	# If this doesn't work, we'll have to look into dejagnu.
	local cpus
	cpus=$(cat abe/host.conf | grep "^cpus=" | sed -e "s/^cpus=\(.*\)/\1/")
	if [ "$cpus" -gt 1 ]; then
	    cp abe/host.conf abe/host.conf.orig
	    sed -i -e "s/^cpus=.*/cpus=1/" abe/host.conf
	fi
    elif [ -f abe/host.conf.orig ]; then
	mv abe/host.conf.orig abe/host.conf
    fi

    local project stage action check
    check=false

    # Check if component starts with an action (eg. "check")
    case "$component" in
	check_*)
	    component="${component#check_}"
	    check=true
	    ;;
    esac

    case "$component" in
	stage1)
	    project=gcc
	    stage="--stage 1"
	    ;;
	stage2)
	    project=gcc
	    stage="--stage 2"
	    ;;
	bootstrap)
	    project=gcc
	    stage="--set buildconfig=bootstrap"
	    ;;
	bootstrap_ubsan)
	    project=gcc
	    stage="--set buildconfig=bootstrap-ubsan"
	    ;;
	bootstrap_O3)
	    project=gcc
	    stage="--set buildconfig=bootstrap-O3"
	    ;;
	bootstrap_O1)
	    project=gcc
	    stage="--set buildconfig=bootstrap-O1"
	    ;;
	bootstrap_lto)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto"
	    ;;
	bootstrap_debug)
	    project=gcc
	    stage="--set buildconfig=bootstrap-debug"
	    ;;
	bootstrap_profiled)
	    project=gcc
	    stage="--set buildconfig=bootstrap --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	bootstrap_profiled_lto)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	bootstrap_profiled_lto_lean)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto-lean --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	*)
	    project=$component
	    stage=""
	    ;;
    esac

    while [ $# -gt 0 ]; do
	if [ x"$1" = x"--" ]; then
	    shift
	    break
	fi
	shift
    done

    local patch_branch=""
    if [ $# -gt 0 ] && [ x"$1" = x"--patch" ]; then
	patch_branch="$2"
	shift 2
    fi

    if [ x"$project" = x"gcc" ]; then
	# Don't build Go, which has a funky and unstable testsuite.
	# Stage1 ignores "--set languages=" option, so this applies only
	# to stage2 builds.
	stage="$stage --set languages=c,c++,fortran,lto"
	if [ $# != 0 ]; then
	    stage="$stage $*"
	fi
    fi

    action="--build $project"
    # When checking a component we still need to force its build,
    # otherwise ABE does nothing.
    if $check; then
	action="$action --check $project"
    fi

    local custom_abe_src_opt=""
    local git_dir="$project"
    local n_patches=0
    case "$component" in
	# Use our custom sources for everything, but dejagnu.
	dejagnu) ;;
	*)
	    git_dir="$git_dir.git"
	    custom_abe_src_opt="$project=http://git.l.o/$git_dir~master --disable update"

	    if ! $check; then
		clone_repo $project

		if [ x"$patch_branch" != x"" ]; then
		    git -C $project fetch "https://git.linaro.org/toolchain/$project.git" "refs/heads/$patch_branch"
		    git -C $project cherry-pick FETCH_HEAD
		    n_patches=1
		fi

		# Don't use ABE's repo clone functions and setup abe/snapshots/
		# directory to have the right entries.
		local git_path
		git_path="abe/snapshots/$git_dir"
		rm -rf $git_path $git_path~master
		ln -s "$(pwd)/$project" $git_path
		ln -s "$(pwd)/$project" $git_path~master
	    fi
	    ;;
    esac

    cd abe

    # Remove previous build directories and .stamp files.
    # (we rely on ccache for fast rebuilds)
    # These stamp files tell ABE whether it needs to re-run
    # checkout/configure/build steps. Which we do not want
    # to do on a check step.
    if ! $check; then
	set +f; rm -rf builds/*/*/$project-*; set -f
    fi

    PATH=$(pwd)/bin:$PATH
    export PATH

    ccache -z
    local target_opt=""
    if [ x"${rr[target]}" != x"native" ]; then
        target_opt="--target $(print_gnu_target ${rr[target]})"

	# Disable TSan execution tests when using QEMU.
	# QEMU can't handle TSan's shadow memory and it sends host machine
	# into swap.
	if $check && [ x"$project" = x"gcc" ]; then
	    if echo "$stage" | grep -q -e "--testcontainer"; then
		assert_with_msg "ERROR: Testing is not using QEMU" false
	    fi
	    sed -i \
		-e 's/set dg-do-what-default run/set dg-do-what-default link/' \
		../gcc/gcc/testsuite/lib/tsan-dg.exp
	fi
    fi

    # Run "./abe.sh --build $project".
    ./abe.sh \
	$action \
	$target_opt \
	--extraconfigdir config/master \
	--disable make_docs \
	$custom_abe_src_opt \
	$stage &
    res=0 && wait $! || res=$?

    # Revert patches if applied.
    if [ -d ../$project ]; then
        git -C ../$project reset --hard HEAD~$n_patches
    fi

    # If abe failed to build component, return exit status.
    if [ x"$res" != x"0" ]; then
        return $res
    fi
    ccache -s

    if $check; then
	local sum log
	rm -rf ${rr[top_artifacts]}/sumfiles
	mkdir -p ${rr[top_artifacts]}/sumfiles
	while IFS= read -r -d '' sum; do
	    cp "$sum" ${rr[top_artifacts]}/sumfiles/
	    log="${sum%.sum}.log"
	    # Testsuite logs grow 50-400MB in size, so compress them to save
	    # disk space on ci.linaro.org.
	    # Delete previous XZ'ed log, which can occur when we are bisecting
	    # QEMU and not rebuilding compiler (therefore not cleaning compiler
	    # build/test directory).
	    rm -f "$log.xz"
	    (
		xz "$log"
		cp "$log.xz" ${rr[top_artifacts]}/sumfiles/
	    ) &
	done < <(find builds/ \( -name "*.sum" -not -path "*/gdb/testsuite/outputs/*" \) -print0)
	# Wait for logs to compress
	wait
    fi
    )
}

# If we bisect a regression between different major versions of Glibc,
# then we might get a mixed sysroot with several versions of ld-M.N.so and
# other binaries installed side-by-side.  Such a sysroot will break
# benchmarking, which requires a single ld-*.so binary to be present.
# Similarly, weird problems can occur if we are bisecting linux
# and re-installing kernel headers one on top another.
# Forcefully delete sysroot before building C library or linux headers.
clean_sysroot ()
{
    (
    set -euf -o pipefail

    local gnu_host gnu_target sysroot
    gnu_host=$(print_gnu_target native)
    gnu_target=$(print_gnu_target ${rr[target]})
    sysroot="$(pwd)/abe/builds/destdir/$gnu_host/$gnu_target/libc"

    rm -rf "$sysroot"
    )
}

# Build LLVM
build_llvm ()
{
    (
    set -euf -o pipefail

    local use_abe=${1-false}

    clone_repo llvm

    sanity_check_pwd

    local workspace
    workspace=$(pwd)

    # Setup ccache and ninja wrappers.
    # shellcheck disable=SC2115
    rm -rf "$(pwd)/bin"
    mkdir "$(pwd)/bin"

    cat > "$(pwd)/bin/cc" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/gcc "\$@"
EOF
    chmod +x "$(pwd)/bin/cc"

    cat > "$(pwd)/bin/c++" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/g++ "\$@"
EOF
    chmod +x "$(pwd)/bin/c++"

    if [ -f /usr/local/bin/ninja.bin ]; then
	# Use ninja configuration from llvm buildbots to avoid running out of RAM.
	cat > "$(pwd)/bin/ninja" <<EOF
#!/bin/sh
exec /usr/local/bin/ninja.bin -m 30 -M 50 -D 5000 "\$@"
EOF
	chmod +x "$(pwd)/bin/ninja"
    fi

    PATH=$(pwd)/bin:$PATH
    export PATH

    local binutils_incdir
    # Freshen up build and install directories.  We rely on ccache for fast rebuilds.
    if $use_abe; then
	local gnu_host
	gnu_host=$(print_gnu_target native)
	rsync -a --del abe/builds/destdir/$gnu_host/ llvm-install/
	binutils_incdir=$(pwd)/binutils/include
    else
	rm -rf llvm-install
	binutils_incdir=/usr/include
    fi
    rm -rf llvm-build
    mkdir -p llvm-build
    cd llvm-build

    cmake -G Ninja  ../llvm/llvm "-DLLVM_ENABLE_PROJECTS=clang;lld" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install "-DLLVM_TARGETS_TO_BUILD=$(print_llvm_target ${rr[target]})" -DLLVM_BINUTILS_INCDIR=$binutils_incdir
    ccache -z
    ninja
    ninja install
    ccache -s
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_build_regression_p ()
{
    (
    set -euf -o pipefail

    local ref_artifacts=$1
    local new_artifacts=$2

    # The following check tests for an empty branch, and we return 0 if results
    # are not present, since that's an inherently better state. If base-artifacts
    # were not existing, then it would fail in reset_artifacts() stage.

    if ! [ -f $ref_artifacts/results ]; then
	return 0
    fi

    local build_result_ref build_result_new
    build_result_ref=$(grep -v "^#" $ref_artifacts/results | tail -n1)
    build_result_new=$(grep -v "^#" $new_artifacts/results | tail -n1)

    if [ $build_result_new -ge $build_result_ref ]; then
	return 0
    fi

    local last_log
    last_log=$(find $new_artifacts/ -name console.log.xz | sort -g | tail -n1)
    # In log scan for errors below
    #   - sed -e 's/"[^"]*"//g' -- removes quoted "error: app diagnostics" strings
    #   - " error:" detects compiler errors from GCC and Clang (including GCC ICEs)
    #   - "^ERROR:" detects linker errors
    #   - ": undefined reference" detects missing symbols during linking
    #   - "] Error " detects GNU make errors
    # Then grep for "grep" to exclude other uses of this search.
    cat > $run_step_artifacts/results.regressions <<EOF
# First few build errors in logs:
$(xzcat $last_log | sed -e 's/"[^"]*"//g' | grep " error:\|^ERROR:\|: undefined reference\|\] Error " | grep -v "grep" | head | sed -e "s/^/# /")
EOF
    return 1
    )
}

# Check if current build regressed compared to the baseline
# (unless ${rr[update_baseline]} is set to "reset").
check_regression ()
{
    (
    set -euf -o pipefail

    local score
    score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)

    if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then
	echo "ERROR: We have encountered some infrastructure problem (e.g.,"
	echo "       benchmarking boards are offline), and we can't finish"
	echo "       the build."
	# Exit now and don't update baseline artifacts.
	# By not creating trigger-build-* files, we signal
	# round-robin-bisect.sh to skip this build/revision.
	exit $EXTERNAL_FAIL
    fi

    local res

    # Generate comparison artifacts for update, reset, init and push modes.
    ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} &
    res=0 && wait $! || res=$?

    # Move extra artifacts that no_regression_p generated to $top_artifacts.
    if [ -d $run_step_artifacts/top-artifacts ]; then
	rsync -a $run_step_artifacts/top-artifacts/ ${rr[top_artifacts]}/
	rm -rf $run_step_artifacts/top-artifacts
    fi

    if [ $res = 0 ]; then
	# All good, no regression
	return
    fi

    if [ -f $run_step_artifacts/results.regressions ]; then
	# Add regression info generated by no_regression_p to top-level
	# results file.
	cat $run_step_artifacts/results.regressions \
	    >> ${rr[top_artifacts]}/results
    fi

    # We've got a regression.  Generate trigger-* files.
    local single_component
    single_component=$(print_single_updated_component)
    local trigger_dest

    if [ x"${rr[update_baseline]}" = x"update" ]; then
	# When "updating" baseline place trigger-* files at top level
	# where jenkins expects them -- and trigger the followup builds.
	trigger_dest="${rr[top_artifacts]}"
    else
	# We don't want to trigger follow up builds when "pushing"
	# baseline.  So, for the record, place trigger-* files in
	# the step's artifacts directory.
	trigger_dest="$run_step_artifacts"
    fi

    # 1. If $score is less-than 0, then the regression is not very
    # interesting, so reduce down to component, but don't bisect.  This
    # allows non-broken components to advance their revisions.
    # 2. Otherwise, trigger builds with narrowed-down list of updated
    # components -- by generating trigger-build-* files.  Also generate
    # these files as a favor to round-robin-bisect.sh script.
    # 3. Otherwise, we have narrowed-down regression to a single component,
    # so trigger bisect build for this component -- by generating
    # trigger-bisect file.  Note that the negative "! [ $score -lt 0 ]"
    # condition is to handle non-numeric scores like "all" and "boot".
    if [ $score -lt 0 ] 2>/dev/null && [ x"${rr[mode]}" = x"bisect" ]; then
	# Don't bisect uninteresting regressions.
	:
    elif [ x"$single_component" = x"" ] || [ x"${rr[mode]}" = x"bisect" ]; then
	local -a update_components

	while read -a update_components; do
	    local c update_components2
	    update_components2=$(echo "${update_components[@]}" | tr ' ' '-')

	    for c in ${rr[components]}; do
		if echo "${update_components[@]}" | tr ' ' '\n' | grep -q "^$c\$"; then
		    echo "${c}_git=${rr[${c}_git]}"
		else
		    echo "${c}_git=baseline"
		fi
	    done > $trigger_dest/trigger-build-$update_components2
	done < <(${rr[breakup_updated_components]})
    elif ! [ $score -lt 0 ] 2>/dev/null; then
	# Bisect failures in all steps after "-1" step.
	local cur_rev
	cur_rev=$(git -C $single_component rev-parse HEAD)

	cat > $trigger_dest/trigger-bisect <<EOF
current_project=$single_component
bad_git=${rr[${single_component}_git]%#*}#$cur_rev
EOF
	if [ -f $run_step_artifacts/extra-bisect-params ]; then
	    cat $run_step_artifacts/extra-bisect-params >> $trigger_dest/trigger-bisect
	fi
    fi

    if [ x"$single_component" != x"" ] && ! [ $score -lt 0 ] 2>/dev/null; then
	local mail_dir=${rr[top_artifacts]}/mail
	mkdir -p $mail_dir

	if ! [ -f $run_step_artifacts/mail-subject.txt ]; then
	    cat > $run_step_artifacts/mail-subject.txt <<EOF
[TCWG CI] Regression caused by \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
EOF
	fi

	if ! [ -f $run_step_artifacts/jira-body.txt ]; then
	    cat > $run_step_artifacts/jira-body.txt <<EOF
$(cat $run_step_artifacts/mail-subject.txt):
\$COMMIT_LOG
EOF
	fi

	if ! [ -f $run_step_artifacts/mail-body.txt ]; then
	    cat > $run_step_artifacts/mail-body.txt <<EOF
$(cat $run_step_artifacts/jira-body.txt)

Results regressed to
$(cat ${rr[top_artifacts]}/results)

from
$(cat base-artifacts/results)
EOF
	fi

	(
	    COMMIT_COMPONENT=$single_component
	    export COMMIT_COMPONENT
	    COMMIT_SUBJECT=$(git -C $single_component log --pretty=%s -n 1 HEAD)
	    export COMMIT_SUBJECT
	    COMMIT_LOG=$(git -C $single_component log --pretty=short -n 1 HEAD)
	    export COMMIT_LOG
	    for i in mail-subject.txt jira-body.txt mail-body.txt; do
		envsubst '$COMMIT_COMPONENT $COMMIT_SUBJECT $COMMIT_LOG' \
			 < $run_step_artifacts/$i > $mail_dir/$i
	    done
	)
    fi

    if [ x"${rr[update_baseline]}" = x"update" ]; then
	echo "Detected a regression in \"update\" mode!"
	false
    else
	# Compare results to generate logs and other artifacts,
	# but we don't really care about regressions.
	:
    fi
    )
}

# Commit current result and artifacts to the baseline repository
update_baseline ()
{
    (
    set -euf -o pipefail

    local amend=""

    if [ x"${rr[update_baseline]}" = x"init" ]; then
	amend="--amend"
    elif [ x"${rr[update_baseline]}" = x"push" ]; then
	:
    else
	# ${rr[update_baseline]} == update, reset
	local prev_head=""

	# We discard baseline entries for results worse or same than
	# the current one, but keep entries for results that are better
	# (so that we have a record of pending regressions).
	while true; do
	    ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} &
	    if ! wait $!; then
		break
	    fi

	    prev_head=""
	    if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then
		# For every regression we want to keep artifacts for the first-bad
		# build, so reset to the most relevant regression (marked by reset-baseline).
		if [ -f base-artifacts/reset-baseline ] \
		       && [ x"${rr[update_baseline]}" = x"update" ]; then
		    prev_head=$(git -C base-artifacts rev-parse HEAD)
		fi
		git -C base-artifacts reset --hard HEAD^
	    else
		# We got to the beginning of git history, so amend the current
		# commit.  The initial state of baseline is "empty" branch,
		# which we treat as worst possible in ${rr[no_regression_p]}().
		amend="--amend"
		break
	    fi
	done

	if [ x"$prev_head" != x"" ]; then
	    git -C base-artifacts reset --hard $prev_head
	fi
    fi

    # Rsync current artifacts.  Make sure to use -I rsync option since
    # quite often size and timestamp on artifacts/results will be the same
    # as on base-artifacts/results due to "git reset --hard HEAD^" below.
    # This caused rsync's "quick check" heuristic to skip "results" file.
    # !!! From this point on, logs and other artifacts won't be included
    # in base-artifacts.git repo (though they will be uploaded to jenkins).
    rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/

    local rev_count
    if [ x"$amend" = x"" ]; then
	rev_count=$(git -C base-artifacts rev-list --count HEAD)
    else
	rev_count="0"
    fi

    local msg_title="$rev_count: ${rr[update_baseline]}"

    if [ x"${rr[update_baseline]}" = x"reset" ]; then
	# Create a marker for builds that reset baselines (these are builds
	# for bisected regressions).
	touch base-artifacts/reset-baseline
    fi

    local single_component
    single_component=$(print_single_updated_component)
    if [ x"$single_component" != x"" ]; then
	local single_rev
	single_rev=$(git -C $single_component rev-parse HEAD)
	msg_title="$msg_title: $single_component-$single_rev"
    else
	msg_title="$msg_title: $(print_updated_components "-")"
    fi
    msg_title="$msg_title: $(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)"

    git -C base-artifacts add .
    git -C base-artifacts commit $amend -m "$msg_title

BUILD_URL: ${BUILD_URL-$(pwd)}

results:
$(cat ${rr[top_artifacts]}/results)"
    )
}

# Push to baseline branches and to base-artifacts repo.
push_baseline ()
{
    (
    set -euf -o pipefail

    git_init_linaro_local_remote base-artifacts baseline false
    git_push base-artifacts baseline ${rr[baseline_branch]}

    local url
    local c
    for c in $(print_updated_components); do
	# Clone (but don't checkout) always-present "empty" branch of
	# the baseline repo.  This initializes read/write "baseline" remote.
	url=$(print_baseline_repo "$c" false)
	git_set_remote "$c" baseline "$url"
	git_push $c baseline ${rr[baseline_branch]}
    done
    )
}