path: root/round-robin.sh

#!/bin/bash

# shellcheck source=jenkins-helpers.sh
. "$(dirname $0)"/jenkins-helpers.sh

# Round-Robin associative array.
declare -gA rr

# PROJECT's git url#branch or url#SHA1 revision parsable by git rev-parse.
# A special value "baseline" means that PROJECT is not being updated
# in this build, and its baseline branch should be used.
# In a successful build "update_baseline" step will update baseline
# branches of all PROJECTs to the current values, thus setting
# a baseline for the next build.
#rr[PROJECT_git]

# PROJECT's git SHA1 revision.  This is used in manifest and overrides
# any branch setting in ${rr[PROJECT_git]}.
#rr[PROJECT_rev]

# Baseline branch name for current configuration.
#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}"

# Run mode: build or bisect.  In bisect mode we do a couple things
# slightly differently (e.g., don't touch repo in clone_repo() ), and
# it is allowed to have only a single component updated.
#rr[mode]="$mode"

# How to handle baseline:
# - onsuccess: update baseline branches of base-artifacts and components' repos
#           on success (generate trigger files for bisection on failure).
# - force: ignore failures in check_regression(), which will make current
#          build successful.  Push our artifacts to the top of base-artifacts/.
# - init:  use "empty" results for base-artifacts, which will make current
#          build successful.  Push our artifacts as the one and only entry.
# - ignore: Do not affect baseline. Useful for developer testing.
#rr[update_baseline]=onsuccess/force/init/ignore


# Target architecture to build for: arm or aarch64.
#rr[target]="$target"

# Top-level artifacts directory.
#rr[top_artifacts]="$top_artifacts"

# Predicate function to determine whether there is regression between
# 2 sets of results.
# shellcheck disable=SC2154
rr[no_regression_p]=no_regression_p

# Hook to break up updated component (see print_updated_components) into
# smaller sets: print one set per line.  By default, breakup into singletons.
# shellcheck disable=SC2154
rr[breakup_updated_components]=breakup_updated_components

# Abe's repository and branch to use for the build.
rr[abe_repo]="https://git-us.linaro.org/toolchain/abe.git"
rr[abe_branch]="tested"

# Print round-robin components that are being updated in this build
# (the ones using non-baseline branches).
print_updated_components ()
{
    (
    set -euf -o pipefail

    local delim=""
    local c
    for c in ${rr[components]}; do
	if [ x"${rr[${c}_git]}" != x"baseline" ]; then
	    echo -ne "$delim$c"
	    delim=${1- }
	fi
    done
    echo
    )
}

# By default, print each component on its own line.
breakup_updated_components ()
{
    print_updated_components "\n"
}

# Print the single round-robin component being updated in this build.
# Print nothing if multiple components are being updated.
print_single_updated_component ()
{
    (
    set -euf -o pipefail

    local update_components
    IFS=" " read -r -a updated_components <<< "$(print_updated_components)"

    if [ ${#updated_components[@]} -eq 1 ]; then
	echo "${updated_components[0]}"
    fi
    )
}

# Reset artifacts to an empty state.  ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
reset_artifacts ()
{
    (
    set -euf -o pipefail

    # Clean ${rr[top_artifacts]} but preserve
    # - ${rr[top_artifacts]}/console.log and $run_step_artifacts/console.log, which
    # are being written to by run_step().
    # - ${rr[top_artifacts]}/jenkins/*, which is cleaned by tcwg_kernel.yaml.
    # shellcheck disable=SC2154
    fresh_dir $run_step_top_artifacts \
	      $run_step_top_artifacts/console.log \
	      $run_step_artifacts/console.log \
	      $run_step_top_artifacts/manifest.sh \
	      "$run_step_top_artifacts/jenkins/*"

    if [ ${rr[update_baseline]} = "onsuccess" ] &&
       [ x"$(git ls-remote --heads https://git-us.linaro.org/toolchain/ci/base-artifacts.git refs/heads/${rr[baseline_branch]})" = x ]; then
       echo "WARNING: BASELINE IS NOT FOUND; INITIALIZING BASELINE"
       rr[update_baseline]="init"
    fi

    # Clone base-artifacts here so that bisect runs (which skip this step)
    # don't overwrite it.
    # base-artifacts repo is big and changes all the time, so we
    # fetch only the $baseline_branch, instead of all branches.
    local single_branch
    if [ x"${rr[update_baseline]}" = x"init" ]; then
	single_branch=empty
    else
	single_branch=${rr[baseline_branch]}
    fi
    rr[base-artifacts_rev]="${rr[base-artifacts_rev]-$single_branch}"
    clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch
    cat <<EOF | manifest_out
rr[base-artifacts_rev]=$(git -C base-artifacts rev-parse HEAD)
EOF

    if ! [ -d base-artifacts/git/ ]; then
	# We are in "init" baseline build, apparently.  "Init" builds should
	# have a full set of git data for all components specified on
	# the command line, so that get_baseline_git() is not called.
	mkdir base-artifacts/git

	# Transitional workaround
	# Extract baseline git from manifest of baseline build
	(
	    local components c
	    components="${rr[components]}"
	    # Remove our current settings from ${rr[]}.
	    unset rr
	    # For a short time manifests used "debug" array, so we need
	    # to declare it when sourcing these manifests.
	    # shellcheck disable=SC2034
	    declare -A rr debug
	    if [ -f base-artifacts/manifest.sh ]; then
		source base-artifacts/manifest.sh
	    elif [ -f base-artifacts/jenkins/manifest.sh ]; then
		source base-artifacts/jenkins/manifest.sh
	    else
		# Do nothing and hope get_baseline_git() is not called.
		exit 0
	    fi

	    for c in $components; do
		if [ x"${rr[${c}_url]}" != x"" ]; then
		    echo "${rr[${c}_url]}"
		else
		    print_baseline_repo "$project" true
		fi > base-artifacts/git/${c}_url

		echo "${rr[${c}_rev]}" > base-artifacts/git/${c}_rev
	    done
	)
    fi
    # Copy baseline git_url/git_rev settings into the current build,
    # which will then be overwritten in due course by clone_repo()
    # of various components.
    # Note that we need to copy the whole directory to correctly handle
    # builds that fail before all their components are checked out.
    rsync -a base-artifacts/git/ ${rr[top_artifacts]}/git/
    )
}

get_baseline_git ()
{
    (
    set -euf -o pipefail
    assert_with_msg "ERROR: No $1 in baseline git" \
		    [ -f "base-artifacts/git/$1" ]
    cat "base-artifacts/git/$1"
    )
}

set_current_git ()
{
    (
    set -euf -o pipefail
    mkdir -p ${rr[top_artifacts]}/git
    cat > "${rr[top_artifacts]}/git/$1"
    )
}

# Clone repository for $project
# $1: Repo / project name
clone_repo ()
{
    (
    set -euf -o pipefail
    local project="$1"

    if [ x"${rr[mode]}" = x"bisect" ]; then
	# Cleanup current checkout in bisect mode.
	git_clean "$project"
    else
	local url branch

	if [ x"${rr[${project}_git]}" != x"baseline" ]; then
	    # Fetch and checkout from the specified repo.
	    url="${rr[${project}_git]%#*}"
	    branch="${rr[${project}_git]#*#}"
	else
	    url=$(get_baseline_git ${project}_url)
	    branch=$(get_baseline_git ${project}_rev)
	fi

	# Allow manifest override for $url
	url="${rr[${project}_url]-$url}"
	clone_or_update_repo_no_checkout "$project" "$url" auto "" origin \
					 > /dev/null

	# Store git url in the manifest.
	cat <<EOF | manifest_out
rr[${project}_url]=$url
EOF
	# Store baseline git_url data.  This is then used by
	# subsequent builds to fetch baseline commits.
	echo "$url" | set_current_git ${project}_url

	# Allow manifest override for $branch
	branch="${rr[${project}_rev]-$branch}"
	git_checkout "$project" "$branch" origin
    fi

    local cur_rev
    cur_rev=$(git -C $project rev-parse HEAD)

    local debug_project_date
    debug_project_date=$(git -C $project show --no-patch --pretty="%ct # %cr" HEAD)

    # Store git rev info in the manifest.
    cat <<EOF | manifest_out
rr[${project}_rev]=$cur_rev
rr[debug_${project}_date]=$debug_project_date
EOF
    # Store baseline git_rev data.  This is then used by
    # subsequent builds to fetch baseline commits.
    echo "$cur_rev" | set_current_git ${project}_rev
    )
}

# Prepare ABE tree for [partial] GNU builds
prepare_abe ()
{
    (
    set -euf -o pipefail

    clone_or_update_repo abe ${rr[abe_branch]}  ${rr[abe_repo]} > /dev/null

    local workspace
    workspace=$(pwd)

    cd abe

    # Add ccache wrappers.
    # shellcheck disable=SC2115
    rm -rf "$(pwd)/bin"
    mkdir "$(pwd)/bin"

    cat > "$(pwd)/bin/gcc" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/gcc "\$@"
EOF
    chmod +x "$(pwd)/bin/gcc"
    cp "$(pwd)/bin/gcc" "$(pwd)/bin/cc"

    cat > "$(pwd)/bin/g++" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/g++ "\$@"
EOF
    chmod +x "$(pwd)/bin/g++"
    cp "$(pwd)/bin/g++" "$(pwd)/bin/c++"

    # Disable building documention.  Apparently, this is one of
    # the most popular ways.
    cat > "$(pwd)/bin/makeinfo" <<EOF
#!/bin/sh
exec true
EOF
    chmod +x "$(pwd)/bin/makeinfo"

    PATH=$(pwd)/bin:$PATH
    export PATH

    ./configure --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref
    )
}

# Build ABE component
# $1: Component -- ABE component to build.
build_abe ()
{
    (
    set -euf -o pipefail

    local component="$1"

    if [ x"$component" = x"check_gdb" ]; then
	# Limit GDB testsuites to single-thread parallelism.
	# We've tried running GDB testsuites with 16-thread parallelism,
	# but could not shake out flaky tests in the course of several weeks.
	# Try stabilizing GDB testsuites with single-thread parallelism.
	# If this doesn't work, we'll have to look into dejagnu.
	local cpus
	cpus=$(cat abe/host.conf | grep "^cpus=" | sed -e "s/^cpus=\(.*\)/\1/")
	if [ "$cpus" -gt 1 ]; then
	    cp abe/host.conf abe/host.conf.orig
	    sed -i -e "s/^cpus=.*/cpus=1/" abe/host.conf
	fi
    elif [ -f abe/host.conf.orig ]; then
	mv abe/host.conf.orig abe/host.conf
    fi

    local project stage action check
    check=false

    # Check if component starts with an action (eg. "check")
    case "$component" in
	check_*)
	    component="${component#check_}"
	    check=true
	    ;;
    esac

    case "$component" in
	stage1)
	    project=gcc
	    stage="--stage 1"
	    ;;
	stage2)
	    project=gcc
	    stage="--stage 2"
	    ;;
	bootstrap)
	    project=gcc
	    stage="--set buildconfig=bootstrap"
	    ;;
	bootstrap_ubsan)
	    project=gcc
	    stage="--set buildconfig=bootstrap-ubsan"
	    ;;
	bootstrap_O3)
	    project=gcc
	    stage="--set buildconfig=bootstrap-O3"
	    ;;
	bootstrap_O1)
	    project=gcc
	    stage="--set buildconfig=bootstrap-O1"
	    ;;
	bootstrap_lto)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto"
	    ;;
	bootstrap_debug)
	    project=gcc
	    stage="--set buildconfig=bootstrap-debug"
	    ;;
	bootstrap_profiled)
	    project=gcc
	    stage="--set buildconfig=bootstrap --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	bootstrap_profiled_lto)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	bootstrap_profiled_lto_lean)
	    project=gcc
	    stage="--set buildconfig=bootstrap-lto-lean --set makeflags=profiledbootstrap --set gcc_override_configure=--disable-werror"
	    ;;
	*)
	    project=$component
	    stage=""
	    ;;
    esac

    while [ $# -gt 0 ]; do
	if [ x"$1" = x"--" ]; then
	    shift
	    break
	fi
	shift
    done

    local patch_branch=""
    if [ $# -gt 0 ] && [ x"$1" = x"--patch" ]; then
	patch_branch="$2"
	shift 2
    fi

    if [ x"$project" = x"gcc" ]; then
	# Don't build Go, which has a funky and unstable testsuite.
	# Stage1 ignores "--set languages=" option, so this applies only
	# to stage2 builds.
	stage="$stage --set languages=c,c++,fortran,lto"
	if [ $# != 0 ]; then
	    stage="$stage $*"
	fi
    fi

    action="--build $project"
    # When checking a component we still need to force its build,
    # otherwise ABE does nothing.
    if $check; then
	action="$action --check $project"
    fi

    local custom_abe_src_opt=""
    local git_dir="$project"
    local n_patches=0
    case "$component" in
	# Use our custom sources for everything, but dejagnu.
	dejagnu) ;;
	*)
	    git_dir="$git_dir.git"
	    custom_abe_src_opt="$project=http://git.l.o/$git_dir~master --disable update"

	    if ! $check; then
		clone_repo $project

		if [ x"$patch_branch" != x"" ]; then
		    git -C $project fetch "https://git.linaro.org/toolchain/$project.git" "refs/heads/$patch_branch"
		    git -C $project cherry-pick FETCH_HEAD
		    n_patches=1
		fi

		# Don't use ABE's repo clone functions and setup abe/snapshots/
		# directory to have the right entries.
		local git_path
		git_path="abe/snapshots/$git_dir"
		rm -rf $git_path $git_path~master
		ln -s "$(pwd)/$project" $git_path
		ln -s "$(pwd)/$project" $git_path~master
	    fi
	    ;;
    esac

    cd abe

    # Remove previous build directories and .stamp files.
    # (we rely on ccache for fast rebuilds)
    # These stamp files tell ABE whether it needs to re-run
    # checkout/configure/build steps. Which we do not want
    # to do on a check step.
    if ! $check; then
	set +f; rm -rf builds/*/*/$project-*; set -f
    fi

    PATH=$(pwd)/bin:$PATH
    export PATH

    ccache -z
    local target_opt=""
    if [ x"${rr[target]}" != x"native" ]; then
        target_opt="--target $(print_gnu_target ${rr[target]})"

	# Disable TSan execution tests when using QEMU.
	# QEMU can't handle TSan's shadow memory and it sends host machine
	# into swap.
	if $check && [ x"$project" = x"gcc" ]; then
	    if echo "$stage" | grep -q -e "--testcontainer"; then
		assert_with_msg "ERROR: Testing is not using QEMU" false
	    fi
	    sed -i \
		-e 's/set dg-do-what-default run/set dg-do-what-default link/' \
		../gcc/gcc/testsuite/lib/tsan-dg.exp
	fi
    fi

    # Run "./abe.sh --build $project".
    ./abe.sh \
	$action \
	$target_opt \
	--extraconfigdir config/master \
	--disable make_docs \
	$custom_abe_src_opt \
	$stage &
    res=0 && wait $! || res=$?

    # Revert patches if applied.
    if [ -d ../$project ]; then
        git -C ../$project reset --hard HEAD~$n_patches
    fi

    # If abe failed to build component, return exit status.
    if [ x"$res" != x"0" ]; then
        return $res
    fi
    ccache -s

    if $check; then
	local sum log
	rm -rf ${rr[top_artifacts]}/sumfiles
	mkdir -p ${rr[top_artifacts]}/sumfiles
	while IFS= read -r -d '' sum; do
	    cp "$sum" ${rr[top_artifacts]}/sumfiles/
	    log="${sum%.sum}.log"
	    # Testsuite logs grow 50-400MB in size, so compress them to save
	    # disk space on ci.linaro.org.
	    # Delete previous XZ'ed log, which can occur when we are bisecting
	    # QEMU and not rebuilding compiler (therefore not cleaning compiler
	    # build/test directory).
	    rm -f "$log.xz"
	    (
		xz "$log"
		cp "$log.xz" ${rr[top_artifacts]}/sumfiles/
	    ) &
	done < <(find builds/ \( -name "*.sum" -not -path "*/gdb/testsuite/outputs/*" \) -print0)
	# Wait for logs to compress
	wait
    fi
    )
}

# If we bisect a regression between different major versions of Glibc,
# then we might get a mixed sysroot with several versions of ld-M.N.so and
# other binaries installed side-by-side.  Such a sysroot will break
# benchmarking, which requires a single ld-*.so binary to be present.
# Similarly, weird problems can occur if we are bisecting linux
# and re-installing kernel headers one on top another.
# Forcefully delete sysroot before building C library or linux headers.
clean_sysroot ()
{
    (
    set -euf -o pipefail

    local gnu_host gnu_target sysroot
    gnu_host=$(print_gnu_target native)
    gnu_target=$(print_gnu_target ${rr[target]})
    sysroot="$(pwd)/abe/builds/destdir/$gnu_host/$gnu_target/libc"

    rm -rf "$sysroot"
    )
}

# Build LLVM
build_llvm ()
{
    (
    set -euf -o pipefail

    local use_abe="${1-false}"
    local extra_targets="${2-}"

    clone_repo llvm
    wget -O llvm-vect-metric.diff "https://git.linaro.org/toolchain/jenkins-scripts.git/plain/downstream_patches/llvm-vect-metric.diff"
    git -C llvm apply "$(pwd)/llvm-vect-metric.diff"

    sanity_check_pwd

    local workspace
    workspace=$(pwd)

    # Setup ccache and ninja wrappers.
    # shellcheck disable=SC2115
    rm -rf "$(pwd)/bin"
    mkdir "$(pwd)/bin"

    cat > "$(pwd)/bin/cc" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/gcc "\$@"
EOF
    chmod +x "$(pwd)/bin/cc"

    cat > "$(pwd)/bin/c++" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache /usr/bin/g++ "\$@"
EOF
    chmod +x "$(pwd)/bin/c++"

    if [ -f /usr/local/bin/ninja.bin ]; then
	# Use ninja configuration from llvm buildbots to avoid running out of RAM.
	cat > "$(pwd)/bin/ninja" <<EOF
#!/bin/sh
exec /usr/local/bin/ninja.bin -m 30 -M 50 -D 5000 "\$@"
EOF
	chmod +x "$(pwd)/bin/ninja"
    fi

    PATH=$(pwd)/bin:$PATH
    export PATH

    local binutils_incdir
    # Freshen up build and install directories.  We rely on ccache for fast rebuilds.
    if $use_abe; then
	local gnu_host
	gnu_host=$(print_gnu_target native)
	rsync -a --del abe/builds/destdir/$gnu_host/ llvm-install/
	binutils_incdir=$(pwd)/binutils/include
    else
	rm -rf llvm-install
	binutils_incdir=/usr/include
    fi

    local llvm_targets
    llvm_targets="$(print_llvm_target ${rr[target]})${extra_targets}"

    rm -rf llvm-build
    mkdir -p llvm-build
    cd llvm-build

    cmake -G Ninja  ../llvm/llvm "-DLLVM_ENABLE_PROJECTS=clang;lld" \
	  -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True \
	  -DCMAKE_INSTALL_PREFIX=../llvm-install \
	  "-DLLVM_TARGETS_TO_BUILD=$llvm_targets" \
	  -DLLVM_BINUTILS_INCDIR=$binutils_incdir
    ccache -z
    ninja
    ninja install
    ccache -s
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_build_regression_p ()
{
    (
    set -euf -o pipefail

    local ref_artifacts=$1
    local new_artifacts=$2

    # The following check tests for an empty branch, and we return 0 if results
    # are not present, since that's an inherently better state. If base-artifacts
    # were not existing, then it would fail in reset_artifacts() stage.

    if ! [ -f $ref_artifacts/results ]; then
	return 0
    fi

    local build_result_ref build_result_new
    build_result_ref=$(grep -v "^#" $ref_artifacts/results | tail -n1)
    build_result_new=$(grep -v "^#" $new_artifacts/results | tail -n1)

    if [ $build_result_new -ge $build_result_ref ]; then
	return 0
    fi

    local last_log
    last_log=$(find $new_artifacts/ -name console.log.xz | sort -g | tail -n1)
    # In log scan for errors below
    #   - sed -e 's/"[^"]*"//g' -- removes quoted "error: app diagnostics" strings
    #   - " error:" detects compiler errors from GCC and Clang (including GCC ICEs)
    #   - "^ERROR:" detects linker errors
    #   - ": undefined reference" detects missing symbols during linking
    #   - "] Error " detects GNU make errors
    # Then grep for "grep" to exclude other uses of this search.
    cat > $run_step_artifacts/results.regressions <<EOF
# First few build errors in logs:
$(xzcat $last_log | sed -e 's/"[^"]*"//g' | grep " error:\|^ERROR:\|: undefined reference\|\] Error " | grep -v "grep" | head | sed -e "s/^/# /")
EOF
    return 1
    )
}

# Check if current build regressed compared to the baseline
check_regression ()
{
    (
    set -euf -o pipefail

    local score
    score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)

    if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then
	echo "ERROR: We have encountered some infrastructure problem (e.g.,"
	echo "       benchmarking boards are offline), and we can't finish"
	echo "       the build."
	# Exit now and don't update baseline artifacts.
	# By not creating trigger-build-* files, we signal
	# round-robin-bisect.sh to skip this build/revision.
	exit $EXTERNAL_FAIL
    fi

    local res

    # Generate comparison artifacts for update, reset, init and push modes.
    ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} &
    res=0 && wait $! || res=$?

    # Move extra artifacts that no_regression_p generated to $top_artifacts.
    if [ -d $run_step_artifacts/top-artifacts ]; then
	rsync -a $run_step_artifacts/top-artifacts/ ${rr[top_artifacts]}/
	rm -rf $run_step_artifacts/top-artifacts
    fi

    if [ $res = 0 ]; then
	# All good, no regression
	return
    elif [ $res = $EXTERNAL_FAIL ]; then
	# Comparison failed to produce a meaningful result
	exit $EXTERNAL_FAIL
    fi

    assert_with_msg "no_regression_p should succeed in init baseline mode" \
		    [ x"${rr[update_baseline]}" != x"init" ]

    if [ -f $run_step_artifacts/results.regressions ]; then
	# Add regression info generated by no_regression_p to top-level
	# results file.
	cat $run_step_artifacts/results.regressions \
	    >> ${rr[top_artifacts]}/results
    fi

    # We've got a regression.  Generate trigger-* files.
    local single_component
    single_component=$(print_single_updated_component)
    local trigger_dest

    if [ x"${rr[update_baseline]}" != x"force" ]; then
	# We are seeing a failure, so instead of updating baseline start
	# reducing/bisecting the failure.  Create trigger-* files at top level
	# where jenkins expects them -- and trigger the followup builds.
	trigger_dest="${rr[top_artifacts]}"
    else
	# We don't want to trigger follow up builds when forcing
	# the baseline.  So, for the record, place trigger-* files in
	# the step's artifacts directory.
	trigger_dest="$run_step_artifacts"
    fi

    if [ x"$single_component" = x"" ] \
	   || { [ x"${rr[mode]}" = x"bisect" ] \
		    && ! [ $score -lt 0 ] 2>/dev/null; }; then
	# If we have several changed components, then trigger individual builds
	# for these components to narrow down the problem to a single component.
	# Also generate trigger-build-* files as a favor to
	# round-robin-bisect.sh script to distinguish "bad" from "skip" builds
	# and to provide templates for triggering follow-up builds.
	# Note that the negative "! [ $score -lt 0 ]" condition is to handle
	# non-numeric scores like "all" and "boot".
	local -a update_components

	while read -a update_components; do
	    local c update_components2
	    update_components2=$(echo "${update_components[@]}" | tr ' ' '-')

	    for c in ${rr[components]}; do
		if echo "${update_components[@]}" | tr ' ' '\n' | grep -q "^$c\$"; then
		    echo "${c}_git=${rr[${c}_git]}"
		fi
	    done > $trigger_dest/trigger-build-$update_components2
	    # Add update_baseline setting to the parameters, so that
	    # "ignore" builds will trigger "ignore" reduction builds.
	    echo "update_baseline=${rr[update_baseline]}" \
		 >> $trigger_dest/trigger-build-$update_components2
	done < <(${rr[breakup_updated_components]})
    elif [ x"$single_component" != x"" ] \
	     && ! [ $score -lt 0 ] 2>/dev/null; then
	# Trigger bisect failures in a single changed component in all steps
	# with a positive result.  If $score is less-than 0, then
	# the regression is not very interesting, so don't bisect.

	cat > $trigger_dest/trigger-bisect <<EOF
current_project=$single_component
bad_git=${rr[${single_component}_git]%#*}#$cur_rev
EOF
	if [ -f $run_step_artifacts/extra-bisect-params ]; then
	    cat $run_step_artifacts/extra-bisect-params >> $trigger_dest/trigger-bisect
	fi
    fi

    if [ x"$single_component" != x"" ] && ! [ $score -lt 0 ] 2>/dev/null; then
	local mail_dir=${rr[top_artifacts]}/mail
	mkdir -p $mail_dir

	if ! [ -f $run_step_artifacts/mail-subject.txt ]; then
	    cat > $run_step_artifacts/mail-subject.txt <<EOF
[TCWG CI] Regression caused by \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
EOF
	fi

	if ! [ -f $run_step_artifacts/jira-body.txt ]; then
	    cat > $run_step_artifacts/jira-body.txt <<EOF
$(cat $run_step_artifacts/mail-subject.txt):
\$COMMIT_LOG
EOF
	fi

	if ! [ -f $run_step_artifacts/mail-body.txt ]; then
	    cat > $run_step_artifacts/mail-body.txt <<EOF
$(cat $run_step_artifacts/jira-body.txt)

Results regressed to
$(cat ${rr[top_artifacts]}/results)

from
$(cat base-artifacts/results)
EOF
	fi

	(
	    COMMIT_COMPONENT=$single_component
	    export COMMIT_COMPONENT
	    COMMIT_SUBJECT=$(git -C $single_component log --pretty=%s -n 1 HEAD)
	    export COMMIT_SUBJECT
	    COMMIT_LOG=$(git -C $single_component log --pretty=short -n 1 HEAD)
	    export COMMIT_LOG
	    for i in mail-subject.txt jira-body.txt mail-body.txt; do
		envsubst '$COMMIT_COMPONENT $COMMIT_SUBJECT $COMMIT_LOG' \
			 < $run_step_artifacts/$i > $mail_dir/$i
	    done
	)
    fi

    if [ x"${rr[update_baseline]}" != x"force" ]; then
	echo "Detected a regression!"
	false
    fi
    )
}

# Commit current result and artifacts to the baseline repository
update_baseline ()
{
    (
    set -euf -o pipefail

    if [ x"${rr[update_baseline]}" = x"ignore" ]; then
	return
    fi

    # - For the last 100 builds: keep everything
    # - For the rest of the builds: delete non-essential artifacts
    # - NN-<step> directories are non-essential, the rest -- jenkins/,
    #   dashboard/, etc. -- are essential.
    local old_commit
    old_commit=$(git -C base-artifacts rev-parse --verify HEAD~100 \
		     2>/dev/null || true)

    if [ x"$old_commit" != x"" ]; then
	local head
	head=$(git -C base-artifacts rev-parse HEAD)

	# Remove directories starting with number from $old_commit
	# and older.
	git -C base-artifacts checkout -detach $old_commit
	git -C base-artifacts filter-repo --force \
	    --invert-paths --path-regex '^[0-9].*' \
	    --refs HEAD

	# Reparent history on the new version of $old_commit.
	local new_old_commit
	new_old_commit=$(git -C base-artifacts rev-parse HEAD)
	git -C base-artifacts checkout --detach $head
	git -C base-artifacts replace $old_commit $new_old_commit
	git -C base-artifacts filter-repo --force --refs HEAD
    fi

    # Rsync current artifacts.  Make sure to use -I rsync option since
    # quite often size and timestamp on artifacts/results will be the same
    # as on base-artifacts/results due to "git reset --hard HEAD^" below.
    # This caused rsync's "quick check" heuristic to skip "results" file.
    # !!! From this point on, logs and other artifacts won't be included
    # in base-artifacts.git repo (though they will be uploaded to jenkins).
    rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/

    local rev_count amend=""

    if [ x"${rr[update_baseline]}" = x"init" ]; then
	rev_count="0"
	amend="--amend"
    else
	rev_count=$(git -C base-artifacts rev-list --count HEAD)
    fi

    local msg_title="$rev_count: ${rr[update_baseline]}"

    if [ x"${rr[update_baseline]}" = x"force" ]; then
	# Create a marker for builds that reset baselines (these are builds
	# for bisected regressions).
	touch base-artifacts/reset-baseline
    fi

    local single_component
    single_component=$(print_single_updated_component)
    if [ x"$single_component" != x"" ]; then
	local single_rev
	single_rev=$(git -C $single_component rev-parse HEAD)
	msg_title="$msg_title: $single_component-$single_rev"
    else
	msg_title="$msg_title: $(print_updated_components "-")"
    fi
    msg_title="$msg_title: $(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)"

    git -C base-artifacts add .
    git -C base-artifacts commit $amend -m "$msg_title

BUILD_URL: ${BUILD_URL-$(pwd)}

results:
$(cat ${rr[top_artifacts]}/results)"

    # Generate artifacts/jenkins/git-push.sh, which the CI job
    # will use to update baseline branches in git repos.
    git_init_linaro_local_remote base-artifacts baseline false
    mkdir -p ${rr[top_artifacts]}/jenkins
    cat > ${rr[top_artifacts]}/jenkins/git-push.sh <<EOF
git -C base-artifacts push --force baseline HEAD:refs/heads/${rr[baseline_branch]}
EOF
    chmod +x ${rr[top_artifacts]}/jenkins/git-push.sh
    )
}