path: root/jenkins-helpers.sh

#!/bin/bash

sanity_check_pwd ()
{
    (
    set -euf -o pipefail

    if [[ $(pwd) == "/" ]]; then
        echo "pwd is unexpectedly \"/\", exiting"
        exit 1;
    fi
    )
}

# Print absolute path to a file or directory
# $1: Path (must exist)
abs_path ()
{
    (
    set -euf -o pipefail

    echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
    )
}

# Assert that $@[1:] returns success.
# $1 should be a message to print on failure e.g.
# assert "will always fail" false
assert_with_msg ()
{
    (
    set -euf -o pipefail +x

    local failure_message=$1
    shift

    eval "$*" || (echo "$failure_message" >&2 && exit 1)
    )
}

# Assert that $@ returns success.
assert ()
{
    (
    set -euf -o pipefail +x

    eval "$*"
    )
}

# Cleanup contents of directory preserving specified parts.
# $1: Directory to clean
# $@: Find patterns of files to preserve (must start with $1)
fresh_dir ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    shift 1

    # Make sure $dir doesn't have "/" at the end.
    dir=$(dirname "$dir/something")

    mkdir -p "$dir"

    find_opts=("!" "-path" "$dir")
    for keep_pattern in "$@"; do
	while : ; do
	    find_opts+=("!" "-path" "$keep_pattern")
	    keep_pattern=$(dirname "$keep_pattern")
	    if [ x"$keep_pattern" = x"$dir" ]; then
		break
	    fi
	    # This assert will trigger if one of keep_patterns doesn't start
	    # with "$dir": eventually dirname will get to either "/" or ".".
	    assert [ "$keep_pattern" != "/" -a "$keep_pattern" != "." ]
	done
    done

    # Make sure we can delete the files
    find "$dir" "${find_opts[@]}" -type d -exec chmod +rwx {} \;
    chmod -R +rw "$dir"
    find "$dir" "${find_opts[@]}" -delete
    )
}

# $@: Jenkins labels
# Prints nodes corresponding to jenkins labels.
print_nodes_in_labels ()
{
    (
    set -euf -o pipefail

    local labels=( "$@" )
    local label
    local tmpfile
    tmpfile=$(mktemp)

    for label in "${labels[@]}"; do
	# Handle gracefully the case where the label contains no
	# machine: we don't want to generate an error, rather return
	# an empty string. Use a tmp file so that we still generate an
	# error in case wget fails.
	wget --retry-connrefused --waitretry=1 -O - https://ci.linaro.org/label/$label/api/json?pretty=true 2>/dev/null > $tmpfile
	grep nodeName $tmpfile | cut -d: -f 2 | sed -e 's/"//g' || true
    done

    rm $tmpfile
    )
}

# $@: Jenkins labels, typically tcwg-t[kx]1_{32/64}-test
# Returns node from one of the labels with least number of running containers.
print_node_with_least_containers ()
{
    (
    set -euf -o pipefail

    local tester_labels=( "$@" )
    local testers
    local load_value
    local tester_min_load_name=""
    local tester_min_load_value="999"
    local ret

    # Re. --random-sort below: shuffle node list to mitigate races
    # when starting multiple containers at the same time
    testers=$(print_nodes_in_labels "${tester_labels[@]}" | sort --random-sort)
    for tester in $testers; do
        ret=0
	tester_host=$(print_host_for_node $tester "ignore_fail")
	if [ x"${tester_host}" != x"" ]; then
            load_value=$(timeout 30s ssh ${tester_host} docker ps | wc -l) || ret=$?
            if [ $ret -eq 0 ]; then
		if [ "$load_value" -lt "$tester_min_load_value" ]; then
                    tester_min_load_name=$tester
                    tester_min_load_value=$load_value
		fi
	    fi
	fi
    done
    echo $tester_min_load_name
    )
}

# $1: Jenkins tcwg-*-build label
# Prints out architecture for container image
print_arch_for_label ()
{
    (
    set -euf -o pipefail

    local label="$1"

    case $label in
        tcwg-x86_64-*) echo amd64 ;;
        tcwg-x86_32-*) echo i386 ;;
        tcwg-amp_64-*|tcwg-apm_64-*|tcwg-armv8_64|tcwg-d05_64-*|tcwg-lc_64*|tcwg-sq_64-*|tcwg-thx1_64-*|tcwg-tx1_64-*) echo arm64 ;;
        tcwg-amp_32-*|tcwg-apm_32-*|tcwg-armv7|tcwg-armv8_32|tcwg-d05_32-*|tcwg-sq_32-*|tcwg-tk1_32-*|tcwg-tx1_32-*) echo armhf ;;
        *) echo "ERROR: Unsupported label: $label" >&2; exit 1 ;;
    esac
    )
}

# $1: Jenkins tcwg-*-build label
# Prints out host type
print_type_for_label ()
{
    (
    set -euf -o pipefail

    echo "$1" | sed -e "s/^tcwg-\(.*\)-build\$/\1/"
    )
}

# $1: Jenkins $NODE_NAME
# $2: (optional) "ignore_fail"
# Prints SSH host
print_host_for_node ()
{
    (
    set -euf -o pipefail

    local host

    # All jenkins nodes have either <NODE> or <NODE>.tcwglab entry in
    # .ssh/config (in dockerfiles.git/tcwg-base/tcwg-buildslave/).
    for suffix in "" ".tcwglab"; do
	host="$1$suffix"
	if timeout 30s ssh "$host" true &> /dev/null; then
	    break
	fi
	host=""
    done

    if [ x"$host" = x"" ] && [ x"${2-}" != x"ignore_fail" ]; then
        echo "Error: print_host_for_node() cannot ssh to $1 or $1.tcwglab" >&2
        exit 1
    fi

    echo "$host"
    )
}

# $1: Host name or "localhost".
# Prints docker-friendly arch of host
print_arch_for_host ()
{
    (
    set -euf -o pipefail

    local host="$1"
    local arch

    case "$host" in
	"localhost")
	    arch=$(uname -m)
	    case "$arch" in
		"aarch64") arch="arm64" ;;
		"arm"*) arch="armhf" ;;
		"x86_64") arch="amd64" ;;
		*) echo "ERROR: Unknown uname -m arch: $arch" >&2; exit 1 ;;
	    esac
	    echo "$arch"
	    ;;
	*)
	    # While not strictly correct, print_arch_for_label is relaxed
	    # enough to handle this.
	    print_arch_for_label "$host"
	    ;;
    esac
    )
}

# $1: target triplet
# Prints tester label for remote cross-testing
print_tester_label_for_target ()
{
    (
    set -euf -o pipefail

    local target="$1"

    case "$target" in
        aarch64-linux-gnu_ilp32)
	    # We test ILP32 using QEMU KVM, and TX1s run 3.10 kernel that
	    # doesn't support KVM.  Test on APM builders for now.
	    echo "tcwg-apm_64-build"
	    ;;
	# We allocate all TK1/TX1 boards to benchmarking, so use APMs
	# for cross-testing. This means we no longer test on armv7
	# hardware.
        aarch64-linux*) echo "tcwg-apm_64-test" ;;
        armv8l-linux*) echo "tcwg-armv8_32" ;;
        arm-linux*) echo "tcwg-armv7" ;;
    esac
    )
}

# Print number of busy executors on a jenkins node
# $1: node
print_number_of_busy_executors ()
{
    (
    set -euf -o pipefail
    local node="$1"

    local json
    json=$(mktemp)
    # shellcheck disable=SC2064
    trap "rm $json" EXIT

    curl -s "https://ci.linaro.org/computer/$node/api/json?depth=1" > "$json"

    local n n_busy idle

    n=$(jq -r ".numExecutors" < "$json")
    n_busy="$n"

    while [ "$n" -gt "0" ]; do
        n=$(($n - 1))
        idle=$(jq -r ".executors[$n].idle" < "$json")
        if [ "$idle" = "true" ]; then
            n_busy=$(($n_busy - 1))
        fi
    done

    n=0
    while true; do
        idle=$(jq -r ".oneOffExecutors[$n].idle" < "$json")
        if [ "$idle" = "null" ]; then
            break
        elif [ "$idle" != "true" ]; then
            n_busy=$(($n_busy + 1))
        fi
        n=$(($n + 1))
    done

    echo "$n_busy"
    )
}

# Run command on remote machine in given directory via ssh on a given port
# "$1" -- <host>[:<port>[:<dir>[:<ssh_opts>[:<env>]]]]
# "$2, $3, etc" -- command and its arguments
# E.g., remote_exec dev-01.tcwglab::/tmp find -name "my file.bak"
# NOTE: The environment variables are not escaped, so pass only simple things.
#       This is because we want ability to pass multiple variables "a=b c=d",
#       and escaping will make that into a single a="b c=d" variable.
remote_exec ()
{
    (
    set -euf -o pipefail

    local host
    host="$(echo "$1" | cut -d: -f 1)"
    local port
    port="$(echo "$1" | cut -s -d: -f 2)"
    local dir
    dir="$(echo "$1" | cut -s -d: -f 3)"
    local opts
    opts="$(echo "$1" | cut -s -d: -f 4)"
    local env_vars
    env_vars="$(echo "$1" | cut -s -d: -f 5)"
    shift
    local -a cmd
    cmd=()
    # Add quotes to every parameter
    for i in "$@"; do cmd+=("$(printf '%q' "$i")"); done
    # Be careful to prepend statements before ${cmd[@]} only if necessary.
    # E.g., when triggering jobs via jenkins-cli, the command is not a binary,
    # so we can't "exec" it.
    # We use flock if $JENKINS_FLOCK is set.
    ${JENKINS_FLOCK+$JENKINS_FLOCK} ssh $opts ${port:+-p$port} $host "${env_vars:+export $env_vars && }${dir:+cd "$(printf '%q' "$dir")" && exec }${cmd[*]}"
    )
}

# Resolve git ref to sha1
# $1 -- repo directory
# $2 -- branch, tag or refspec
# $3 -- remote name
# $4 -- extra options to git rev-parse
git_rev_parse_1 ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="$3"
    local opts="$4"

    local ret

    cd "$dir"

    # Convert git branch/tag names into SHA1
    local sha1 try_ref
    case "$ref" in
	"refs/"*) try_ref="$ref";;
	*) try_ref="refs/remotes/$remote/$ref" ;;
    esac
    ret=0; sha1=$(git rev-parse $opts "$try_ref" 2>/dev/null) || ret=$?
    if [ $ret -ne 0 ]; then
	# Assume that $ref is already a SHA1
	ret=0; sha1=$(git rev-parse $opts "$ref") || ret=$?
	if [ $ret -ne 0 ]; then
	    echo "ERROR: Cannot parse $ref in repo $dir" >&2
	    exit 1
	fi
    fi
    echo "$sha1"
    )
}

# Resolve git ref to short sha1
# $1 -- repo directory
# $2 -- branch, tag or refspec
# $3 -- (optional) remote name
git_rev_parse ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="origin"

    if [ $# -ge 3 ]; then
	remote="$3"
    fi

    git_rev_parse_1 "$dir" "$ref" "$remote" "--short"
    )
}

# Resolve git ref to full sha1
# $1 -- repo directory
# $2 -- branch, tag or refspec
# $3 -- (optional) remote name
git_rev_parse_long ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="origin"

    if [ $# -ge 3 ]; then
	remote="$3"
    fi

    git_rev_parse_1 "$dir" "$ref" "$remote" ""
    )
}

# Run a command with the timeout program, with retries if the command times
# out.
# $1: is the DURATION to pass to timeout.
# $2: is the maximum number of attempts at running the command.
# $@: remainder is the command to run
# Returns the status from the timeout command.
#
# Example: run_with_timeout_and_retry 90s 3 git checkout /large/repo
function run_with_timeout_and_retry {
    local duration=$1
    shift
    local tries=$1
    shift
    local n=0
    local ret=0
    until [ ${n} -eq ${tries} ]
    do
        echo timeout "${duration}" "$@"
        timeout "${duration}" "$@" &
        ret=0 && wait $! || ret=$?
        if [ ${ret} -eq 0 ]; then break; fi
	# Handle special errors
	case "$@" in
	    # git clone failed, removed the (incomplete) directory
	    # (last parameter).
	    # We could try to be smarter and handle the case where git
	    # clone actually timed-out and where git suggests to retry
	    # the checkout with 'git checkout -f HEAD', but this seems
	    # a bit awkward for a case that happens very rarely.
	    "git clone")
		# In case there was no (optional) $dir parameter, this
		# will expand to the repo URL, which probably does not
		# exist as a directory name. Still, don't try to
		# remove a directory named *://*.
		dir=$(echo "$@" | awk '{print $NF;}')
		case "$dir" in
		    *://*) ;;
		    *)
			if [ -d $dir ]; then
			    rm -rf $dir
			fi
			;;
		esac
		;;
	esac
        n=$((n + 1))
    done
    return ${ret}
}

# Configure git remote
# $1: git repo directory
# $2: remote name
# $3: git url
# $4: optional single branch
git_set_remote ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local remote="$2"
    local url="$3"
    local single_branch="${4-}"

    git -C "$dir" remote rm "$remote" > /dev/null 2>&1 || true
    git -C "$dir" remote add ${single_branch:+-t "$single_branch"} "$remote" "$url"
    )
}

# Clone or update a git repo
# $1 -- repo directory
# $2 -- master git repo
# $3 -- reference git repo (to speedup initial cloning)
# $4 -- single-branch to reduce fetching from remote repo
# $5 -- name of remote
clone_or_update_repo_no_checkout ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local url="$2"
    local reference="$3"
    local single_branch="$4"
    local remote="$5"

    local refopt=""
    case "$reference" in
	auto)
	    local ref_dir
	    for ref_dir in $url $dir; do
		ref_dir=$(basename $ref_dir .git)
		ref_dir="/home/tcwg-buildslave/snapshots-ref/$ref_dir.git"
		if git -C $ref_dir rev-parse --git-dir >/dev/null 2>&1; then
		    refopt="--reference $ref_dir"
		    break
		elif [ -d $ref_dir ]; then
		    refopt="--reference-if-able $ref_dir"
		    break
		fi
	    done
	    ;;
	none) ;;
	*) refopt="--reference $reference" ;;
    esac

    if ! git -C "$dir" status >/dev/null 2>&1; then
	# Git repo doesn't exist or is corrupted.  Make a new clone.
	rm -rf "$dir"
    fi

    if [ -d "$dir" ] && [ x"$refopt" != x"" ] \
	   && [ "$(du -s "$dir/.git" | cut -f 1)" -gt $((1024*1024)) ]; then
	# Current clone has grown above 1GB, and we have a reference repo,
	# which should cut down the size significantly.
	# Redo the clone to save disk space.
	# PS: Unfortunately, I could not find a way to make the current clone
	# use new objects from the reference repo without a full re-clone.
	# Fortunately, a new clone with a reference repo is quick.
	rm -rf "$dir"
    fi

    # This is a not a real loop; just easier to skip parts with "break".
    while [ -d "$dir" ]; do
	# Clean up the clone (this is supposed to re-share objects from
	# reference clone and keep the size of the clone minimal).
	# It's possible that previous GC process was interrupted and left
	# a lock.  Use --force to workaround that.  It should be safe
	# to override the lock since directories should not be shared
	# between concurrent builds.
	#
	# Also, prune all loose objects to avoid "git gc --auto" failing
	# and creating .git/gc.log to warn us.
	rm -f "$dir/.git/gc.log"
	# Do not detach into background for GC.  Running in the background may
	# cause a failure during bisect's rsync, which may see some of
	# the files disappering mid-rsync.
	git -C "$dir" config gc.autoDetach false
	if ! git -C "$dir" gc --auto --force --prune=all; then
	    # "git gc" can fail due to corrupted packs.
	    rm -rf "$dir"
	    break
	fi

	# Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock
	# These occur when builds are aborted during "git remote update" or
	# similar.
	find "$dir/.git" -name "*.lock" -delete

	# Recover from any previous am/cherry-pick/rebase.
	# In pre-commit CI we apply patches with "git am", which can fail
	# and leave clone in a bad state.
	local i
	for i in am cherry-pick rebase; do
	    git -C "$dir" "$i" --abort &>/dev/null || true
	done

	break
    done

    local fresh_clone=false
    while true; do
	if ! [ -d "$dir" ]; then
	    local single_branch_opt=""
	    if [ x"$single_branch" != x"" ]; then
		single_branch_opt="--single-branch --branch $single_branch"
	    fi

	    run_with_timeout_and_retry 1h 3 git clone \
				       $refopt $single_branch_opt "$url" "$dir"
	    fresh_clone=true
	fi

	git_set_remote "$dir" "$remote" "$url" "$single_branch"

	local refspec
	if [ x"$single_branch" = x"" ]; then
	    run_with_timeout_and_retry 1h 3 git -C "$dir" remote update -p \
				       "$remote" 2>/dev/null
	    refspec="+refs/changes/*:refs/changes/*"
	else
	    refspec="+refs/heads/$single_branch:refs/remotes/$remote/$single_branch"
	fi

	if ! run_with_timeout_and_retry 1h 3 git -C "$dir" fetch -q \
	     $remote $refspec --prune; then
	    # "git fetch --prune" can fail due to running out of memory space
	    # on 32-bit architectures on big repos.  Remove the repo and retry
	    # with a fresh clone.
	    if $fresh_clone; then
		return 1
	    fi

	    rm -rf "$dir"
	    continue
	fi

	break
    done
    )
}

# Checkout branch/ref/SHA1 in a git repo
# $1 -- repo directory
# $2 -- ref to checkout
# $3 -- name of the git remote
git_checkout ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="$3"

    git_clean "$dir"
    # Convert git branch/tag names into SHA1
    local sha1
    sha1=$(git_rev_parse "$dir" "$ref" "$remote")
    # Checkout
    git -C "$dir" checkout --detach "$sha1"
    )
}

# Clone or update a git repo
# $1 -- repo directory
# $2 -- ref to checkout
# $3 -- master git repo
# $4 -- optional reference git repo (to speedup initial cloning)
# $5 -- optional single-branch to reduce fetching from remote repo
# $6 -- optional name of remote (default is "origin")
clone_or_update_repo ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local url="$3"
    local reference="${4-auto}"
    local single_branch="${5-}"
    local remote="${6-origin}"

    clone_or_update_repo_no_checkout "$dir" "$url" "$reference" \
				     "$single_branch" "$remote"

    git_checkout "$dir" "$ref" "$remote"
    )
}

# Wget files from URL that may have wildcards; only the last "basename"
# part of URL is allowed to contain wildcards.  Safe to use on normal URLs.
# Return N-1 of files downloaded, or 127 if no files were downloaded.
# $1 -- URL
# $2,... -- additional parameters to wget
wget_wildcard_url ()
{
    (
    set -eu -o pipefail

    local url="$1"
    shift

    local url_basename
    url_basename="$(basename "$url")"

    local tmpdir
    tmpdir="$(mktemp -d)"

    wget_opts=""
    case "$(echo "$url" | cut -d/ -f3)" in
	*".tcwglab") wget_opts="$wget_opts --no-check-certificate" ;;
    esac

    # $(dirname "$url") may not be a valid URL.  Since we only use '*'
    # as wildcards, check if a '*' is present in $url_basename, and if
    # not, do a direct wget on $url to avoid accessing $(dirname "$url")
    if echo "$url_basename" | grep '\*' ; then
	wget --progress=dot:giga -r --no-parent --no-directories --level 1 "--directory-prefix=$tmpdir" -A "$url_basename" $wget_opts "$@" "$(dirname "$url")/"
    else
	wget --progress=dot:giga -r --no-parent --no-directories --level 1 "--directory-prefix=$tmpdir" $wget_opts "$@" "$url"
    fi

    local count=-1
    for i in "$tmpdir"/$url_basename; do
	mv "$i" .
	count=$((count+1))
    done

    rm -rf "$tmpdir"

    return $count
    )
}

# Fetch a tarball using wget_wildcard_url and untar it into a directory
# named after the tarball.
# $1 -- URL
# $2 -- base directory to untar to
# $3 -- extra tar options, e.g. "--strip-components 1"
untar_url ()
{
    (
    set -eu -o pipefail
    local url="$1"
    local basedir="$2"
    local taropts="$3"
    local tarball
    local dirname

    wget_wildcard_url "$url"
    # shellcheck disable=SC2046
    tarball="$(ls $(basename "$url"))"
    dirname="$basedir/${tarball%.tar*}"
    mkdir "$dirname"
    tar xf "$tarball" --directory "$dirname" $taropts
    echo "$dirname"
    )
}

# Wait until the ssh server is ready to accept connexions
# $1: host
# $2: retry count; use "" for the default
# $3+: ssh options
# Returns 0 on success, 1 in case of error
wait_for_ssh_server ()
{
    (
    set -euf -o pipefail
    local session_host="$1"
    local count="${2:-20}"
    shift 2
    local -a session_opts=("$@")

    while [ $count -gt 0 ]
    do
	timeout 30s ssh "${session_opts[@]}" $session_host true && break
	echo "SSH server not ready, waiting....."
	sleep 5
	count=$((count - 1))
    done

    if [ $count -eq 0 ]; then
	echo "ERROR: SSH server did not respond (ssh ${session_opts[*]} $session_host)"
	return 1
    fi
    return 0
    )
}

# Print CPU share allocation for $task and $weight.
# $1: task
# $2: weight
print_cpu_shares ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local weight="$2"
    local cpus
    cpus=$(( $weight * 1000 )) # 1000 cpu shares per executor
    echo "$cpus"
    )
}

# Print memory allocation for $task and $weight.
# $1: task
# $2: weight
# $3: number of expected parallel processes
# $4: amount of system RAM in MB
print_memory_limit ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local weight="$2"
    local nproc="$3"
    local memlimit="$4"
    local memory
    case "$task" in
	build|precommit)
	    if [ "$memlimit" -lt "4000" ]; then
		# Don't limit memory on machines with less than 4GB RAM.
		memory="unlimited"
	    else
		# We want to have at least 2GB of RAM for every core.  E.g.,
		# on a machine with 32 cores and 128GB RAM we can run
		# 2 concurrent builds, while on a 32-core machine with
		# 64GB RAM we can run only 1 build at a time.
		# Note that number of concurrent builds is controlled by number
		# of node executors in jenkins.
		memory=$(( 2000 * $weight * $nproc ))

		# Also, trim 5% off total RAM to have a bit of RAM reserved
		# for processes on the bare machine, which really helps when
		# build container goes into swap.
		memlimit=$(( $memlimit * 95 / 100 ))

		if [ "$memory" -gt "$memlimit" ]; then
		    memory="$memlimit"
		fi
	    fi
	    ;;
	test)
	    # 0.75GB per session
	    memory=$(( 750 * $weight ))
	    ;;
	bench)
	    memory="unlimited"
	    ;;
    esac
    echo "$memory"
    )
}

# Print PID allocation for $task and $weight.
# $1: task
# $2: weight
print_pids_limit ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local weight="$2"
    local pids

    # On startup of GCC's guality tests we have $NCPUs guality_check$PID.exe
    # processes, each of which forks into GDB with ($NCPUs+1) threads.
    # This means that on a 160-core system we need around 30k PID limit.
    pids=$(nproc --all)
    pids=$(( pids * (pids + 1) + 5000 ))
    pids=$(( pids * weight ))

    # Make sure we are using at most half of system PID limit
    local pid_max_2
    pid_max_2=$(cat /proc/sys/kernel/pid_max)
    pid_max_2=$(( pid_max_2 / 2 ))

    if [ $pids -gt $pid_max_2 ]; then
	pids=$pid_max_2
    fi

    echo "$pids"
    )
}

# Print default bind and volume mounts for $task and $job
# $1: task
# $2: job
# $3: Suffix to be appended to the volume names (e.g., -$container_arch-$distro)
# $4+: ssh command
print_mounts ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local job="$2"
    local suffix="$3"
    shift 3
    local ssh=("$@")

    if [ "${WORKSPACE+set}" = "set" ]; then
	case $task in
	    bench|build)
		echo "$WORKSPACE:$WORKSPACE"
		;;
	    precommit)
		# Note the difference between "-v $WORKSPACE:$WORKSPACE" above
		# and "-v $WORKSPACE" here.  In the above case $WORKSPACE is
		# bind-mounted from the host; but in this case a scratch volume
		# is created and mounted inside container.
		echo "$WORKSPACE"
		# The only reason why we are bind-mounting base-artifacts/
		# is that it can be very big.  Rsync-ing tens of gigs back and
		# forth can take as much time as the actual pre-commit test.
		# As a nice side-effect having base-artifacts/ read-only checks
		# that our build scripts don't try to modify it by mistake.
		echo "$WORKSPACE/base-artifacts:$WORKSPACE/base-artifacts:ro"
		;;
	esac
    fi

    case $task in
	build|precommit)
	    echo /home/tcwg-buildslave/snapshots-ref:/home/tcwg-buildslave/snapshots-ref:ro
	    ;;
	bench)
	    echo /home/shared/git:/home/shared/git:ro
	    ;;
    esac

    local key
    for key in $("${ssh[@]}" find /etc/ssh/ -name "ssh_host_*_key" \
			     -o -name "ssh_host_*_key.pub"); do
	echo "$key:$key:ro"
    done

    case "$job" in
	tcwg_*-*)
	    # Add ccache volume for tcwg_* jobs.
	    # These jobs depend on ccache for fast rebuilds of LLVM and GCC with
	    # the host compiler.
	    local prefix
	    if [ x"${WORKSPACE+set}" = x"set" ]; then
		prefix=$(basename $WORKSPACE)
	    else
		prefix=$(echo $job | cut -d- -f 1)
	    fi
	    # tcwg_* jobs use per-executor WORKSPACES, and we configure ccache
	    # to use CCACHE_BASEDIR=$WORKSPACE so that ccache sees same paths
	    # for builds on different executors.
	    # Strip "_$EXECUTOR_NUMBER" from the job/workspace ID.
	    prefix="${prefix%_[0-9]*}"

	    local volume_id
	    volume_id=$(print_docker_name "$prefix$suffix")

	    local readonly=""
	    if [ "$task" = "precommit" ]; then
		readonly=":ro"
	    fi
	    echo "ccache-$volume_id:$HOME/.ccache$readonly"
	    ;;
    esac

    case "$job" in
	tcwg_bmk*)
	    assert_with_msg "Precommit benchmarking requires more thought" \
			    [ "$task" != "precommit" ]
	    # Add scratch mount for tcwg-benchmark's $HOME.
	    # tcwg_bmk-* jobs trigger tcwg-benchmark jenkins jobs, which
	    # then ssh to the build container to compile benchmark objects
	    # and then link them into executables (via ssh:// toolchain_url
	    # parameter -- see tcwg_bmk-build.sh:benchmark()).
	    # This generates a fair bit of disk trafic on /home/tcwg-benchmark,
	    # and it's best to use docker scratch volume, rather than overlayfs.
	    echo /home/tcwg-benchmark
	;;
    esac
    )
}

# Return zero if bash array is defined.
# $1: Name of bash array
test_array()
{
    local var
    var="$1[@]"
    if [ x"${!var+set}" = x"set" ]; then
	return 0
    else
	return 1
    fi
}

# Manifest filename for manifest_out and convert_args_to_variables to write to.
# This is set using "%%" directive to convert_args_to_variables.
# This is a stack maintained by manifest_push and manifest_pop.
__manifest_filename=("/dev/null")

# Set new file name for manifest
# $1: File name
# $2: Optional true/false on whether start a new manifest
manifest_push ()
{
    local filename="$1"
    local clean="${2-true}"

    # Resolve absolute path to manifest.
    local dir
    dir=$(dirname "$filename")
    mkdir -p "$dir"
    dir=$(cd "$dir"; pwd)

    __manifest_filename=("$dir/$(basename "$filename")" "${__manifest_filename[@]}")
    if $clean; then
	rm -f "${__manifest_filename[0]}"
    fi
}

# Return to previous manifest filename
manifest_pop ()
{
    __manifest_filename=("${__manifest_filename[@]:1}")
}

# Output stdout to the manifest file.  Most common case would be
# cat << EOF | manifest_out
# # Component revision
# component_rev="$component_rev"
# EOF
manifest_out ()
{
    cat >> "${__manifest_filename[0]}"
}

# Fetch and print value from manifest
# $1: Manifest file
# $2: Variable to fetch
# $3: Whether to ignore lack of the variable or lack of the manifest
get_manifest ()
{
    (
    set +x
    set -euf -o pipefail
    local manifest="$1"
    local var="$2"
    local strict="${3-true}"

    # Emtpy result if no manifest found (udpate_baseline=init for instance)
    if ! [ -f "$manifest" ]; then
	if $strict; then
	    return 1
	fi
	return 0
    fi

    # Unwrap $var down to variable name that we can unset.
    local name="$var"
    # {name} -> name
    name=$(echo "$name" | sed -e 's/^{\(.*\)}$/\1/')
    # Strip "+, -, :+, :-" suffixes
    name=$(echo "$name" | sed -e 's/[-+:].*$//')

    # remove any existing declarations of $name
    unset "$name"

    # FIXME: manifest should declare "rr" itself
    declare -A rr

    # shellcheck disable=SC1090
    source "$manifest"

    if ! $strict; then
	# Do not complain about unbound variables
	set +u
    fi

    eval echo "\$$var"
    )
}

# Fetch and print value from manifest of a baseline build
# $1: Variable to fetch.
get_baseline_manifest ()
{
    get_manifest base-artifacts/manifest.sh "$1" false
}

# Fetch and print value from manifest of the current build
# $1: Variable to fetch.
get_current_manifest ()
{
    get_manifest "${rr[top_artifacts]}/manifest.sh" "$1"
}

get_baseline_git ()
{
    (
    set -euf -o pipefail

    local base_artifacts="base-artifacts"

    assert_with_msg "ERROR: No $1 in baseline git" \
		    [ -f "$base_artifacts/git/$1" ]
    cat "$base_artifacts/git/$1"
    )
}

get_current_git ()
{
    (
    set -euf -o pipefail
    assert_with_msg "ERROR: No $1 in current git" \
		    [ -f "${rr[top_artifacts]}/git/$1" ]
    cat "${rr[top_artifacts]}/git/$1"
    )
}

set_current_git ()
{
    (
    set -euf -o pipefail
    mkdir -p ${rr[top_artifacts]}/git
    cat > "${rr[top_artifacts]}/git/$1"
    )
}

# returns the date of the last component ($1) commit
get_baseline_component_date ()
{
    (
    set -euf -o pipefail
    local base_artifacts="base-artifacts"
    assert_with_msg "ERROR: No $1 in current git" \
		    [ -f "$base_artifacts/git/${1}_rev" ]

    git -C "$1" show --no-patch --pretty=%ct "$(cat "$base_artifacts/git/${1}_rev")"
    )
}

get_current_component_date ()
{
    (
    set -euf -o pipefail
    assert_with_msg "ERROR: No $1 in current git" \
		    [ -f "${rr[top_artifacts]}/git/${1}_rev" ]

    git -C "$1" show --no-patch --pretty=%ct "$(cat "${rr[top_artifacts]}/git/${1}_rev")"
    )
}

# Print round-robin components that are being updated in this build
# (the ones using non-baseline branches).
print_updated_components ()
{
    (
    set -euf -o pipefail

    local c delim=""
    for c in ${rr[components]}; do
	if [ x"${rr[${c}_git]}" != x"baseline" ]; then
	    echo -ne "$delim$c"
	    delim=" "
	fi
    done
    echo
    )
}

# Print the single round-robin component being updated in this build.
# Print nothing if multiple components are being updated.
print_single_updated_component ()
{
    (
    set -euf -o pipefail

    local -a updated_components
    IFS=" " read -r -a updated_components <<< "$(print_updated_components)"

    if [ ${#updated_components[@]} -eq 1 ]; then
	echo "${updated_components[0]}"
    fi
    )
}

# Print round-robin components that have new commits in this build
# compared to the baseline.
# This expects all components to be cloned and checked out at appropriate revs.
# During bisect we have only a single component updated by definition, and
# it is guaranteed to have clone_repo() called for it.
print_changed_components ()
{
    (
    set -euf -o pipefail

    local c delim=""
    for c in $(print_updated_components); do
	if [ x"$(get_current_git ${c}_rev)" \
	     != x"$(get_baseline_git ${c}_rev)" ]; then
	    echo -ne "$delim$c"
	    delim=${1- }
	fi
    done
    echo
    )
}

# Breakup changed components into $culprit and the rest of components.
# This will reduce the number of builds when $culprit is responsible for
# majority of regressions.
breakup_changed_components ()
{
    (
    set -euf -o pipefail

    local culprit="${1-}"

    if [ "$culprit" = "" ] \
	   || ! print_changed_components "\n" \
	       | grep "^$culprit\$" >/dev/null; then
	print_changed_components "\n"
    else
	echo "$culprit"
	print_changed_components "\n" | grep -v "^$culprit\$" | tr '\n' ' ' \
	    | sed -e "s/ \$//g"
	echo
    fi
    )
}

# Fetch paths from git history
# $1 -- number of versions to fetch; if none of the paths are present in
#       a particular revision that revision doesn't count towards this number;
#       positive values will fetch the most recent N revisions starting from
#       most recent to less recent;
#       negative values will fetch the oldest -N revisions starting from oldest
#       to less old.
#       As a special case "0" will fetch you all revisions in "positive" order,
#       and "-0" will fetch you all revisions in "negative" order.
# $2 -- git repo
# $3 -- paths in git repo; can be files or directories, only the 1st path existing
#       in a revision is fetched (useful for renamed/moved files)
#
# This function fetches files into a temporary directory (pointed to by the first
# line of output) and prints out paths under that temporary directory for subsequent
# fetches of ${paths[@]}" from appropriate revisions.
# Once one of the paths is found in a given revision, we check it out and
# move on to the next revision.
get_git_history ()
{
    (
    set -euf -o pipefail

    local n_revs="$1"
    local repo="$2"
    shift 2
    local -a paths=("$@")

    local -a rev_list_cmd
    rev_list_cmd=(git -C "$repo" rev-list)
    if [ "$n_revs" = "-0" ] || [ "$n_revs" -lt "0" ]; then
	rev_list_cmd+=(--reverse)
	n_revs=$((-$n_revs))
    fi
    rev_list_cmd+=(HEAD -- "${paths[@]}")

    local rev tmp_root
    tmp_root=$(mktemp -d)
    echo "$tmp_root"

    while read rev; do
	local tmp_dir found path
	tmp_dir="$tmp_root/$rev"
	mkdir "$tmp_dir"

	found=false
	for path in "${paths[@]}"; do
	    git -C "$repo" archive "$rev" -- "$path" | tar -x -C "$tmp_dir" &
	    # "git archive" fails when $path was deleted in $rev.
	    if wait $!; then
		found=true
		break
	    fi
	done

	if $found; then
	    echo "$tmp_dir/$path"
	    n_revs=$(($n_revs-1))
	    if [ $n_revs = 0 ]; then
		break
	    fi
	else
	    rm -r "$tmp_dir"
	fi
    done < <("${rev_list_cmd[@]}")
    )
}

convert_arg_var ()
{
    declare -g "$1=$2"
    cat <<EOF | manifest_out
declare -g "$1=$2"
EOF
}

convert_arg_arr ()
{
    if ! test_array $1; then
	declare -ag $1
	cat <<EOF | manifest_out
declare -ga $1
EOF
    fi
    eval "$1+=(\"$2\")"
    cat <<EOF | manifest_out
$1+=("$2")
EOF
}

convert_arg_declare ()
{
    local name="$1"

    case "$name" in
	*"["*"]")
	    local arr="${1%\[*\]}"
	    if ! test_array $arr; then
		declare -Ag $arr
		cat <<EOF | manifest_out
declare -gA $arr
EOF
	    fi
	    ;;
	*)
	    declare -g "$name"
	    cat <<EOF | manifest_out
declare -g $name
EOF
	    ;;
    esac
}

convert_arg_set ()
{
    eval "$1=\"$2\""
    cat <<EOF | manifest_out
$1="$2"
EOF
}

convert_arg_assarr ()
{
    convert_arg_declare "$1"
    convert_arg_set "$1" "$2"
}

convert_arg_source ()
{
    assert_with_msg "ERROR: manifest/include does not exist: $1" \
		    [ -f "$1" ]
    # shellcheck disable=SC1090
    source "$1"
    echo "# Start of include $1" | manifest_out
    cat "$1" | manifest_out
    echo "# End of include $1" | manifest_out
}

# Process "--var value" and "++arr elem" arguments and define corresponding
# variables and arrays.
# "--var value" defines shell variable "$var" to "value".
# "__var value" defines shell variable "$var" to "value", but doesn't store
#               it to the manifest.  This is useful for passing secrets.
# "++arr elem" defines shell array "$arr[@]" and adds "elem" to it.
# "==arr[key] value" defines shell associative array "$arr[@]" and sets
#                    "${arr[key]}" to "value".
# "@@ file" sources file.
# "@@artifacts_var dir" defines artifacts directory and sources the manifest in
#                       from dir/manifest.sh.  This is useful for reproducing
#                       builds.
# "%%artifacts_var dir" defines artifacts directory and starts manifest in
#                       dir/manifest.sh.  Also see "^^ true".
# "^^ true/false %%artifacts_var dir" whether to reproduce the build using manifest.
#     If "true" -- source dir/manifest.sh instead of generating it, then discard
#     all following options up to separator "--".
#     If "false" -- do nothing and proceed as usual.
#
# Shell array $CONVERTED_ARGS is set to the arguments processed.
# Shell variable $SHIFT_CONVERTED_ARGS is set to number of arguments processed.
# $@: Pairs of def/val arguments, stops at "--" marker.
convert_args_to_variables ()
{
    local arr name num
    local total="0"
    eval "CONVERTED_ARGS=(--)"
    while [ $# -gt 0 ]; do
	case "$1" in
	    "--")
		# Finish marker
		total=$(($total+1))
		shift 1
		break
		;;
	    "--"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_var "${1#--}" "$2"
		num=2
		;;
	    "__"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		name="${1#__}"
		# FIXME: Can we add "set +x" here?
		declare -g "$name=$2"
		num=2
		;;
	    "++"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_arr "${1#++}" "$2"
		num=2
		;;
	    "=="*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_assarr "${1#==}" "$2"
		num=2
		;;
	    "@@")
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_source "$2"
		num=2
		;;
	    "@@"*)
		# TODO: It should be possible to simplify handling of "^^"
		# now that we have @@artifacts dir.
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]

		name="${1#@@}"

		# FIXME: This should not be necessary since manifests should
		# "declare -Ag rr" themselves, but current manifests don't
		# do that, due to "declare -A rr" in round-robin.sh.  That
		# declaration makes convert_arg_declare think that rr was
		# already added to the manifest.
		convert_arg_declare "$name"

		convert_arg_source "$2/manifest.sh"
		manifest_push "$2/manifest.sh" false

		# Builds are supposed to be re-runnable from different
		# directories, so do not put artifacts directory into manifest.
		eval "$name=\"$2\""

		num=2
		;;
	    "%%"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		manifest_push "$2/manifest.sh"
		cat <<EOF | manifest_out
declare -g "jenkins_scripts_rev=$(git -C "$(dirname "$0")" rev-parse HEAD)"
# Artifacts directory
EOF
		name="${1#%%}"
		convert_arg_declare "$name"

		# Builds are supposed to be re-runnable from different
		# directories, so do not put artifacts directory into manifest.
		eval "$name=\"$2\""

		cat <<EOF | manifest_out
# Recording parameters to manifest: $2/manifest.sh
EOF
		num=2
		;;
	    "^^")
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 4 ]
		if [ x"$2" = x"true" ]; then
		    name="${3#%%}"
		    case "$name" in
			*"["*"]") convert_arg_assarr "$name" "$4" ;;
			*) convert_arg_var "$name" "$4" ;;
		    esac
		    convert_arg_source "$4/manifest.sh"

		    # Skip processing all following arguments.
		    num=0
		    for i in "$@"; do
			if [ x"$i" = x"--" ]; then
			    break
			fi
			num=$(($num+1))
		    done
		else
		    num=2
		fi
		;;
	    *)
		echo "ERROR: option does not start with '--' / '++' / '@@' / '%%' : $1"
		exit 1
		;;
	esac
	total=$(($total+$num))
	while [ $num -gt 0 ]; do
	    eval "CONVERTED_ARGS+=(\"$1\")"
	    shift 1
	    num=$(($num-1))
	done
    done
    eval "SHIFT_CONVERTED_ARGS=$total"
}

# Check that varible names in "$@" are set
# $@: names of shell variable
obligatory_variables ()
{
    (
	set -euf -o pipefail
	for i in "$@"; do
	    case "$i" in
		*"["*"]")
		    if eval "[ x\"\${$i+set}\" != x\"set\" ]"; then
			echo "ERROR: required parameter $i not set"
			exit 1
		    fi
		    ;;
		*)
		    if [[ "$(declare -p "$i" 2>/dev/null)" \
			      != "declare "* ]]; then
			echo "ERROR: required parameter $i not set"
			exit 1
		    fi
		    ;;
	    esac
	done
    )
}

# Print docker-friendly name for a container
# $1: docker name
print_docker_name ()
{
    (
    set -euf -o pipefail
    local name="$1"
    echo "$name" | sed -e "s#[/=,+]#-#g"
    )
}

# Print GNU-friendly triplet for "uname -m" target
# $1: "uname -m"-style target or "native"
print_gnu_target ()
{
    (
    set -euf -o pipefail
    local target="$1"

    case "$target" in
	"aarch64") target="aarch64-linux-gnu" ;;
        arm*_eabi) target="arm-eabi" ;;
        thumb*_eabi) target="arm-eabi" ;;
	"arm"*) target="arm-linux-gnueabihf" ;;
	"woa64") target="aarch64-w64-mingw32" ;;
	"x86_64") target="x86_64-linux-gnu" ;;
	"native")
	    case "$(uname -m)" in
		"aarch64") target="aarch64-unknown-linux-gnu" ;;
		"armv7l") target="armv7l-unknown-linux-gnueabihf" ;;
		"armv8l") target="armv8l-unknown-linux-gnueabihf" ;;
		"x86_64") target="x86_64-pc-linux-gnu" ;;
		*)
		    echo "ERROR: Unknown native target $(uname -m)" >&2
		    exit 1
		    ;;
	    esac
	    ;;
	*) echo "ERROR: Unknown target $target" >&2; exit 1 ;;
    esac
    echo "$target"
    )
}

# Print LLVM-friendly target for "uname -m" target
# $1: "uname -m"-style target or "native"
print_llvm_target ()
{
    (
    set -euf -o pipefail
    local target="$1"

    if [ x"$target" = x"native" ]; then
	target=$(uname -m)
    fi
    case "$target" in
	"aarch64") target="AArch64" ;;
	"arm"*) target="ARM" ;;
	"x86_64") target="X86" ;;
	*) echo "ERROR: Unknown target $target" >&2; exit 1 ;;
    esac
    echo "$target"
    )
}

# Print Linux make-friendly target for "uname -m" target
# $1: "uname -m"-style target or "native"
print_kernel_target ()
{
    (
    set -euf -o pipefail
    local target="$1"

    if [ x"$target" = x"native" ]; then
	target=$(uname -m)
    fi
    case "$target" in
	"aarch64") target="arm64" ;;
	"arm"*) target="arm" ;;
	*) echo "ERROR: Unknown target $target" >&2; exit 1 ;;
    esac
    echo "$target"
    )
}

# Thoroughly clean git repo, leave only .git/ directory
# $1: Git clone directory
git_clean () {
    (
    set -euf -o pipefail
    local dir="$1"
    shift

    fresh_dir "$dir" "$dir/.git/*"
    if ! git -C "$dir" reset -q --hard "$@"; then
	# "git reset" may fail if index gets corrupted -- remove it and retry.
	rm -f "$dir/.git/index"
	git -C "$dir" reset -q --hard "$@"
    fi
    )
}

# Push HEAD of git repo to a given remote/branch
# $1: Git clone dir
# $2: remote name
# $3: branch to force push to
git_push ()
{
    (
    set -euf -o pipefail
    local dir="$1"
    local remote="$2"
    local branch="$3"

    cd $dir
    case "$branch" in
	"refs/"*) ;;
	*) branch=refs/heads/$branch ;;
    esac
    git push --force $remote HEAD:$branch
    )
}

# Initialize run_step state
# $1: Step to start execution at (or "" to start at the very first step)
#     Appending "+" to the step name, e.g., "__start_at reset_artifacts+"
#     makes us start on the step right AFTER the specified step.
# $2: Step to finish execution at (or "" to run till the very end)
# $3: Top artifact directory
# $4: Whether to enable "set -x" verbosity for execution steps.
run_step_init ()
{
    run_step_start_at="$1"
    run_step_finish_at="$2"
    run_step_top_artifacts="$3"
    run_step_verbose="$4"

    echo "run_step_init: starting at step \"$run_step_start_at\" \
finishing at step \"$run_step_finish_at\""

    run_step_count="0"
    run_step_prev_step=""
    run_step_active=false
    run_step_status=0
    run_step_artifacts=""

    # We need absolute paths for $run_step_artifacts, which is constructed from
    # $run_step_top_artifacts.
    mkdir -p "$run_step_top_artifacts"
    run_step_top_artifacts=$(cd "$run_step_top_artifacts"; pwd)

    rm -f $run_step_top_artifacts/console.log
    rm -f $run_step_top_artifacts/console.log.xz
    rm -f $run_step_top_artifacts/results

    # If no manifest file was provided, supply a default one.
    if [ ${#__manifest_filename[@]} -eq 1 ]; then
	manifest_push "$run_step_top_artifacts/manifest.sh"
    fi
}

# Patch environment for subsequent steps.  This works by generating
# a source-able file patch-env.sh in the artifacts of the current step.
# Run_step() then sources this file to update the environment.
# Note that we build walls around individual steps on purpose.  This allows
# us to SKIP several initial steps during bisect builds, and have a clear
# record of environment modifications in artifacts/NN-step/patch-env.sh
# scripts, which could be applied in correct order.
#
# $@: parameters in the format that convert_args_to_variables() understands.
run_step_patch_env ()
{
    # !!! Each step is limited to a single invocation of run_step_patch_env()
    # !!! due to manifest_push() re-writing the manifest.
    assert_with_msg "patch-env.sh manifest already exists" \
		    ! [ -e $run_step_artifacts/patch-env.sh ]
    manifest_push $run_step_artifacts/patch-env.sh
    convert_args_to_variables "$@"
    manifest_pop
}

# Run execution step and handle its failure as requested
# This function manages
#   1. step skipping -- skip steps before START_AT and after FINISH_AT,
#      as well as "skip_on_fail" steps during failure.
#   2. artifact handling -- create/clean artifact directories per step.
#      Also, copy baseline artifacts for steps before START_AT to simulate
#      skipped steps.
#      Step commands have $run_step_artifacts pointing to artifact directory
#      for current step.
#   3. logging -- dump stdout and and stderr output of step commands
#      into per-step console.log files
#   4. result handling -- output provided success result to artifacts/results
#      for successful steps.  Special value "x" means to let the step itself
#      update artifacts/results.  Results are written to artifacts/results
#      for both skipped and executed steps as long as $run_step_status doesn't
#      indicate failure.
#   5. in run mode "stop_on_fail" -- kill script if $run_step_status indicates
#      failure, otherwise run the step and kill script on failure.
#   6. in run mode "skip_on_fail" -- skip step if $run_step_status indicates
#      a failure, otherwise run the step and set $run_step_status to the exit
#      status of the step.  Normally, a series of "skip_on_fail" steps should
#      be followed by "reset_on_fail" step that handles the cumulative result.
#   7. in run mode "reset_on_fail" -- run the step regardless and set
#      $run_step_status to the exit status of the step (thus resetting it).
# $1: Run mode: stop_on_fail, skip_on_fail, reset_on_fail.
# $2: Result to write to artifacts/results in the absence of failures.
# $@: Step command and its arguments
run_step ()
{
    local run_mode="$1"
    local success_result="$2"
    shift 2

    local -a step
    local pretty_step

    step=("$@")

    if [ "$success_result" != "x" ]; then
	cat >> $run_step_top_artifacts/results <<EOF
# ${step[@]}:
EOF
    fi

    pretty_step="$1"
    shift
    while [ $# -gt 0 ]; do
	if [ x"$1" = x"--" ]; then
	    break
	fi
	pretty_step="$pretty_step-$1"
	shift
    done
    pretty_step=$(echo "$pretty_step" | tr " /" "-")

    run_step_count=$(($run_step_count+1))

    local full_step_name
    full_step_name=$(printf "%02d" $run_step_count)-$pretty_step
    # This is used when accessing the workspace
    run_step_artifacts=$run_step_top_artifacts/$full_step_name

    # Start running steps if:
    # the current step is the starting step OR
    # we haven't run any steps yet and
    # there is no set starting step
    if [ "$pretty_step" = "$run_step_start_at" ] \
	   || [ "${run_step_prev_step}+" = "$run_step_start_at" ] \
	   || ( [ "$run_step_start_at" = "" ] \
		    && [ "$run_step_prev_step" = "" ] ); then
	run_step_active=true
    fi

    if $run_step_active; then
	local skip=false
	case "$run_step_status:$run_mode" in
	    0:*) ;;
	    "$EXTERNAL_FAIL:stop_on_fail")
		echo "STOPPING before ${step[*]} due to previous external failure"
		return $EXTERNAL_FAIL
		;;
	    *:stop_on_fail)
		echo "STOPPING before ${step[*]} due to previous internal failure"
		return $INTERNAL_FAIL
		;;
	    *:skip_on_fail)
		echo "SKIPPING ${step[*]} due to previous failure"
		skip=true
		;;
	    *:reset_on_fail)
		echo "HANDLING ${step[*]} will handle previous failure"
		;;
	    *)
		assert false
	esac

	if ! $skip; then
	    local log_url=""
	    if [ -v BUILD_URL ]; then
	        # Link to jenkins, valid once the job has finished
	        log_url="(${BUILD_URL}artifact/artifacts/$full_step_name/console.log.xz)"
	    fi

	    rm -rf "$run_step_artifacts"
	    mkdir -p "$run_step_artifacts"

	    echo "RUNNING ${step[*]}; see tail -f $run_step_artifacts/console.log" $log_url
	    run_step_status=0
	    # We are running "${step[@]}" in a sub-shell, so that any
	    # modifications to environment will be lost.
	    # The steps can modify environment for subsequent steps by using
	    # run_step_patch_env().
	    # We redirect stdout and stderr of "${step[@]} to a pipe, which
	    # is connected to timestamping console.  Piping "step | ts -s"
	    # directly causes weird issue with failed exit code always being
	    # "1" instead of, e.g., 125.

	    local pipe step_pid ts_pid
	    pipe=$(mktemp -u)
	    mkfifo "$pipe"

	    (
		if $run_step_verbose; then
		    set -x
		else
		    set +x
		fi
		"${step[@]}"
	    ) &> "$pipe" &
	    step_pid=$!

	    ts -s "%T" < "$pipe" > $run_step_artifacts/console.log &
	    ts_pid=$!

	    wait $step_pid || run_step_status=$?
	    wait $ts_pid
	    rm "$pipe"

	    xz $run_step_artifacts/console.log

	    if [ x"$success_result" != x"x" ] \
		   && [ x"$run_step_status" != x"0" ]; then
		cat >> $run_step_top_artifacts/results <<EOF
# FAILED
EOF
	    fi

	    case "$run_step_status:$run_mode" in
		0:*) ;;
		"$EXTERNAL_FAIL:stop_on_fail"|"$EXTERNAL_FAIL:reset_on_fail")
		    echo "STOPPING at ${step[*]} due to external failure"
		    return $EXTERNAL_FAIL
		    ;;
		*:stop_on_fail|*:reset_on_fail)
		    echo "STOPPING at ${step[*]} due to failure"
		    return $INTERNAL_FAIL
		    ;;
		*:skip_on_fail)
		    echo "CARRYING ON after failure in ${step[*]}"
		    ;;
		*)
		    assert false
	    esac
	fi
    else
	echo "SKIPPING ${step[*]}"
    fi

    if [ x"$success_result" != x"x" ] && [ x"$run_step_status" = x"0" ]; then
	cat >> $run_step_top_artifacts/results <<EOF
$success_result
EOF
    fi

    if [ -f $run_step_artifacts/patch-env.sh ]; then
	# shellcheck disable=SC1090
	source $run_step_artifacts/patch-env.sh
    fi

    if [ x"$pretty_step" = x"$run_step_finish_at" ]; then
	run_step_active=false
    fi

    run_step_prev_step="$pretty_step"
}

# Print Python style traceback from a trap EXIT
# More readable version of running "caller"
# Use this as follows:
# trap 'print_traceback' EXIT
# Then remove the trap at the end of your script:
# trap "" EXIT
# Use $INTERNAL_FAIL in your code to mark a non zero return from
# an expected internal failure (e.g., failure to build, test regression, etc.).
# Use $EXTERNAL_FAIL in your code to mark a non zero return from
# a foreseeable external failure (e.g., git server going down or benchmarking
# infrastructure failure).
# The main difference between $INTERNAL_FAIL and $EXTERNAL_FAIL is that
# $INTERNAL_FAIL can be bisected and reduced at the commit or source code
# level.  Conversely, there is no point in bisecting $EXTERNAL_FAIL problems,
# and we should just ignore the current failure and wait for the external
# system to return back to life.
# Any other return code will get you a
# traceback (including assert/assert_with_msg)
INTERNAL_FAIL=123
EXTERNAL_FAIL=125
print_traceback ()
{
  local exit_status=$?
  case $exit_status in
      "$INTERNAL_FAIL"|"$EXTERNAL_FAIL") ;;
      *)
	  echo "ERROR Traceback (most recent call last):"
	  # Show most recent calls last
	  # >=1 to skip the trap handler entry
	  # Start from end-2 to skip the top level "main" entry
	  # which isn't useful
	  for (( i=${#FUNCNAME[@]}-2 ; i>=1 ; i-- )) ; do
	      source_file=${BASH_SOURCE[$i+1]}
	      line_no=${BASH_LINENO[$i]}
	      echo "  File: $source_file, line $line_no"
	      # Remove leading whitespace to keep indentation readable
	      echo "    $(sed -e "${line_no}!d" -e 's/^[[:space:]]*//' "$source_file")"
	  done
	  # We don't know the line number of the exit itself when we trap EXIT
	  echo "  File: ${BASH_SOURCE[0]}, line ${BASH_LINENO[0]}"
	  echo "    (trap handler, exit line unknown, exit status was $exit_status)"
	  ;;
  esac
}

# Print destination sub-directory of interesting-commit.git for ...
# $1: component
# $2: sha1 of the commit
# $3: ci_project
# $4: ci_config
interesting_subdir ()
{
    local dir="$1/sha1"				# $component/sha1
    if [ $# -ge 2 ]; then dir="$dir/$2"; fi	# /$sha1
    if [ $# -ge 3 ]; then dir="$dir/$3"; fi	# /$ci_project
    if [ $# -ge 4 ]; then dir="$dir/$4"; fi	# /$ci_config
    echo "$dir"
}

# Print user-friendly "git describe" of a given commit
# $1: Component (gcc, llvm, etc.)
# $2: Commit hash
# $3: If "true", never fail to describe and print out something sensible.
#     Otherwise return empty string on failure.
describe_sha1 ()
{
    local component="$1"
    local sha1="$2"
    local anything="$3"

    local -a match=()
    case "$component" in
	gcc) match=(--match "basepoints/*" --match "releases/*") ;;
	binutils) match=(--match "binutils*") ;;
	gdb) match=(--match "gdb*") ;;
	newlib) match=(--match "newlib*") ;;
    esac

    if ! git -C "$component" describe "${match[@]}" $sha1 2>/dev/null \
	    && $anything; then
	echo "$component#$(git -C "$component" rev-parse --short $sha1)"
    fi
}

# To avoid committing unwanted files into git (e.g., raw benchmarking
# data) we implement "annex" support.  Files in base-artifacts/annex
# can be either symlinks to directories or regular files containing
# rsync-able urls.
#
# Here we convert directory symlinks into tarballs, upload to bkp-01
# and replace symlinks with files pointing to their uploaded location.
#
# In git_annex_download we do the opposite: download and extract tarball
# into a temporary directory, and replace the file with a symlink
# to that directory.
#
# The end result is that during a build base-artifacts/annex/bmk-data
# is a symlink with directory-like behavior.  Outside of a build
# base-artifacts/ repo contains a regular file pointing to a tarball
# on a private fileserver.
#
# FIXME: We do not automatically remove annex tarballs when trimming
# or rewriting history.  We rely on tcwg-cleanup-stale-results.sh for that.
#
# $1: git repo
# $2: annex directory inside the repo
# $3: tarball name prefix
git_annex_upload ()
{
    (
    set -euf -o pipefail
    local repo="$1"
    local annex_dir="$2"
    local pretty_id="$3"

    if ! [ -d "$repo/$annex_dir" ]; then
	return 0
    fi

    local n_cpus=0
    if [ "$(getconf LONG_BIT)" = "32" ]; then
	# XZ allocates few hundred megabytes per thread, which can easily
	# exhaust VM in armhf containers on 160-core machines.  Limit xz
	# parallelism to 8.
	n_cpus=$(nproc --all)
	if [ "$n_cpus" -gt "8" ]; then
	    n_cpus=8
	fi
    fi

    # Convert annex symlinks to remote links
    local symlink dir md5 remote_path newlink
    while IFS= read -r -d '' symlink; do
	dir=$(readlink "$repo/$annex_dir/$symlink")

	# Generate MD5 hash of the contents of the annex: find all files
	# and generate md5sum for each of them, and then generate md5sum
	# of that list.
	# We avoid using md5sum of the tarball because using tar with
	# multi-threaded xz compression may produce different tarballs.
	md5=$(cd "$dir"; find -L -type f -print0 | xargs -0 md5sum | sort \
		  | md5sum - | awk '{ print $1 }')

	remote_path="$HOME/$repo/$annex_dir/${pretty_id}$md5.tar.xz"
	newlink="bkp-01.tcwglab:$remote_path"

	# Check if bkp-01.tcwglab already has an annexed tarball with our data.
	# When re-writing history in round-robin-baseline.sh we download and
	# re-upload same data multiple times.  This optimization saves up
	# on compression and upload time.
	if ! ssh -n bkp-01.tcwglab test -f "$remote_path"; then
	    local tarball
	    tarball=$(mktemp --suffix=.tar.xz)
	    chmod 0644 "$tarball"
	    # We have a local link to the annex -- make it remote.
	    XZ_OPT=-T$n_cpus tar cJf "$tarball" -C "$dir" .

	    ssh -n bkp-01.tcwglab mkdir -p "$(dirname "$remote_path")"
	    rsync -a "$tarball" "$newlink"

	    rm "$tarball"
	fi

	# In normal builds files inside $dir will be owned by tcwg-benchmark,
	# so we will fail trying to delete them.  Still, try to delete
	# the directory to avoid running of disk space when re-writing history.
	rm -rf "$dir" &>/dev/null || true

	git -C "$repo" rm "$annex_dir/$symlink"
	# if $symlink is the last file in $annex_dir, then "git rm" will
	# remove the directory as well.  Re-create it.
	mkdir -p "$repo/$annex_dir"
	echo "$newlink" > "$repo/$annex_dir/$symlink"
	git -C "$repo" add "$annex_dir/$symlink"
    done < <(cd "$repo/$annex_dir"; find . -type l -print0)

    # update commit with new links.
    git -C "$repo" commit --amend -C HEAD
    )
}

# $1: git repo
# $2: annex directory inside the repo
git_annex_download ()
{
    (
    set -euf -o pipefail
    local repo="$1"
    local annex_dir="$2"

    # FIXME: Remove workaround for old-style bmk-data after history rewrite.
    if [ -f "$repo/results_id" ]; then
	local link dir
	link=$(cat "$repo/results_id")

	dir=$(mktemp -d)
	rsync -a --del "bkp-01.tcwglab:/home/tcwg-benchmark/results-$link/" \
	      "$dir/"

	rm -rf "${repo:?}/$annex_dir"
	mkdir "$repo/$annex_dir"

	ln -s "$dir" "$repo/$annex_dir/bmk-data"
    fi

    if ! [ -d "$repo/$annex_dir" ]; then
	return 0
    fi

    # Resolve annex links to local symlinks.
    # See round-robin-baseline.sh:push_baseline() for details.
    local linkfile link tarball dir
    while IFS= read -r -d '' linkfile; do
	link=$(cat "$repo/$annex_dir/$linkfile")

	tarball=$(mktemp --suffix=.tar.xz)
	rsync -a "$link" "$tarball"

	dir=$(mktemp -d)
	tar xf "$tarball" -C "$dir"
	rm "$tarball"

	rm "$repo/$annex_dir/$linkfile"
	ln -s "$dir" "$repo/$annex_dir/$linkfile"
    done < <(cd "$repo/$annex_dir"; find . -type f -print0)
    )
}