path: root/jenkins-helpers.sh

#!/bin/bash

sanity_check_pwd ()
{
    (
    set -euf -o pipefail

    if [[ $(pwd) == "/" ]]; then
        echo "pwd is unexpectedly \"/\", exiting"
        exit 1;
    fi
    )
}

# Print absolute path to a file or directory
# $1: Path (must exist)
abs_path ()
{
    (
    set -euf -o pipefail

    echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
    )
}

# Assert that $@[1:] returns success.
# $1 should be a message to print on failure e.g.
# assert "will always fail" false
assert_with_msg ()
{
    (
    set -euf -o pipefail +x

    local failure_message=$1
    shift

    eval "$@" || (echo "$failure_message" && exit 1)
    )
}

# Assert that $@ returns success.
assert ()
{
    (
    set -euf -o pipefail +x

    eval "$@"
    )
}

# Cleanup contents of directory preserving specified parts.
# $1: Directory to clean
# $@: Find patterns of files to preserve (must start with $1)
fresh_dir ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    shift 1

    # Make sure $dir doesn't have "/" at the end.
    dir=$(dirname "$dir/something")

    mkdir -p "$dir"

    find_opts=("!" "-path" "$dir")
    for keep_pattern in "$@"; do
	while : ; do
	    find_opts+=("!" "-path" "$keep_pattern")
	    keep_pattern=$(dirname "$keep_pattern")
	    if [ x"$keep_pattern" = x"$dir" ]; then
		break
	    fi
	    # This assert will trigger if one of keep_patterns doesn't start
	    # with "$dir": eventually dirname will get to either "/" or ".".
	    assert [ "$keep_pattern" != "/" -a "$keep_pattern" != "." ]
	done
    done

    # Make sure we can delete the files
    find "$dir" "${find_opts[@]}" -type d -exec chmod +rwx {} \;
    chmod -R +rw "$dir"
    find "$dir" "${find_opts[@]}" -delete
    )
}

# $@: Jenkins labels
# Prints nodes corresponding to jenkins labels.
print_nodes_in_labels ()
{
    (
    set -euf -o pipefail

    local labels=( "$@" )
    local label
    local tmpfile
    tmpfile=$(mktemp)

    for label in "${labels[@]}"; do
	# Handle gracefully the case where the label contains no
	# machine: we don't want to generate an error, rather return
	# an empty string. Use a tmp file so that we still generate an
	# error in case wget fails.
	wget --retry-connrefused --waitretry=1 -O - https://ci.linaro.org/label/$label/api/json?pretty=true 2>/dev/null > $tmpfile
	grep nodeName $tmpfile | cut -d: -f 2 | sed -e 's/"//g' || true
    done

    rm $tmpfile
    )
}

# $@: Jenkins labels, typically tcwg-t[kx]1_{32/64}-test
# Returns node from one of the labels with least number of running containers.
print_node_with_least_containers ()
{
    (
    set -euf -o pipefail

    local tester_labels=( "$@" )
    local testers
    local load_value
    local tester_min_load_name=""
    local tester_min_load_value="999"
    local ret

    # Re. --random-sort below: shuffle node list to mitigate races
    # when starting multiple containers at the same time
    testers=$(print_nodes_in_labels ${tester_labels[*]} | sort --random-sort)
    for tester in $testers; do
        ret=0
	tester_host=$(print_host_for_node $tester "ignore_fail")
	if [ x"${tester_host}" != x"" ]; then
            load_value=$(timeout 30s ssh ${tester_host} docker ps | wc -l) || ret=$?
            if [ $ret -eq 0 ]; then
		if [ "$load_value" -lt "$tester_min_load_value" ]; then
                    tester_min_load_name=$tester
                    tester_min_load_value=$load_value
		fi
	    fi
	fi
    done
    echo $tester_min_load_name
    )
}

# $1: Jenkins tcwg-*-build label
# Prints out architecture for container image
print_arch_for_label ()
{
    (
    set -euf -o pipefail

    local label="$1"

    case $label in
        tcwg-x86_64-*) echo amd64 ;;
        tcwg-x86_32-*) echo i386 ;;
        tcwg-amp_64-*|tcwg-apm_64-*|tcwg-armv8_64|tcwg-d05_64-*|tcwg-lc_64*|tcwg-sq_64-*|tcwg-thx1_64-*|tcwg-tx1_64-*) echo arm64 ;;
        tcwg-amp_32-*|tcwg-apm_32-*|tcwg-armv8_32|tcwg-d05_32-*|tcwg-sq_32-*|tcwg-tk1_32-*|tcwg-tx1_32-*) echo armhf ;;
        *) echo "ERROR: Unsupported label: $label" >&2; exit 1 ;;
    esac
    )
}

# $1: Jenkins tcwg-*-build label
# Prints out host type
print_type_for_label ()
{
    (
    set -euf -o pipefail

    echo "$1" | sed -e "s/^tcwg-\(.*\)-build\$/\1/"
    )
}

# $1: Jenkins $NODE_NAME
# $2: (optional) "ignore_fail"
# Prints SSH host
print_host_for_node ()
{
    (
    set -euf -o pipefail

    local host

    # All jenkins nodes have either <NODE> or <NODE>.tcwglab entry in
    # .ssh/config (in dockerfiles.git/tcwg-base/tcwg-buildslave/).
    for suffix in "" ".tcwglab"; do
	host="$1$suffix"
	if timeout 30s ssh "$host" true >& /dev/null; then
	    break
	fi
	host=""
    done

    if [ x"$host" = x"" ] && [ x"${2-}" != x"ignore_fail" ]; then
        echo "Error: print_host_for_node() cannot ssh to $1 or $1.tcwglab" >&2
        exit 1
    fi

    echo "$host"
    )
}

# $1: Host name or "localhost".
# Prints docker-friendly arch of host
print_arch_for_host ()
{
    (
    set -euf -o pipefail

    local host="$1"
    local arch

    case "$host" in
	"localhost")
	    arch=$(uname -m)
	    case "$arch" in
		"aarch64") arch="arm64" ;;
		"arm"*) arch="armhf" ;;
		"x86_64") arch="amd64" ;;
		*) echo "ERROR: Unknown uname -m arch: $arch" >&2; exit 1 ;;
	    esac
	    echo "$arch"
	    ;;
	*)
	    # While not strictly correct, print_arch_for_label is relaxed
	    # enough to handle this.
	    print_arch_for_label "$host"
	    ;;
    esac
    )
}

# $1: target triplet
# Prints tester label for remote cross-testing
print_tester_label_for_target ()
{
    (
    set -euf -o pipefail

    local target="$1"

    case "$target" in
        aarch64-linux-gnu_ilp32)
	    # We test ILP32 using QEMU KVM, and TX1s run 3.10 kernel that
	    # doesn't support KVM.  Test on APM builders for now.
	    echo "tcwg-apm_64-build"
	    ;;
	# We allocate all TK1/TX1 boards to benchmarking, so use APMs
	# for cross-testing. This means we no longer test on armv7
	# hardware.
        aarch64-linux*) echo "tcwg-apm_64-test" ;;
        armv8l-linux*) echo "tcwg-apm_32-test" ;;
        arm-linux*) echo "tcwg-apm_32-test" ;;
    esac
    )
}

# Run command on remote machine in given directory via ssh on a given port
# "$1" -- <host>[:<port>[:<dir>[:<ssh_opts>[:<env>]]]]
# "$2, $3, etc" -- command and its arguments
# E.g., remote_exec dev-01.tcwglab::/tmp find -name "my file.bak"
# NOTE: The environment variables are not escaped, so pass only simple things.
#       This is because we want ability to pass multiple variables "a=b c=d",
#       and escaping will make that into a single a="b c=d" variable.
remote_exec ()
{
    (
    set -euf -o pipefail

    local host
    host="$(echo "$1" | cut -d: -f 1)"
    local port
    port="$(echo "$1" | cut -s -d: -f 2)"
    local dir
    dir="$(echo "$1" | cut -s -d: -f 3)"
    local opts
    opts="$(echo "$1" | cut -s -d: -f 4)"
    local env_vars
    env_vars="$(echo "$1" | cut -s -d: -f 5)"
    shift
    local -a cmd
    cmd=()
    # Add quotes to every parameter
    for i in "$@"; do cmd+=("$(printf '%q' "$i")"); done
    # Be careful to prepend statements before ${cmd[@]} only if necessary.
    # E.g., when triggering jobs via jenkins-cli, the command is not a binary,
    # so we can't "exec" it.
    # We use flock if $JENKINS_FLOCK is set.
    ${JENKINS_FLOCK+$JENKINS_FLOCK} ssh $opts ${port:+-p$port} $host "${env_vars:+export $env_vars && }${dir:+cd "$(printf '%q' "$dir")" && exec }${cmd[*]}"
    )
}

# Resolve git ref to sha1
# $1 -- repo directory
# $2 -- branch, tag or refspec
# $3 -- remote name
# $4 -- extra options to git rev-parse
git_rev_parse_1 ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="$3"
    local opts="$4"

    local ret

    cd "$dir"

    # Convert git branch/tag names into SHA1
    local sha1 try_ref
    case "$ref" in
	"refs/"*) try_ref="$ref";;
	*) try_ref="refs/remotes/$remote/$ref" ;;
    esac
    ret=0; sha1=$(git rev-parse $opts "$try_ref" 2>/dev/null) || ret=$?
    if [ $ret -ne 0 ]; then
	# Assume that $ref is already a SHA1
	ret=0; sha1=$(git rev-parse $opts "$ref") || ret=$?
	if [ $ret -ne 0 ]; then
	    echo "ERROR: Cannot parse $ref in repo $dir" >&2
	    exit 1
	fi
    fi
    echo "$sha1"
    )
}

# Resolve git ref to short sha1
# $1 -- repo directory
# $2 -- branch, tag or refspec
# $3 -- (optional) remote name
git_rev_parse ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="origin"

    if [ $# -ge 3 ]; then
	remote="$3"
    fi

    git_rev_parse_1 "$dir" "$ref" "$remote" "--short"
    )
}

# Resolve git ref to full sha1
# $1 -- repo directory
# $2 -- branch, tag or refspec
# $3 -- (optional) remote name
git_rev_parse_long ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="origin"

    if [ $# -ge 3 ]; then
	remote="$3"
    fi

    git_rev_parse_1 "$dir" "$ref" "$remote" ""
    )
}

# Run a command with the timeout program, with retries if the command times
# out.
# $1: is the DURATION to pass to timeout.
# $2: is the maximum number of attempts at running the command.
# $@: remainder is the command to run
# Returns the status from the timeout command.
#
# Example: run_with_timeout_and_retry 90s 3 git checkout /large/repo
function run_with_timeout_and_retry {
    local duration=$1
    shift
    local tries=$1
    shift
    local n=0
    local ret=0
    until [ ${n} -eq ${tries} ]
    do
        echo timeout "${duration}" "$@"
        timeout "${duration}" "$@" &
        ret=0 && wait $! || ret=$?
        if [ ${ret} -eq 0 ]; then break; fi
	# Handle special errors
	case "$@" in
	    # git clone failed, removed the (incomplete) directory
	    # (last parameter).
	    # We could try to be smarter and handle the case where git
	    # clone actually timed-out and where git suggests to retry
	    # the checkout with 'git checkout -f HEAD', but this seems
	    # a bit awkward for a case that happens very rarely.
	    "git clone")
		# In case there was no (optional) $dir parameter, this
		# will expand to the repo URL, which probably does not
		# exist as a directory name. Still, don't try to
		# remove a directory named *://*.
		dir=$(echo "$@" | awk '{print $NF;}')
		case "$dir" in
		    *://*) ;;
		    *)
			if [ -d $dir ]; then
			    rm -rf $dir
			fi
			;;
		esac
		;;
	esac
        n=$((n + 1))
    done
    return ${ret}
}

# Configure git remote
# $1: git repo directory
# $2: remote name
# $3: git url
# $4: optional single branch
git_set_remote ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local remote="$2"
    local url="$3"
    local single_branch="${4-}"

    git -C "$dir" remote rm "$remote" > /dev/null 2>&1 || true
    git -C "$dir" remote add ${single_branch:+-t "$single_branch"} "$remote" "$url"
    )
}

# Clone or update a git repo
# $1 -- repo directory
# $2 -- master git repo
# $3 -- reference git repo (to speedup initial cloning)
# $4 -- single-branch to reduce fetching from remote repo
# $5 -- name of remote
clone_or_update_repo_no_checkout ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local url="$2"
    local reference="$3"
    local single_branch="$4"
    local remote="$5"

    local refopt=""
    case "$reference" in
	auto)
	    local ref_dir
	    for ref_dir in $url $dir; do
		ref_dir=$(basename $ref_dir .git)
		ref_dir="/home/tcwg-buildslave/snapshots-ref/$ref_dir.git"
		if git -C $ref_dir rev-parse --git-dir >/dev/null 2>&1; then
		    refopt="--reference $ref_dir"
		    break
		elif [ -d $ref_dir ]; then
		    refopt="--reference-if-able $ref_dir"
		    break
		fi
	    done
	    ;;
	none) ;;
	*) refopt="--reference $reference" ;;
    esac

    if ! git -C "$dir" status >/dev/null 2>&1; then
	# Git repo doesn't exist or is corrupted.  Make a new clone.
	rm -rf "$dir"
    fi

    if [ -d "$dir" ] && [ x"$refopt" != x"" ] \
	   && [ "$(du -s "$dir/.git" | cut -f 1)" -gt $((1024*1024)) ]; then
	# Current clone has grown above 1GB, and we have a reference repo,
	# which should cut down the size significantly.
	# Redo the clone to save disk space.
	# PS: Unfortunately, I could not find a way to make the current clone
	# use new objects from the reference repo without a full re-clone.
	# Fortunately, a new clone with a reference repo is quick.
	rm -rf "$dir"
    fi

    if ! [ -d "$dir" ]; then
	local single_branch_opt=""
	if [ x"$single_branch" != x"" ]; then
	    single_branch_opt="--single-branch --branch $single_branch"
	fi

	run_with_timeout_and_retry 1h 3 git clone $refopt $single_branch_opt "$url" "$dir"
    else
	# Clean up the clone (this is supposed to re-share objects from
	# reference clone and keep the size of the clone minimal).
	# It's possible that previous GC process was interrupted and left
	# a lock.  Use --force to workaround that.  It should be safe
	# to override the lock since directories should not be shared
	# between concurrent builds.
	#
	# Also, prune all loose objects to avoid "git gc --auto" failing
	# and creating .git/gc.log to warn us.
	rm -f "$dir/.git/gc.log"
	git -C "$dir" gc --auto --force --prune=all
	# Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock
	# These occur when builds are aborted during "git remote update" or similar.
	find "$dir/.git" -name "*.lock" -delete
    fi

    git_set_remote "$dir" "$remote" "$url" "$single_branch"

    local refspec
    if [ x"$single_branch" = x"" ]; then
	run_with_timeout_and_retry 1h 3 git -C "$dir" remote update -p \
				   "$remote" 2>/dev/null
	refspec="+refs/changes/*:refs/changes/*"
    else
	refspec="+refs/heads/$single_branch:refs/remotes/$remote/$single_branch"
    fi
    run_with_timeout_and_retry 1h 3 git -C "$dir" fetch -q $remote $refspec --prune
    )
}

# Checkout branch/ref/SHA1 in a git repo
# $1 -- repo directory
# $2 -- ref to checkout
# $3 -- name of the git remote
git_checkout ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local remote="$3"

    git_clean "$dir"
    # Convert git branch/tag names into SHA1
    local sha1
    sha1=$(git_rev_parse "$dir" "$ref" "$remote")
    # Checkout
    git -C "$dir" checkout --detach "$sha1"
    )
}

# Clone or update a git repo
# $1 -- repo directory
# $2 -- ref to checkout
# $3 -- master git repo
# $4 -- optional reference git repo (to speedup initial cloning)
# $5 -- optional single-branch to reduce fetching from remote repo
# $6 -- optional name of remote (default is "origin")
clone_or_update_repo ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local ref="$2"
    local url="$3"
    local reference="${4-auto}"
    local single_branch="${5-}"
    local remote="${6-origin}"

    clone_or_update_repo_no_checkout "$dir" "$url" "$reference" \
				     "$single_branch" "$remote"

    git_checkout "$dir" "$ref" "$remote"
    )
}

# Wget files from URL that may have wildcards; only the last "basename"
# part of URL is allowed to contain wildcards.  Safe to use on normal URLs.
# Return N-1 of files downloaded, or 127 if no files were downloaded.
# $1 -- URL
# $2,... -- additional parameters to wget
wget_wildcard_url ()
{
    (
    set -eu -o pipefail

    local url="$1"
    shift

    local url_basename
    url_basename="$(basename "$url")"

    local tmpdir
    tmpdir="$(mktemp -d)"

    wget_opts=""
    case "$(echo "$url" | cut -d/ -f3)" in
	*".tcwglab") wget_opts="$wget_opts --no-check-certificate" ;;
    esac

    # $(dirname "$url") may not be a valid URL.  Since we only use '*'
    # as wildcards, check if a '*' is present in $url_basename, and if
    # not, do a direct wget on $url to avoid accessing $(dirname "$url")
    if echo "$url_basename" | grep '\*' ; then
	wget --progress=dot:giga -r --no-parent --no-directories --level 1 "--directory-prefix=$tmpdir" -A "$url_basename" $wget_opts "$@" "$(dirname "$url")/"
    else
	wget --progress=dot:giga -r --no-parent --no-directories --level 1 "--directory-prefix=$tmpdir" $wget_opts "$@" "$url"
    fi

    local count=-1
    for i in "$tmpdir"/$url_basename; do
	mv "$i" .
	count=$((count+1))
    done

    rm -rf "$tmpdir"

    return $count
    )
}

# Fetch a tarball using wget_wildcard_url and untar it into a directory
# named after the tarball.
# $1 -- URL
# $2 -- base directory to untar to
# $3 -- extra tar options, e.g. "--strip-components 1"
untar_url ()
{
    (
    set -eu -o pipefail
    local url="$1"
    local basedir="$2"
    local taropts="$3"
    local tarball
    local dirname

    wget_wildcard_url "$url"
    # shellcheck disable=SC2046
    tarball="$(ls $(basename "$url"))"
    dirname="$basedir/${tarball%.tar*}"
    mkdir "$dirname"
    tar xf "$tarball" --directory "$dirname" $taropts
    echo "$dirname"
    )
}

# Wait until the ssh server is ready to accept connexions
# $1: host
# $2: port
# $3: retry count (optional)
# Returns 0 on success, 1 in case of error
wait_for_ssh_server ()
{
    (
    set -euf -o pipefail
    local session_host="$1"
    local session_port="$2"
    local count="${3-20}"

    while [ $count -gt 0 ]
    do
	timeout 30s ssh ${session_port:+-p$session_port} $session_host true && break
	echo "SSH server not ready, waiting....."
	sleep 5
	count=$((count - 1))
    done

    if [ $count -eq 0 ]; then
	echo "ERROR: SSH server did not respond ($session_host:$session_port)"
	return 1
    fi
    return 0
    )
}

# Print CPU share allocation for $task and $weight.
# $1: task
# $2: weight
print_cpu_shares ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local weight="$2"
    local cpus
    cpus=$(( $weight * 1000 )) # 1000 cpu shares per executor
    echo "$cpus"
    )
}

# Print memory allocation for $task and $weight.
# $1: task
# $2: weight
# $3: number of expected parallel processes
# $4: amount of system RAM in MB
print_memory_limit ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local weight="$2"
    local nproc="$3"
    local memlimit="$4"
    local memory
    case "$task" in
	build)
	    if [ "$memlimit" -lt "4000" ]; then
		# Don't limit memory on machines with less than 4GB RAM.
		memory="unlimited"
	    else
		# We want to have at least 2GB of RAM for every core.  E.g.,
		# on a machine with 32 cores and 128GB RAM we can run
		# 2 concurrent builds, while on a 32-core machine with
		# 64GB RAM we can run only 1 build at a time.
		# Note that number of concurrent builds is controlled by number
		# of node executors in jenkins.
		memory=$(( 2000 * $weight * $nproc ))

		# Also, trim 5% off total RAM to have a bit of RAM reserved
		# for processes on the bare machine, which really helps when
		# build container goes into swap.
		memlimit=$(( $memlimit * 95 / 100 ))

		if [ "$memory" -gt "$memlimit" ]; then
		    memory="$memlimit"
		fi
	    fi
	    ;;
	test)
	    # 0.75GB per session
	    memory=$(( 750 * $weight ))
	    ;;
	bench)
	    memory="unlimited"
	    ;;
    esac
    echo "$memory"
    )
}

# Print PID allocation for $task and $weight.
# $1: task
# $2: weight
print_pids_limit ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local weight="$2"
    local pids
    pids=$(( $weight * 5000 )) # 5000 processes per executor
    echo "$pids"
    )
}

# Print default bind mounts for $task
# $1: task
print_bind_mounts ()
{
    (
    set -euf -o pipefail
    local task="$1"
    local ssh="$2"
    local -a bind_mounts

    case $task in
	bench|build)
	    if [ x"${WORKSPACE+set}" = x"set" ]; then
		bind_mounts+=("$WORKSPACE")
	    fi
	    ;;
    esac

    case $task in
	build) bind_mounts+=(/home/tcwg-buildslave/snapshots-ref:ro) ;;
	bench) bind_mounts+=(/home/shared/git:ro) ;;
    esac

    local key
    for key in $($ssh find /etc/ssh/ -name "ssh_host_*_key" \
		      -o -name "ssh_host_*_key.pub"); do
	bind_mounts+=("$key:ro")
    done

    echo "${bind_mounts[@]:+${bind_mounts[@]}}"
    )
}

# Print default volume mounts for $job
# $1: job
# $2: Suffix to be appended to the volume names (e.g., -$container_arch-$distro)
print_volume_mounts ()
{
    (
    set -euf -o pipefail
    local job="$1"
    local suffix="$2"

    local -a mounts
    local volume_id

    case "$job" in
	tcwg_*-*)
	    # Add ccache volume for tcwg_* jobs.
	    # These jobs depend on ccache for fast rebuilds of LLVM and GCC with
	    # the host compiler.
	    local prefix
	    if [ x"${WORKSPACE+set}" = x"set" ]; then
		prefix=$(basename $WORKSPACE)
	    else
		prefix=$(echo $job | cut -d- -f 1)
	    fi
	    # tcwg_* jobs use per-executor WORKSPACES, and we configure ccache
	    # to use CCACHE_BASEDIR=$WORKSPACE so that ccache sees same paths
	    # for builds on different executors.
	    # Strip "_$EXECUTOR_NUMBER" from the job/workspace ID.
	    prefix="${prefix%_[0-9]*}"
	    volume_id=$(print_docker_name "$prefix$suffix")
	    mounts+=(ccache-"$volume_id":"$HOME"/.ccache)
	    ;;
    esac
    case "$job" in
	tcwg_bmk*)
	    # Add scratch mount for tcwg-benchmark's $HOME.
	    # tcwg_bmk-* jobs trigger tcwg-benchmark jenkins jobs, which
	    # then ssh to the build container to compile benchmark objects
	    # and then link them into executables (via ssh:// toolchain_url
	    # parameter -- see tcwg_bmk-build.sh:benchmark()).
	    # This generates a fair bit of disk trafic on /home/tcwg-benchmark,
	    # and it's best to use docker scratch volume, rather than overlayfs.
	    mounts+=(/home/tcwg-benchmark)
	;;
    esac
    echo "${mounts[@]:+${mounts[@]}}"
    )
}

# Return zero if bash array is defined.
# $1: Name of bash array
test_array()
{
    local var
    var="$1[@]"
    if [ x"${!var+set}" = x"set" ]; then
	return 0
    else
	return 1
    fi
}

# Manifest filename for manifest_out and convert_args_to_variables to write to.
# This is set using "%%" directive to convert_args_to_variables.
# This is a stack maintained by manifest_push and manifest_pop.
__manifest_filename=("/dev/null")

# Set new file name for manifest
# $1: File name
# $2: Optional true/false on whether start a new manifest
manifest_push ()
{
    local filename="$1"
    local clean="${2-true}"

    # Resolve absolute path to manifest.
    local dir
    dir=$(dirname "$filename")
    mkdir -p "$dir"
    dir=$(cd "$dir"; pwd)

    __manifest_filename=("$dir/$(basename "$filename")" "${__manifest_filename[@]}")
    if $clean; then
	rm -f "${__manifest_filename[0]}"
    fi
}

# Return to previous manifest filename
manifest_pop ()
{
    __manifest_filename=("${__manifest_filename[@]:1}")
}

# Output stdout to the manifest file.  Most common case would be
# cat << EOF | manifest_out
# # Component revision
# component_rev="$component_rev"
# EOF
manifest_out ()
{
    cat >> "${__manifest_filename[0]}"
}

# Fetch and print value from manifest of a baseline build
# $1: Variable to fetch.
get_baseline_manifest ()
{
    (
    set +x
    set -euf -o pipefail
    unset rr
    declare -A rr
    # Emtpy result if no manifest found (udpate_baseline=init for instance)
    if [ -f base-artifacts/manifest.sh ]; then
        # shellcheck disable=SC1090
        source base-artifacts/manifest.sh
        eval echo "\$$1"
    fi
    )
}

# Fetch and print value from manifest of the current build
# $1: Variable to fetch.
get_current_manifest ()
{
    (
    set +x
    set -euf -o pipefail
    local top_artifacts=${rr[top_artifacts]}
    unset rr
    declare -A rr
    # shellcheck disable=SC1090
    source $top_artifacts/manifest.sh
    eval echo "\$$1"
    )
}

get_baseline_git ()
{
    (
    set -euf -o pipefail
    assert_with_msg "ERROR: No $1 in baseline git" \
		    [ -f "base-artifacts/git/$1" ]
    cat "base-artifacts/git/$1"
    )
}

get_current_git ()
{
    (
    set -euf -o pipefail
    assert_with_msg "ERROR: No $1 in current git" \
		    [ -f "${rr[top_artifacts]}/git/$1" ]
    cat "${rr[top_artifacts]}/git/$1"
    )
}

set_current_git ()
{
    (
    set -euf -o pipefail
    mkdir -p ${rr[top_artifacts]}/git
    cat > "${rr[top_artifacts]}/git/$1"
    )
}

# Exit with 0 status if given components collectively have a single commit
# compared to baseline.
# $@: components
single_commit_p ()
{
    if [ $# = 1 ]; then
	local c base_rev cur_rev sha1
	c=$1
	base_rev=$(get_baseline_git ${c}_rev)
	cur_rev=$(get_current_git ${c}_rev)
	for sha1 in $(git -C $c rev-parse $cur_rev^@); do
	    if [ x"$sha1" = x"$base_rev" ]; then
		# We have a single-commit build
		return 0
	    fi
	done
    fi
    return 1
}

# Print round-robin components that are being updated in this build
# (the ones using non-baseline branches).
print_updated_components ()
{
    (
    set -euf -o pipefail

    local c delim=""
    for c in ${rr[components]}; do
	if [ x"${rr[${c}_git]}" != x"baseline" ]; then
	    echo -ne "$delim$c"
	    delim=" "
	fi
    done
    echo
    )
}

# Print the single round-robin component being updated in this build.
# Print nothing if multiple components are being updated.
print_single_updated_component ()
{
    (
    set -euf -o pipefail

    local -a updated_components
    IFS=" " read -r -a updated_components <<< "$(print_updated_components)"

    if [ ${#updated_components[@]} -eq 1 ]; then
	echo "${updated_components[0]}"
    fi
    )
}

# Print round-robin components that have new commits in this build
# compared to the baseline.
# This expects all components to be cloned and checked out at appropriate revs.
# During bisect we have only a single component updated by definition, and
# it is guaranteed to have clone_repo() called for it.
print_changed_components ()
{
    (
    set -euf -o pipefail

    local c delim=""
    for c in $(print_updated_components); do
	if [ x"$(get_current_git ${c}_rev)" \
	     != x"$(get_baseline_git ${c}_rev)" ]; then
	    echo -ne "$delim$c"
	    delim=${1- }
	fi
    done
    echo
    )
}

# Breakup changed components into $culprit and the rest of components.
# This will reduce the number of builds when $culprit is responsible for
# majority of regressions.
breakup_changed_components ()
{
    (
    set -euf -o pipefail

    local culprit="${1-}"

    if [ "$culprit" = "" ] \
	   || ! print_changed_components "\n" \
	       | grep "^$culprit\$" >/dev/null; then
	print_changed_components "\n"
    else
	echo "$culprit"
	print_changed_components "\n" | grep -v "^$culprit\$" | tr '\n' ' ' \
	    | sed -e "s/ \$//g"
	echo
    fi
    )
}

# Fetch paths from git history
# $1 -- number of revisions to fetch -- "0" for all; positive for the most recent
#       N revisions; negative for the oldest -N revisions
# $2 -- git repo
# $3 -- paths in git repo; can be files or directories, only the 1st path existing
#       in a revision is fetched (useful for renamed/moved files)
#
# This function fetches files into a temporary directory (pointed to by the first
# line of output) and prints out paths under that temporary directory for subsequent
# fetches of ${paths[@]}" from appropriate revisions.
# Once one of the paths is found in a given revision, we check it out and
# move on to the next revision.
get_git_history ()
{
    (
    set -euf -o pipefail

    local n_revs="$1"
    local repo="$2"
    shift 2
    local -a paths=("$@")

    local -a rev_list_cmd
    rev_list_cmd=(git -C "$repo" rev-list)
    if [ "$n_revs" -lt "0" ]; then
	rev_list_cmd+=(--reverse)
	n_revs=$((-$n_revs))
    fi
    rev_list_cmd+=(HEAD -- "${paths[@]}")

    local rev tmp_root
    tmp_root=$(mktemp -d)
    echo "$tmp_root"

    while read rev; do
	local tmp_dir found path
	tmp_dir="$tmp_root/$rev"
	mkdir "$tmp_dir"

	found=false
	for path in "${paths[@]}"; do
	    git -C "$repo" archive "$rev" -- "$path" | tar -x -C "$tmp_dir" &
	    # "git archive" fails when $path was deleted in $rev.
	    if wait $!; then
		found=true
		break
	    fi
	done

	if $found; then
	    echo "$tmp_dir/$path"
	    n_revs=$(($n_revs-1))
	    if [ $n_revs = 0 ]; then
		break
	    fi
	else
	    rm -r "$tmp_dir"
	fi
    done < <("${rev_list_cmd[@]}")
    )
}

convert_arg_var ()
{
    declare -g "$1=$2"
    cat <<EOF | manifest_out
declare -g "$1=$2"
EOF
}

convert_arg_arr ()
{
    if ! test_array $1; then
	declare -ag $1
	cat <<EOF | manifest_out
declare -ga $1
EOF
    fi
    eval "$1+=(\"$2\")"
    cat <<EOF | manifest_out
$1+=("$2")
EOF
}

convert_arg_assarr ()
{
    local arr="${1%\[*\]}"
    if ! test_array $arr; then
	declare -Ag $arr
	cat <<EOF | manifest_out
declare -gA $arr
EOF
    fi
    eval "$1=\"$2\""
    cat <<EOF | manifest_out
$1="$2"
EOF
}

convert_arg_source ()
{
    assert_with_msg "ERROR: manifest/include does not exist: $1" \
		    [ -f "$1" ]
    # shellcheck disable=SC1090
    source "$1"
    echo "# Start of include $1" | manifest_out
    cat "$1" | manifest_out
    echo "# End of include $1" | manifest_out
}

# Process "--var value" and "++arr elem" arguments and define corresponding
# variables and arrays.
# "--var value" defines shell variable "$var" to "value".
# "__var value" defines shell variable "$var" to "value", but doesn't store
#               it to the manifest.  This is useful for passing secrets.
# "++arr elem" defines shell array "$arr[@]" and adds "elem" to it.
# "==arr[key] value" defines shell associative array "$arr[@]" and sets
#                    "${arr[key]}" to "value".
# "@@ file" sources file.
# "@@artifacts_var dir" defines artifacts directory and sources the manifest in
#                       from dir/manifest.sh.  This is useful for reproducing
#                       builds.
# "%%artifacts_var dir" defines artifacts directory and starts manifest in
#                       dir/manifest.sh.  Also see "^^ true".
# "^^ true/false %%artifacts_var dir" whether to reproduce the build using manifest.
#     If "true" -- source dir/manifest.sh instead of generating it, then discard
#     all following options up to separator "--".
#     If "false" -- do nothing and proceed as usual.
#
# Shell array $CONVERTED_ARGS is set to the arguments processed.
# Shell variable $SHIFT_CONVERTED_ARGS is set to number of arguments processed.
# $@: Pairs of def/val arguments, stops at "--" marker.
convert_args_to_variables ()
{
    local arr name num
    local total="0"
    eval "CONVERTED_ARGS=(--)"
    while [ $# -gt 0 ]; do
	case "$1" in
	    "--")
		# Finish marker
		total=$(($total+1))
		shift 1
		break
		;;
	    "--"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_var "${1#--}" "$2"
		num=2
		;;
	    "__"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		name="${1#__}"
		declare -g "$name=$2"
		num=2
		;;
	    "++"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_arr "${1#++}" "$2"
		num=2
		;;
	    "=="*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_assarr "${1#==}" "$2"
		num=2
		;;
	    "@@")
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_source "$2"
		num=2
		;;
	    "@@"*)
		# TODO: It should be possible to simplify handling of "^^"
		# now that we have @@artifacts dir.
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		convert_arg_source "$2/manifest.sh"
		manifest_push "$2/manifest.sh" false

		name="${1#@@}"
		case "$name" in
		    *"["*"]") convert_arg_assarr "$name" "$2" ;;
		    *) convert_arg_var "$name" "$2" ;;
		esac
		num=2
		;;
	    "%%"*)
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 2 ]
		manifest_push "$2/manifest.sh"
		cat <<EOF | manifest_out
declare -g "jenkins_scripts_rev=$(git -C "$(dirname "$0")" rev-parse HEAD)"
# Artifacts directory
EOF
		name="${1#%%}"
		case "$name" in
		    *"["*"]") convert_arg_assarr "$name" "$2" ;;
		    *) convert_arg_var "$name" "$2" ;;
		esac

		cat <<EOF | manifest_out
# Recording parameters to manifest: $2/manifest.sh
EOF
		num=2
		;;
	    "^^")
		assert_with_msg "ERROR: Parameter value not provided for $1." \
				[ $# -ge 4 ]
		if [ x"$2" = x"true" ]; then
		    name="${3#%%}"
		    case "$name" in
			*"["*"]") convert_arg_assarr "$name" "$4" ;;
			*) convert_arg_var "$name" "$4" ;;
		    esac
		    convert_arg_source "$4/manifest.sh"

		    # Skip processing all following arguments.
		    num=0
		    for i in "$@"; do
			if [ x"$i" = x"--" ]; then
			    break
			fi
			num=$(($num+1))
		    done
		else
		    num=2
		fi
		;;
	    *)
		echo "ERROR: option does not start with '--' / '++' / '@@' / '%%' : $1"
		exit 1
		;;
	esac
	total=$(($total+$num))
	while [ $num -gt 0 ]; do
	    eval "CONVERTED_ARGS+=(\"$1\")"
	    shift 1
	    num=$(($num-1))
	done
    done
    eval "SHIFT_CONVERTED_ARGS=$total"
    cat <<EOF | manifest_out
# Processed $total options
EOF
}

# Check that varible names in "$@" are set
# $@: names of shell variable
obligatory_variables ()
{
    (
	set -euf -o pipefail
	for i in "$@"; do
	    if eval "[ x\"\${$i+set}\" != x\"set\" ]"; then
		echo "ERROR: required parameter $i not set"
		exit 1
	    fi
	done
    )
}

# Print docker-friendly name for a container
# $1: docker name
print_docker_name ()
{
    (
    set -euf -o pipefail
    local name="$1"
    echo "$name" | sed -e "s#[/=,+]#-#g"
    )
}

# Print GNU-friendly triplet for "uname -m" target
# $1: "uname -m"-style target or "native"
print_gnu_target ()
{
    (
    set -euf -o pipefail
    local target="$1"

    case "$target" in
	"aarch64") target="aarch64-linux-gnu" ;;
	"arm_eabi") target="arm-eabi" ;;
	"arm"*) target="arm-linux-gnueabihf" ;;
	"woa64") target="aarch64-w64-mingw32" ;;
	"x86_64") target="x86_64-linux-gnu" ;;
	"native")
	    case "$(uname -m)" in
		"aarch64") target="aarch64-unknown-linux-gnu" ;;
		"armv7l") target="armv7l-unknown-linux-gnueabihf" ;;
		"armv8l") target="armv8l-unknown-linux-gnueabihf" ;;
		"x86_64") target="x86_64-pc-linux-gnu" ;;
		*)
		    echo "ERROR: Unknown native target $(uname -m)" >&2
		    exit 1
		    ;;
	    esac
	    ;;
	*) echo "ERROR: Unknown target $target" >&2; exit 1 ;;
    esac
    echo "$target"
    )
}

# Print LLVM-friendly target for "uname -m" target
# $1: "uname -m"-style target or "native"
print_llvm_target ()
{
    (
    set -euf -o pipefail
    local target="$1"

    if [ x"$target" = x"native" ]; then
	target=$(uname -m)
    fi
    case "$target" in
	"aarch64") target="AArch64" ;;
	"arm"*) target="ARM" ;;
	"x86_64") target="X86" ;;
	*) echo "ERROR: Unknown target $target" >&2; exit 1 ;;
    esac
    echo "$target"
    )
}

# Print Linux make-friendly target for "uname -m" target
# $1: "uname -m"-style target or "native"
print_kernel_target ()
{
    (
    set -euf -o pipefail
    local target="$1"

    if [ x"$target" = x"native" ]; then
	target=$(uname -m)
    fi
    case "$target" in
	"aarch64") target="arm64" ;;
	"arm"*) target="arm" ;;
	*) echo "ERROR: Unknown target $target" >&2; exit 1 ;;
    esac
    echo "$target"
    )
}

# Thoroughly clean git repo, leave only .git/ directory
# $1: Git clone directory
git_clean () {
    (
    set -euf -o pipefail

    fresh_dir "$1" "$1/.git/*"
    git -C "$1" reset -q --hard
    )
}

# Add git remote pointing to linaro's git repo/mirrors with writable
# toolchain/ci/* repo.  Deduce repo's URL from URL of existing
# "origin" git remote.
# $1: Git clone directory (must have "origin" remote configured)
# $2: Name of the new remote.
git_init_linaro_local_remote ()
{
    (
    set -euf -o pipefail
    local dir="$1"
    local remote="$2"

    local origin_url
    local new_url
    origin_url=$(git -C "$dir" remote get-url origin)

    # Figure out mirror repo on linaro's servers.
    new_url="toolchain/ci/$(basename $origin_url)"

    case "$new_url" in
	"toolchain/ci/interesting-commits"*)
	    # We push to "master" branch of interesting-commits, which
	    # gitolite rejects -- gitolite allows pushes only under
	    # refs/heads/linaro-local/*.  Our pushes to interesting-commits
	    # are tiny, so use gerrit.
	    new_url=ssh://review.linaro.org:29418/toolchain/ci/interesting-commits
	    ;;
	*)
	    # Use git-us.l.o to avoid delays between review.l.o and git.l.o
	    # Use gitolite access.  Gerrit's ssh access verifies pushed commits,
	    # which can slow-down server on big pushes.
	    new_url="ssh://git-us.linaro.org/$new_url"
	    ;;
    esac

    git_set_remote "$dir" "$remote" "$new_url"
    )
}

# Push HEAD of git repo to a given remote/branch
# $1: Git clone dir
# $2: remote name
# $3: branch to force push to
git_push ()
{
    (
    set -euf -o pipefail
    local dir="$1"
    local remote="$2"
    local branch="$3"

    cd $dir
    case "$branch" in
	"refs/"*) ;;
	*) branch=refs/heads/$branch ;;
    esac
    git push --force $remote HEAD:$branch
    )
}

# Initialize run_step state
# $1: Step to start execution at (or "" to start at the very first step)
# $2: Step to finish execution at (or "" to run till the very end)
# $3: Top artifact directory
# $4: Whether to enable "set -x" verbosity for execution steps.
run_step_init ()
{
    run_step_start_at="$1"
    run_step_finish_at="$2"
    run_step_top_artifacts="$3"
    run_step_verbose="$4"

    echo "run_step_init: starting at step \"$run_step_start_at\" \
finishing at step \"$run_step_finish_at\""

    run_step_count="0"
    run_step_prev_step=""
    run_step_active=false
    run_step_status=0
    run_step_artifacts=""

    # We need absolute paths for $run_step_artifacts, which is constructed from
    # $run_step_top_artifacts.
    mkdir -p "$run_step_top_artifacts"
    run_step_top_artifacts=$(cd "$run_step_top_artifacts"; pwd)

    rm -f $run_step_top_artifacts/console.log
    rm -f $run_step_top_artifacts/console.log.xz
    rm -f $run_step_top_artifacts/results

    # If no manifest file was provided, supply a default one.
    if [ ${#__manifest_filename[@]} -eq 1 ]; then
	manifest_push "$run_step_top_artifacts/manifest.sh"
    fi
}

# Patch environment for subsequent steps.  This works by generating
# a source-able file patch-env.sh in the artifacts of the current step.
# Run_step() then sources this file to update the environment.
# Note that we build walls around individual steps on purpose.  This allows
# us to SKIP several initial steps during bisect builds, and have a clear
# record of environment modifications in artifacts/NN-step/patch-env.sh
# scripts, which could be applied in correct order.
#
# $@: parameters in the format that convert_args_to_variables() understands.
run_step_patch_env ()
{
    # !!! Each step is limited to a single invocation of run_step_patch_env()
    # !!! due to manifest_push() re-writing the manifest.
    assert_with_msg "patch-env.sh manifest already exists" \
		    ! [ -e $run_step_artifacts/patch-env.sh ]
    manifest_push $run_step_artifacts/patch-env.sh
    convert_args_to_variables "$@"
    manifest_pop
}

# Run execution step and handle its failure as requested
# This function manages
#   1. step skipping -- skip steps before START_AT and after FINISH_AT,
#      as well as "skip_on_fail" steps during failure.
#   2. artifact handling -- create/clean artifact directories per step.
#      Also, copy baseline artifacts for steps before START_AT to simulate
#      skipped steps.
#      Step commands have $run_step_artifacts pointing to artifact directory
#      for current step.
#   3. logging -- dump stdout and and stderr output of step commands
#      into per-step console.log files
#   4. result handling -- output provided success result to artifacts/results
#      for successful steps.  Special value "x" means to let the step itself
#      update artifacts/results.  Results are written to artifacts/results
#      for both skipped and executed steps as long as $run_step_status doesn't
#      indicate failure.
#   5. in run mode "stop_on_fail" -- kill script if $run_step_status indicates
#      failure, otherwise run the step and kill script on failure.
#   6. in run mode "skip_on_fail" -- skip step if $run_step_status indicates
#      a failure, otherwise run the step and set $run_step_status to the exit
#      status of the step.  Normally, a series of "skip_on_fail" steps should
#      be followed by "reset_on_fail" step that handles the cumulative result.
#   7. in run mode "reset_on_fail" -- run the step regardless and set
#      $run_step_status to the exit status of the step (thus resetting it).
# $1: Run mode: stop_on_fail, skip_on_fail, reset_on_fail.
# $2: Result to write to artifacts/results in the absence of failures.
# $@: Step command and its arguments
run_step ()
{
    local run_mode="$1"
    local success_result="$2"
    shift 2

    local -a step
    local pretty_step

    step=("$@")

    pretty_step="$1"
    shift
    while [ $# -gt 0 ]; do
	if [ x"$1" = x"--" ]; then
	    break
	fi
	pretty_step="$pretty_step-$1"
	shift
    done
    pretty_step=$(echo "$pretty_step" | tr " /" "-")

    run_step_count=$(($run_step_count+1))

    local full_step_name
    full_step_name=$(printf "%02d" $run_step_count)-$pretty_step
    # This is used when accessing the workspace
    run_step_artifacts=$run_step_top_artifacts/$full_step_name

    # Start running steps if:
    # the current step is the starting step OR
    # we haven't run any steps yet and
    # there is no set starting step
    if [ x"$pretty_step" = x"$run_step_start_at" ] || \
	   ( [ x"$run_step_start_at" = x"" ] && \
		 [ x"$run_step_prev_step" = x"" ] ); then
	run_step_active=true
    fi

    if $run_step_active; then
	local skip=false
	case "$run_step_status:$run_mode" in
	    0:*) ;;
	    $EXTERNAL_FAIL:stop_on_fail)
		echo "STOPPING before ${step[*]} due to previous external failure"
		return $EXTERNAL_FAIL
		;;
	    *:stop_on_fail)
		echo "STOPPING before ${step[*]} due to previous internal failure"
		return $INTERNAL_FAIL
		;;
	    *:skip_on_fail)
		echo "SKIPPING ${step[*]} due to previous failure"
		skip=true
		;;
	    *:reset_on_fail)
		echo "HANDLING ${step[*]} will handle previous failure"
		;;
	    *)
		assert false
	esac

	if ! $skip; then
	    if [ x"$success_result" != x"x" ]; then
		cat >> $run_step_top_artifacts/results <<EOF
# ${step[@]}:
EOF
	    fi

	    local log_url=""
	    if [ -v BUILD_URL ]; then
	        # Link to jenkins, valid once the job has finished
	        log_url="(${BUILD_URL}artifact/artifacts/$full_step_name/console.log.xz)"
	    fi

	    rm -rf "$run_step_artifacts"
	    mkdir -p "$run_step_artifacts"

	    echo "RUNNING ${step[*]}; see tail -f $run_step_artifacts/console.log" $log_url
	    run_step_status=0
	    # We are running below "eval" in a sub-shell (due to "&"), so
	    # any modifications to environment by "${step[*]}" will be lost.
	    # The steps can modify environment for subsequent steps by using
	    # run_step_patch_env().
	    eval "
if $run_step_verbose; then
  set -x
else
  set +x
fi
${step[*]}" 2>&1 | ts -s "%T" > $run_step_artifacts/console.log &
	    wait $! || run_step_status=$?
	    xz $run_step_artifacts/console.log

	    if [ x"$success_result" != x"x" ] \
		   && [ x"$run_step_status" != x"0" ]; then
		cat >> $run_step_top_artifacts/results <<EOF
# FAILED
EOF
	    fi

	    case "$run_step_status:$run_mode" in
		0:*) ;;
		$EXTERNAL_FAIL:stop_on_fail|$EXTERNAL_FAIL:reset_on_fail)
		    echo "STOPPING at ${step[*]} due to external failure"
		    return $EXTERNAL_FAIL
		    ;;
		*:stop_on_fail|*:reset_on_fail)
		    echo "STOPPING at ${step[*]} due to failure"
		    return $INTERNAL_FAIL
		    ;;
		*:skip_on_fail)
		    echo "CARRYING ON after failure in ${step[*]}"
		    ;;
		*)
		    assert false
	    esac
	fi
    else
	echo "SKIPPING ${step[*]}"
    fi

    if [ x"$success_result" != x"x" ] && [ x"$run_step_status" = x"0" ]; then
	cat >> $run_step_top_artifacts/results <<EOF
$success_result
EOF
    fi

    if [ -f $run_step_artifacts/patch-env.sh ]; then
	# shellcheck disable=SC1090
	source $run_step_artifacts/patch-env.sh
    fi

    if [ x"$pretty_step" = x"$run_step_finish_at" ]; then
	run_step_active=false
    fi

    run_step_prev_step="$pretty_step"
}

# Print Python style traceback from a trap EXIT
# More readable version of running "caller"
# Use this as follows:
# trap 'print_traceback' EXIT
# Then remove the trap at the end of your script:
# trap "" EXIT
# Use $INTERNAL_FAIL in your code to mark a non zero return from
# an expected internal failure (e.g., failure to build, test regression, etc.).
# Use $EXTERNAL_FAIL in your code to mark a non zero return from
# a foreseeable external failure (e.g., git server going down or benchmarking
# infrastructure failure).
# The main difference between $INTERNAL_FAIL and $EXTERNAL_FAIL is that
# $INTERNAL_FAIL can be bisected and reduced at the commit or source code
# level.  Conversely, there is no point in bisecting $EXTERNAL_FAIL problems,
# and we should just ignore the current failure and wait for the external
# system to return back to life.
# Any other return code will get you a
# traceback (including assert/assert_with_msg)
INTERNAL_FAIL=123
EXTERNAL_FAIL=125
print_traceback ()
{
  local exit_status=$?
  case $exit_status in
      $INTERNAL_FAIL|$EXTERNAL_FAIL) ;;
      *)
	  echo "ERROR Traceback (most recent call last):"
	  # Show most recent calls last
	  # >=1 to skip the trap handler entry
	  # Start from end-2 to skip the top level "main" entry
	  # which isn't useful
	  for (( i=${#FUNCNAME[@]}-2 ; i>=1 ; i-- )) ; do
	      source_file=${BASH_SOURCE[$i+1]}
	      line_no=${BASH_LINENO[$i]}
	      echo "  File: $source_file, line $line_no"
	      # Remove leading whitespace to keep indentation readable
	      echo "    $(sed -e "${line_no}!d" -e 's/^[[:space:]]*//' "$source_file")"
	  done
	  # We don't know the line number of the exit itself when we trap EXIT
	  echo "  File: ${BASH_SOURCE[0]}, line ${BASH_LINENO[0]}"
	  echo "    (trap handler, exit line unknown, exit status was $exit_status)"
	  ;;
  esac
}

# Print destination sub-directory of interesting-commit.git for ...
# $1: component
# $2: sha1 of the commit
# $3: ci_project
# $4: ci_config
interesting_subdir ()
{
    local dir="$1/sha1"				# $component/sha1
    if [ $# -ge 2 ]; then dir="$dir/$2"; fi	# /$sha1
    if [ $# -ge 3 ]; then dir="$dir/$3"; fi	# /$ci_project
    if [ $# -ge 4 ]; then dir="$dir/$4"; fi	# /$ci_config
    echo "$dir"
}

# Print user-friendly "git describe" of a given commit
# $1: Component (gcc, llvm, etc.)
# $2: Commit hash
# $3: If "true", never fail to describe and print out something sensible.
#     Otherwise return empty string on failure.
describe_sha1 ()
{
    local component="$1"
    local sha1="$2"
    local anything="$3"

    local -a match=()
    case "$component" in
	gcc) match=(--match "basepoints/*" --match "releases/*") ;;
	binutils) match=(--exclude "users/*") ;;
	newlib) match=(--match "newlib*") ;;
    esac

    if ! git -C "$component" describe "${match[@]}" $sha1 && $anything; then
	echo "$component#$(git -C "$component" rev-parse --short)"
    fi
}