path: root/tcwg_bmk-build.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh
. $scripts/round-robin.sh

convert_args_to_variables "$@"

obligatory_variables rr[ci_config] ssh_host ssh_port

# Execution mode: baseline, bisect, jenkins-full
rr[mode]="${rr[mode]-baseline}"

# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[ci_project]="${rr[ci_project]-tcwg_bmk}"
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
rr[update_baseline]="${rr[update_baseline]-update}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"

# {toolchain_name}-{toolchain_ver}-{target}-{bmk}-{cflags}
IFS=- read -a ci_config <<EOF
${rr[ci_config]}
EOF
rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
rr[target]=${rr[target]-${ci_config[2]}}
cflags=${cflags-${ci_config[4]}}

gcc_override_configure=""
case "${rr[target]}:$cflags" in
    "arm:Os"*|"arm:Oz"*)
	gcc_override_configure="--set gcc_override_configure=--with-mode=thumb"
	cflags="${cflags}_mthumb"
	;;
    "arm:"*)
	gcc_override_configure="--set gcc_override_configure=--with-mode=arm"
	cflags="${cflags}_marm"
	;;
esac

case "${rr[toolchain]}" in
    llvm) rr[components]="binutils gcc glibc llvm" ;;
    gnu) rr[components]="binutils gcc glibc" ;;
    *) assert false ;;
esac

# Use baseline branches by default.
for c in ${rr[components]}; do
    rr[${c}_branch]=${rr[${c}_branch]-baseline}
    if [ x"${rr[${c}_branch]}" != x"baseline" ]; then
	obligatory_variables rr[${c}_url]
    fi
done

start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"

if $verbose2; then set -x; fi

trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT

# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "${rr[mode]}" in
    "baseline")
	default_finish_at="update_baseline"
	;;
    "bisect")
	case "$(print_single_updated_component)" in
	    binutils) default_start_at="build_abe-binutils" ;;
	    gcc) default_start_at="build_abe-stage1" ;;
	    glibc) default_start_at="build_abe-glibc" ;;
	    llvm) default_start_at="build_llvm-true" ;;
	    *) assert false ;;
	esac
	default_finish_at="check_regression"
	;;
    "jenkins-full") ;;
esac
if [ x"$start_at" = x"default" ]; then
    start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
    finish_at="$default_finish_at"
fi

run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"

benchmark ()
{
    (
    set -euf -o pipefail

    rm -rf $(pwd)/bin
    mkdir $(pwd)/bin

    local bmk_flags bmk_ldflags
    bmk_flags="$(echo -$cflags | sed -e "s/_/ -/g" -e "s/LTO/flto/g")"
    case "$cflags" in
	"Os"*|"Oz"*)
	    # Use static linking for size benchmarking to pickup library
	    # code size increases in the final executable.
	    bmk_flags="$bmk_flags -static"
	    run_profile="parallel9x4"
	    ;;
	*)
	    run_profile="parallel9x1"
	    ;;
    esac

    local bench_list bin cc gnu_target sysroot
    gnu_target=$(print_gnu_target ${rr[target]})
    sysroot="$(pwd)/abe/sysroots/$gnu_target"
    case "${rr[toolchain]}" in
	llvm)
	    bench_list="c_and_cxx"
	    bmk_flags="$bmk_flags --target=$gnu_target --sysroot=$sysroot"
	    # When compiling with -Os/-Oz with LTO -- use "-O2 -flto".
	    # See https://projects.linaro.org/browse/LLVM-557 .
	    bmk_ldflags=$(echo "$bmk_flags" | sed -e "s/-O[sz] -flto/-O2 -flto/")
	    bin="$(pwd)/llvm-install/bin"
	    cc="$bin/"
	    ;;
	gnu)
	    bench_list="all"
	    bmk_ldflags="$bmk_flags"
	    bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin"
	    cc="$bin/$gnu_target-"
	    ;;
    esac
    sysroot="ssh://$ssh_host:$ssh_port:$sysroot"

    local hw results_id
    case "${rr[target]}" in
	arm*) hw=tk1 ;;
	*) hw=tx1 ;;
    esac
    local results_id="$hw/${rr[ci_project]}/${rr[ci_config]}-@build_num@"

    remote_exec "ci.linaro.org:2222::-l $USER@linaro.org" \
		build tcwg-benchmark -w \
		-p bench_list="$bench_list" \
		-p cflags="$bmk_flags" \
		-p ldflags="$bmk_ldflags" \
		-p testmode=benchmark \
		-p displaytag="${rr[mode]}-${rr[ci_config]}" \
		-p toolchain_url=ssh://$ssh_host:$ssh_port:$cc \
		-p toolchain_type=${rr[toolchain]} \
		-p sysroot="$sysroot" \
		-p results_id="$results_id" \
		-p run_profile="$run_profile" \
		-p target_list="$hw" \
		${scripts_branch+-p scripts_branch="$scripts_branch"} \
		${bmk_branch+-p bmk_branch="$bmk_branch"} \
	| tee $run_step_artifacts/benchmark-start.log

    local build_num
    build_num=$(cat $run_step_artifacts/benchmark-start.log \
		    | sed -e "s/.*#\([0-9]\+\).*/\1/")
    assert [ "$build_num" -gt "0" ]

    local build_status
    while true; do
	(remote_exec "ci.linaro.org:2222::-l $USER@linaro.org" \
		     console tcwg-benchmark -n 1 -f $build_num || true) \
	    | tee -a $run_step_artifacts/benchmark.log

	# Ssh connection to ci.linaro.org occasionally drops.  We need
	# to check whether benchmarking has finished, and, if not, continue
	# to watch its output.
	build_status=$(tail -n 1 $run_step_artifacts/benchmark.log)
	case "$build_status" in
	    "Finished: SUCCESS") break ;;
	    "Finished: "*) return 1 ;;
	esac

	# Sleep a little to avoid flooding ci.linaro.org on transient ssh
	# failures.
	sleep 5
    done

    echo "$results_id" | sed -e "s/@build_num@/$build_num/g" \
			     > "${rr[top_artifacts]}/results_id"
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
{
    (
    set -euf -o pipefail

    local ref_artifacts=$1
    local new_artifacts=$2

    if ! [ -f $ref_artifacts/results ]; then
	return 0
    fi

    local build_result_ref build_result_new
    build_result_ref=$(tail -n1 $ref_artifacts/results)
    build_result_new=$(tail -n1 $new_artifacts/results)

    if [ $build_result_new -lt $build_result_ref ]; then
	return 1
    elif [ $build_result_new -gt $build_result_ref ]; then
	return 0
    fi

    if ! [ -f $ref_artifacts/results_id ]; then
	return 0
    elif ! [ -f $new_artifacts/results_id ]; then
	return 1
    fi

    local results_ref results_new
    results_ref=$(cat $ref_artifacts/results_id)
    results_new=$(cat $new_artifacts/results_id)

    $scripts/tcwg-benchmark-results.sh --results_ref $results_ref --results_id $results_new \
				       --top_artifacts "${rr[top_artifacts]}" --verbose $verbose

    # Skip header line
    tail -n +2 ${rr[top_artifacts]}/results.csv \
	| while IFS=, read -r bmk symbol time size; do
	case "$cflags" in
	    "Os"*|"Oz"*) metric="$size" ;;
	    *) metric="$time" ;;
	esac

	# Skip case where we have no info ("n/a")
	if [ "$metric" != "n/a" ]; then
	    # We use 10% tolerance for symbols and rounding tolerance (~0.5%) for
	    # overall profile.
	    case "$symbol" in
		"["*) threshold=110 ;;
		*) threshold=100 ;;
	    esac
	    if [ "$metric" -gt "$threshold" ]; then
		echo "Regression in bench $bmk, symbol $symbol ($metric vs $threshold)"
		return 1
	    fi
	fi
    done
    )
}

run_step stop_on_fail -10 reset_artifacts
run_step stop_on_fail x prepare_abe
run_step skip_on_fail -9 build_abe binutils
run_step skip_on_fail -8 build_abe stage1 $gcc_override_configure
run_step skip_on_fail -7 build_abe linux
run_step skip_on_fail -6 build_abe glibc
run_step skip_on_fail -5 build_abe stage2 $gcc_override_configure
case "${rr[toolchain]}" in
    llvm) run_step skip_on_fail -1 build_llvm true ;;
    *) run_step skip_on_fail -1 true ;;
esac
run_step skip_on_fail 0 benchmark
run_step reset_on_fail x check_regression
run_step stop_on_fail x update_baseline
run_step stop_on_fail x push_baseline

trap "" EXIT