path: root/tcwg_bmk-build.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh
# shellcheck source=round-robin.sh
. $scripts/round-robin.sh

convert_args_to_variables "$@"

obligatory_variables rr[ci_project] rr[ci_config]
declare -A rr

# All bmk config about hw and benchs is implemented in this file
# shellcheck source=tcwg_bmk-config.sh
. $scripts/tcwg_bmk-config.sh

# Execution mode: build or bisect
rr[mode]="${rr[mode]-build}"

# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
rr[update_baseline]="${rr[update_baseline]-ignore}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"

# ${ci_project}--${ci_config} format is :
#  'tcwg_bmk-#{PROFILE_NAME}-#{BMK}--#{TOOLCHAIN}-#{TARGET}-{toolchain_ver}-{cflags}'
IFS=- read -a ci_pjt_cfg <<EOF
${rr[ci_project]}--${rr[ci_config]}
EOF

rr[toolchain]=${rr[toolchain]-${ci_pjt_cfg[4]}}
rr[target]=${rr[target]-${ci_pjt_cfg[5]}}

cflags="${cflags--${ci_pjt_cfg[7]}}"

gcc_mode=""
case "${rr[target]}:$cflags" in
    "arm:"*"mthumb"*) gcc_mode=thumb ;;
    "arm:"*"marm"*) gcc_mode=arm ;;
    "arm:-Os"*|"arm:-Oz"*)
	gcc_mode=thumb
	cflags="${cflags}_mthumb"
	;;
    "arm:"*)
	gcc_mode=arm
	cflags="${cflags}_marm"
	;;
    "arm_eabi:"*)
	cflags="${cflags}_mthumb"
	;;
esac

cflags="$(echo $cflags | sed -e "s/_/ -/g" -e "s/LTO/flto/g")"

case "${rr[ci_project]}" in
    *-*_size-*) rr[metric_id]="size" ;;
    *-*_speed-*) rr[metric_id]="sample" ;;
    *-*_vect-*) rr[metric_id]="num_vect_loops" ;;
    *-*_sve-*) rr[metric_id]="num_sve_loops" ;;
    *) assert_with_msg "Cannot determine metric from ${rr[ci_project]}" false ;;
esac

called_from_notify=${called_from_notify-false}

hw=$(tcwg_bmk_hw)
hw=${hw%_32} ; hw=${hw%_64}

# -----------------------------------------------------------------------

gcc_override_configure=()
# Set default ARM/Thumb mode for AArch32 compiler.  This ensures that libraries
# (Glibc, libgcc, libstdc++, libgfortran, etc.) are built in the "interesting"
# ISA.
case "$gcc_mode" in
    arm|thumb) gcc_override_configure+=("--set" "gcc_override_configure=--with-mode=$gcc_mode") ;;
esac
# Disable libsanitizer because it didn't build in April-June 2018.
# We need to test revisions during that time period for performance regressions.
gcc_override_configure+=("--set" "gcc_override_configure=--disable-libsanitizer")

# Build the right libs depending on the target. Hardcode the
# cpu/float-abit/mode to speed up toolchain builds: the alternative of
# building rmprofile multilibs takes a very long time. Setting the
# values here has the drawback that we have to build one toolchain per
# board type.
case "${rr[target]}" in
    "arm_eabi") gcc_override_configure+=("--set" "gcc_override_configure=--disable-multilib"
					 "--set" "gcc_override_configure=--with-mode=thumb"
					 "--set" "gcc_override_configure=--with-float=hard"
					) ;;
esac


rr[cpu]=$(tcwg_bmk_cpu)
if [ "${rr[cpu]}" != "" ]; then
    gcc_override_configure+=("--set" "gcc_override_configure=--with-cpu=${rr[cpu]}")
    cflags="$cflags -mcpu=${rr[cpu]}"
fi

case "${rr[toolchain]}" in
    llvm)
      rr[components]="llvm" ;;
    gnu)
      rr[components]="binutils gcc linux glibc" ;;
    gnu_eabi)
      rr[components]="binutils gcc newlib" ;;
    *) assert_with_msg "Unknown toolchain \"${rr[toolchain]}\"" false ;;
esac

# Use baseline branches by default.
for c in ${rr[components]}; do
    rr[${c}_git]=${rr[${c}_git]-baseline}
done

start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"

if $verbose2; then set -x; fi

trap print_traceback EXIT

# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "${rr[mode]}" in
    "bisect")
	single_updated_component="$(print_single_updated_component)"
	case $single_updated_component in
	    binutils) default_start_at="build_abe-binutils" ;;
	    gcc) default_start_at="build_abe-stage1" ;;
	    linux|glibc) default_start_at="clean_sysroot" ;;
	    llvm) default_start_at="build_bmk_llvm" ;;
	    newlib) default_start_at="build_abe-newlib" ;;
	    *) assert_with_msg \
		   "Invalid single updated component \"$single_updated_component\"" false
	       ;;
	esac
	;;
esac
if [ x"$start_at" = x"default" ]; then
    start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
    finish_at="$default_finish_at"
fi

case "${rr[ci_project]}/${rr[ci_config]}" in
    tcwg_bmk-code_speed-cpu2017rate/gnu-aarch64-master-O2|\
    tcwg_bmk-code_speed-cpu2017rate/gnu-aarch64-master-O3|\
    tcwg_bmk-code_speed-cpu2017rate/llvm-aarch64-master-O2|\
    tcwg_bmk-code_speed-cpu2017rate/llvm-aarch64-master-O3)
	rr[major]=3
	rr[minor]=0
	;;
    *)
	rr[major]=2
	rr[minor]=3
	;;
esac

run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"

build_bmk_llvm ()
{
    (
    set -euf -o pipefail

    local projects="clang;lld;openmp"
    case "${rr[target]}" in
	aarch64)
	    # Flang is not supported for AArch32
	    projects="$projects;flang"
	    ;;
    esac

    build_llvm "$projects" "" "${rr[metric_id]}"

    # Copy shared libraries to runtime sysroot dir
    mkdir -p llvm-install/libc
    rsync -a --del --include "*/" --include "*.so*" --exclude "*" \
	  --delete-excluded llvm-install/lib/ llvm-install/libc/lib/
    )
}

benchmark ()
{
    obligatory_variables ssh_host ssh_port

    (
    set -euf -o pipefail

    local bmk_flags="$2"

    sanity_check_pwd

    # shellcheck disable=SC2115
    rm -rf "$(pwd)"/bin
    mkdir "$(pwd)"/bin

    local reboot run_profile

    local hw_tag
    hw_tag=$(tcwg_bmk_hw)

    case "${rr[ci_project]}" in
        *_size*|*_vect*|*_sve*)
	    reboot=false
	    run_profile="parallel"
	    testmode="verify"
	    ;;
	*)
	    reboot=true
	    run_profile="serial"
	    testmode="benchmark"
	    ;;
    esac

    local bench_list bin cc sysroot toolchain
    case "${rr[toolchain]}" in
	llvm)
	    sysroot="$(pwd)/llvm-install/libc"
	    bin="$(pwd)/llvm-install/bin"
	    cc="$bin/"
	    toolchain="llvm"
	    ;;
	gnu|gnu_eabi)
	    local gnu_host gnu_target
	    gnu_host=$(print_gnu_target native)
	    gnu_target=$(print_gnu_target ${rr[target]})
	    sysroot="$(pwd)/abe/builds/destdir/$gnu_host/$gnu_target/libc"
	    bin="$(pwd)/abe/builds/destdir/$gnu_host/bin"
	    cc="$bin/$gnu_target-"
	    toolchain="gnu"
	    # Append -fdump-statistics-asmname to obtain compile time metrics.
	    bmk_flags="$bmk_flags -fdump-statistics-asmname -fdump-tree-vect-details"
	    ;;
    esac

    bench_list="$(tcwg_bmk_benchs)"

    # shellcheck disable=SC2154
    sysroot="ssh://$ssh_host:$ssh_port:$sysroot"

    local hw image_arch toolchain_proto

    toolchain_proto=ssh
    case "$hw_tag" in
	stm32)
	    hw=stm32; image_arch=amd64
	    # When running benchmarks on stm32, we prefer to rsync the
	    # toolchain to the board's host machine -- dev-02.tcwglab.
	    toolchain_proto=rsync
	    ;;
	*_32) hw=${hw_tag/_32}; image_arch=armhf ;;
	*_64) hw=${hw_tag/_64}; image_arch=arm64 ;;
	*) echo "ERROR: Unknown hw_tag $hw_tag"; exit 1 ;;
    esac

    # Create directory for tcwg-benchmark to upload results to.
    # Note that files inside $results_dir will be owned by tcwg-benchmark.
    local results_dir
    results_dir="$(mktemp -d)"
    chmod 0777 "$results_dir"

    # Trigger benchmarking job and capture its console output.
    # Ignore exit code of the trigger command to detect various failure
    # conditions from examining the console log.
    # shellcheck disable=SC2154
    remote_exec "ci.linaro.org:2222::-l $USER@linaro.org" \
		build tcwg-benchmark -f -v \
		-p bmk_hw=$hw \
		-p bench_list="$bench_list" \
		-p cflags="$bmk_flags" \
		-p ldflags="$bmk_flags" \
		-p testmode="$testmode" \
		-p displaytag="${rr[ci_project]}/${rr[ci_config]}-${rr[mode]}" \
		-p ignore_errors=true \
		-p toolchain_url=$toolchain_proto://$ssh_host:$ssh_port:$cc \
		-p toolchain_type=$toolchain \
		-p sysroot="$sysroot" \
		-p results_dest="$ssh_host:$ssh_port:$results_dir" \
		-p reboot="$reboot" \
		-p run_profile="$run_profile" \
		-p image_arch="$image_arch" \
		${scripts_branch+-p scripts_branch="$scripts_branch"} \
		${bmk_branch+-p bmk_branch="$bmk_branch"} \
	| tee $run_step_artifacts/benchmark-build.log || true

    local build_num
    build_num=$(head -n1 $run_step_artifacts/benchmark-build.log \
		    | sed -e "s/Started.*#\([0-9]\+\).*/\1/")
    assert_with_msg "Benchmark build number should not be 0!" \
		    [ "$build_num" -gt "0" ]

    cat > $run_step_artifacts/benchmark_job.txt << EOF
Url: https://ci.linaro.org/job/tcwg-benchmark/$build_num
Name: $(curl -s "https://ci.linaro.org/job/tcwg-benchmark/$build_num/api/json?tree=displayName" \
	   | jq -r ".displayName")
EOF

    local build_status
    local build_ret
    while true; do
	# Ssh connection to ci.linaro.org occasionally drops.  We need
	# to check whether benchmarking has finished, and, if not, continue
	# waiting.
	build_status=$(curl -s \
	   "https://ci.linaro.org/job/tcwg-benchmark/$build_num/api/json?tree=result" \
	   | jq -r ".result")
	case "$build_status" in
	    "null")
		# Continue waiting
		true
		;;
	    "SUCCESS")
	        build_ret=0
	        break
	        ;;
	    *)
		echo "# Benchmarking infra is offline:" \
		     >> ${rr[top_artifacts]}/results
		echo "-$EXTERNAL_FAIL" >> ${rr[top_artifacts]}/results
	        build_ret=1
	        break
	        ;;
	esac

	# Wait by following console output
	(ssh -p2222 -l $USER@linaro.org ci.linaro.org \
	     console tcwg-benchmark $build_num -f || true) \
	    | tee $run_step_artifacts/benchmark-wait.log
    done

    rm -rf "${rr[top_artifacts]}/annex"
    mkdir "${rr[top_artifacts]}/annex"
    ln -s "$results_dir" "${rr[top_artifacts]}/annex/bmk-data"

    return $build_ret
    )
}

# Exit with code 0 if no regression compared to base-artifacts/.
no_regression_p ()
{
    (
    set -euf -o pipefail

    # check score-based regression
    no_build_regression_p "$@"

    # At this stage, there's no score-based regression.
    # We are now checking metric-based regression.

    assert_with_msg "Benchmarking succeeded, but bmk-data is missing" \
		    [ -e $run_step_top_artifacts/annex/bmk-data ]

    # Make sure there is no stray results.regression file, which we use
    # as failure marker.
    assert ! [ -f $run_step_artifacts/results.regressions ]

    local compare_opts=""
    case "${rr[target]}:$cflags" in
	"arm_eabi":*) compare_opts="--has_perf_logs no" ;;
	*) compare_opts="" ;;
    esac

    if [ -f /usr/lib/linux-tools/install-armhf-perf-workaround.sh ]; then
	# FIXME:
	# In some cases perf report crashes when run from armhf container on
	# ARMv8 machine.
	# Install a workaround while we are investigating the cause.
	sudo /usr/lib/linux-tools/install-armhf-perf-workaround.sh
    fi

    local new_results="${rr[top_artifacts]}/annex/bmk-data"
    local ref_results="base-artifacts/annex/bmk-data"

    assert_with_msg "Benchmarking succeeded, but no annex/bmk-data results" \
		    [ -d "$new_results" ]

    if ! [ -d "$ref_results" ]; then
	# base-artifacts has no reference results.
	# This can happen on init build (update_baseline=init).
	# In such cases we compare results to themselves just as an exercise.
	ref_results="$new_results"
	assert_with_msg "No reference results" \
			[ "${rr[update_baseline]}" = "init" ]
    fi

    # Compare vs previous run
    mkdir -p ${rr[top_artifacts]}/results-vs-prev
    ln -s ../results-vs-prev $run_step_artifacts/results-vs-prev
    $scripts/tcwg-benchmark-results.sh \
        --results_ref "$ref_results" ++results "$new_results" \
        --top_artifacts "${rr[top_artifacts]}/results-vs-prev" \
	--verbose $verbose --hw_tag "$(tcwg_bmk_hw)" \
	$compare_opts \
        > ${rr[top_artifacts]}/results-vs-prev/tcwg-benchmark-results.log 2>&1 &

    local res
    res=0 && wait $! || res=$?
    if [ $res != 0 ]; then
	return $EXTERNAL_FAIL
    fi

    # Below call to output-bmk-results.py creates *.regression files.
    assert_with_msg "Found stale regression files" \
		    [ x"$(find $run_step_artifacts/ -name "*.regression" | wc -l)" = x"0" ]

    # Extract 5 most recent compare-results-vs-prev-internal.csv files from
    # base-artifacts and compute std deviation out of them
    local -a csvs_paths
    csvs_paths=("results-vs-prev/compare-results-internal.csv"
		  "$(basename $run_step_artifacts)/compare-results-vs-prev-internal.csv")

    local -a history_csvs
    local csv history_root=""
    while read csv; do
       if [ "$history_root" = "" ]; then
           history_root="$csv"
           continue
       fi

       history_csvs+=("$csv")
    done < <(get_git_history -0 base-artifacts "${csvs_paths[@]}")

    local csv tmpf
    local -a compare_results_list=()
    tmpf=$(mktemp)

    # FIXME:
    # To deal with some differences along base-artifacts recent history
    # - remove 'Failed for column' message from csv file
    # - skip emtpy csv files.
    for csv in "${history_csvs[@]}"; do
	grep -v 'Failed for column' "$csv" > "$tmpf" || true
	cp "$tmpf" "$csv"
	if [ -s "$csv" ]; then
	  compare_results_list+=("$csv")
	fi
    done

    if [ ${#compare_results_list[@]} != 0 ]; then
      $scripts/../bmk-scripts/compute-variability.py \
        --inputs "${compare_results_list[@]}" ${rr[top_artifacts]}/results-vs-prev/compare-results-internal.csv \
        --weights linear --method avg \
        --output ${rr[top_artifacts]}/results-vs-prev/bmk-specific-variability-avg.csv || true

      $scripts/../bmk-scripts/compute-variability.py \
        --inputs "${compare_results_list[@]}" ${rr[top_artifacts]}/results-vs-prev/compare-results-internal.csv \
	--weights 2-peaks-linear --method max \
        --output ${rr[top_artifacts]}/results-vs-prev/bmk-specific-variability-max.csv || true
    fi

    rm -rf "$history_root" "$tmpf"

    $scripts/../bmk-scripts/output-bmk-results.py \
	--compare_results ${rr[top_artifacts]}/results-vs-prev/compare-results-internal.csv \
	--variability_file ${rr[top_artifacts]}/results-vs-prev/bmk-specific-variability-avg.csv \
	--variability_file_data "avg" \
	--run_step_dir "$run_step_artifacts"/ \
	--metric "${rr[metric_id]}" --mode "${rr[mode]}" \
	--details quiet > $run_step_artifacts/output-bmk-results.log

    # copy inputs useful to build the mail / jira / .. to mail dir
    for resfile in $run_step_artifacts/{exe,symbol}.{regression,improvement}; do
        if [ -f $resfile ]; then
	   cp $resfile ${rr[top_artifacts]}/notify/
	fi
    done

    # return status rely on the presence of the results.regressions file
    if [ -f $run_step_artifacts/results.regressions ]; then
        assert_with_msg "Found a regression while comparing the build against itself" \
			[ "$ref_results" != "$new_results" ]
	return 1
    fi
    return 0
    )
}

# Compiler changes tend to cause the most regressions.
# Breakup updated components into compiler and the rest of components
# to reduce the number of builds.
case "${rr[toolchain]}" in
    llvm)
	rr[breakup_changed_components]="breakup_changed_components llvm"
	;;
    gnu|gnu_eabi)
	rr[breakup_changed_components]="breakup_changed_components gcc"
	;;
    *) assert false ;;
esac

run_step stop_on_fail -10 reset_artifacts
case "${rr[toolchain]}" in
    gnu*)
	run_step stop_on_fail x prepare_abe
	run_step skip_on_fail -9 build_abe binutils
	run_step skip_on_fail -8 build_abe stage1 -- \
		 "${gcc_override_configure[@]}"
	run_step skip_on_fail x clean_sysroot
	case "${rr[components]}" in
	    *glibc*)
		run_step skip_on_fail -7 build_abe linux
		run_step skip_on_fail -6 build_abe glibc
		;;
	    *newlib*)
		run_step skip_on_fail -6 build_abe newlib
		;;
	esac
	run_step skip_on_fail -5 build_abe stage2 -- \
		 "${gcc_override_configure[@]}"
	;;
    llvm)

	run_step skip_on_fail -3 build_bmk_llvm
	;;
esac
run_step skip_on_fail 1 benchmark -- "$cflags"
run_step reset_on_fail x check_regression

trap "" EXIT