path: root/tcwg_gnu-build.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh
# shellcheck source=round-robin.sh
. $scripts/round-robin.sh

convert_args_to_variables "$@"

obligatory_variables rr[ci_project] rr[ci_config]
declare -A rr

# Execution mode: build or bisect,
rr[mode]="${rr[mode]-build}"

# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
rr[update_baseline]="${rr[update_baseline]-ignore}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"

# Resolve top_artifacts to absolute dir because some of the subsequent
# processes work below pwd and would write their artifacts in a wrong
# location
rr[top_artifacts]=$(abs_path "${rr[top_artifacts]}")

# {toolchain_ver}-{target}[-{type_of_test}]
IFS=- read -a ci_config <<EOF
${rr[ci_config]}
EOF
# Toolchain version -- master or release
toolchain_ver=${toolchain_ver-${ci_config[0]}}
ci_target=${ci_target-${ci_config[1]}}
# optional type_of_test contains the type of action to perform in this test
# campaign: bootstrap, bootstrap_lto, ...
type_of_test=${type_of_test-${ci_config[2]-}}

case "${rr[ci_project]}" in
    tcwg_gcc_*|tcwg_bootstrap_*)
	rr[target]="${rr[target]-native}"
	rr[components]="${rr[components]-gcc}"
	;;
    tcwg_binutils_*)
	rr[target]="${rr[target]-native}"
	rr[components]="${rr[components]-binutils}"
	;;
    tcwg_glibc_*)
	rr[target]="${rr[target]-native}"
	rr[components]="${rr[components]-glibc}"
	;;
    tcwg_gdb_*)
	rr[target]="${rr[target]-native}"
	rr[components]="${rr[components]-gdb}"
	;;
    tcwg_gnu_native_*)
	rr[target]="${rr[target]-native}"
	rr[components]="${rr[components]-binutils gcc linux glibc gdb}"
	;;
    tcwg_gnu_cross_*)
	rr[target]="${rr[target]-$ci_target}"
	rr[components]="${rr[components]-binutils gcc linux glibc gdb qemu}"
	;;
    tcwg_gnu_embed_*)
	rr[target]="${rr[target]-$ci_target}"
	rr[components]="${rr[components]-binutils gcc newlib gdb qemu}"
	;;
    tcwg_gnu_woa_*)
	rr[target]="${rr[target]-$ci_target}"
	rr[components]="${rr[components]-binutils}"
	;;
    *) assert_with_msg "Unknown ci_project: ${rr[ci_project]}" false ;;
esac

case "${rr[ci_project]}" in
    tcwg_gnu_native_fast_check_gdb)
	# Tests that run quickly and have stable results. This list is taken from
	# Sourceware's builder:
	# https://sourceware.org/git/?p=builder.git;a=blob;f=builder/master.cfg;h=612b3177f0e5#l2407
	testsuites=(gdb.base/break-always.exp
		    gdb.base/break-caller-line.exp
		    gdb.base/break-entry.exp
		    gdb.base/break.exp
		    gdb.base/break-fun-addr.exp
		    gdb.base/break-idempotent.exp
		    gdb.base/break-include.exp
		    gdb.base/break-inline.exp
		    gdb.base/break-main-file-remove-fail.exp
		    gdb.base/break-on-linker-gcd-function.exp
		    gdb.base/breakpoint-in-ro-region.exp
		    gdb.base/breakpoint-shadow.exp
		    gdb.base/break-probes.exp
		    gdb.gdb/unittest.exp
		   )
	;;

    tcwg_gnu_native_fast_check_gcc)
	# Use a small subset of GCC testsuite until we have made the
	# results stable.
	testsuites=(compile.exp
		    execute.exp
		   )
	;;
esac

runtestflags=()
if test_array testsuites; then
    # shellcheck disable=SC2154
    for i in "${testsuites[@]}"; do
	runtestflags+=(--set "runtestflags=$i")
    done
fi

# Use baseline branches by default.
for c in ${rr[components]}; do
    rr[${c}_git]=${rr[${c}_git]-baseline}
done

start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"

if $verbose2; then set -x; fi

trap print_traceback EXIT

# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "${rr[mode]}" in
    "bisect")
	case "$(print_single_updated_component):${rr[ci_project]}" in
	    binutils:*) default_start_at="build_abe-binutils" ;;
	    gcc:tcwg_gnu*_cross_*) default_start_at="build_abe-stage1" ;;
	    gcc:tcwg_gnu_native_*) default_start_at="build_abe-gcc" ;;
	    gcc:tcwg_gcc_*) default_start_at="build_abe-gcc" ;;
	    gcc:tcwg_bootstrap_*) default_start_at="build_abe-${type_of_test#check_}" ;;
	    gcc:tcwg_gnu_woa_*) default_start_at="build_abe-stage1" ;;
	    glibc:tcwg_glibc_*) default_start_at="build_abe-glibc" ;;
	    linux:*|glibc:*|newlib:*) default_start_at="clean_sysroot" ;;
	    gdb:*) default_start_at="build_abe-gdb" ;;
	    qemu:*) default_start_at="build_abe-qemu" ;;
	    *) assert_with_msg "Trying to bisecting unknown component: $(print_single_updated_component)" false ;;
	esac
	;;
esac
if [ x"$start_at" = x"default" ]; then
    start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
    finish_at="$default_finish_at"
fi

run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"

# Initialize sysroot directory in ABE's build/ with system headers
# so that [native] glibc can find headers to build against.
init_abe_sysroot ()
{
    (
    set -euf -o pipefail

    local abe_sysroot gnu_target
    abe_sysroot=$(print_abe_sysroot)
    gnu_target=$(print_gnu_target "$ci_target")
    rm -rf "$abe_sysroot/usr/include/"
    mkdir -p "$abe_sysroot/usr/include/"
    rsync -a "/usr/include/" "$abe_sysroot/usr/include/"
    # Debian/Ubuntu have arch headers under target-specific directory.
    rsync -a "/usr/include/$gnu_target/" "$abe_sysroot/usr/include/"
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
{
    (
    set -euf -o pipefail

    no_build_regression_p "$@"

    local ref_artifacts=$1
    local new_artifacts=$2

    local sumfiles_base=$ref_artifacts/sumfiles
    local sumfiles_new=$new_artifacts/sumfiles

    if ! [ -d $sumfiles_base ]; then
	return 0
    elif ! [ -d $sumfiles_new ]; then
	return 1
    fi

    local res

    if [ ${#runtestflags[@]} = 0 ] \
	   || [ "${rr[ci_project]}" = tcwg_gnu_native_fast_check_gdb ] \
	   || [ "${rr[ci_project]}" = tcwg_gnu_native_fast_check_gcc ]; then
	# Compare results using compare_tests for information purposes.
	# It works well only when we compare complete testsuites.
	# shellcheck disable=SC2154
	gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
					  $sumfiles_base $sumfiles_new \
	    | tee $run_step_artifacts/results.compare2 &
	res=0 && wait $! || res=$?
    fi

    local validate_failures="gcc-compare-results/contrib/testsuite-management/validate_failures.py"
    # Use xfail list generated by "build_abe check_FOO".
    local xfails="$run_step_top_artifacts/sumfiles/xfails.xfail"

    "$validate_failures" --manifest="$xfails" \
			 --build_dir=$sumfiles_new  --verbosity=2 \
	| tee $run_step_artifacts/results.compare &
    res=0 && wait $! || res=$?

    # copy to mail dir for reporting
    cp $run_step_artifacts/results.compare ${rr[top_artifacts]}/notify/

    if [ $res != 0 ]; then
	local reg_lines
	reg_lines=$(cat $run_step_artifacts/results.compare | wc -l)
	reg_lines=$(($reg_lines-100))
	cat $run_step_artifacts/results.compare | sed -e "s/^/# /" \
	    | (head -n100; cat >/dev/null) \
		  > $run_step_artifacts/results.regressions
	if [ $reg_lines -gt 0 ]; then
	    echo "# ... and $reg_lines more entries" \
		 >> $run_step_artifacts/results.regressions
	fi
	# Add a short marker to record the status (used by Jenkins build-name)
	echo "# regressed" >> $run_step_artifacts/results.regressions

	# copy to mail dir for reporting
	cp $run_step_artifacts/results.regressions ${rr[top_artifacts]}/notify/

	if [ $res = 2 ]; then
	    # Result comparison found regressions (exit code 2)
	    #
	    # Exit code 1 means that the script has failed to process
	    # .sum files.  This likely indicates malformed or very unusual
	    # results.
	    local res1
	    "$validate_failures" --manifest="$xfails" \
		--build_dir=$sumfiles_new --verbosity=1 \
		> $run_step_artifacts/fails.sum &
	    res1=0 && wait $! || res1=$?
	    assert_with_msg "Result comparison should have failed" \
			    [ $res1 = 2 ]

	    printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
	    local exp
	    while read exp; do
		printf "++testsuites %s " $exp >> $run_step_artifacts/extra-bisect-params
	    done < <(cat $run_step_artifacts/fails.sum \
			 | awk '/^Running .* \.\.\./ { print $2 }')
	    printf "\n" >> $run_step_artifacts/extra-bisect-params
	fi
    fi

    return $res
    )
}

# Implement rr[breakup_changed_components] hook.
tcwg_gnu_breakup_changed_components ()
{
    (
    set -euf -o pipefail

    local cc=""
    case "${rr[ci_project]}" in
	*_check_*)
	    # Changes to "foo" of check_foo projects tend to cause the most
	    # regressions.
	    # Breakup changed components into "foo" and the rest of components
	    # to reduce the number of builds.
	    cc=$(echo "${rr[ci_project]}" | sed -e "s/.*_check_\(.*\)/\1/")
	    ;;
    esac

    if print_changed_components "\n" | grep "^$cc\$" >/dev/null; then
	echo "$cc"
	print_changed_components "\n" | grep -v "^$cc\$" | tr '\n' ' ' \
	    | sed -e "s/ \$//g"
	echo
    else
	print_changed_components "\n"
    fi
    )
}
rr[breakup_changed_components]=tcwg_gnu_breakup_changed_components

run_step stop_on_fail -10 reset_artifacts
run_step stop_on_fail x prepare_abe
case "${rr[ci_project]}" in
    tcwg_gnu*_cross_build)
	run_step skip_on_fail 0 true
	run_step skip_on_fail 1 build_abe binutils
	run_step skip_on_fail 2 build_abe stage1
	run_step skip_on_fail x clean_sysroot
	case "${rr[components]}" in
	    *glibc*)
		run_step skip_on_fail 3 build_abe linux
		run_step skip_on_fail 4 build_abe glibc
		;;
	    *newlib*)
		run_step skip_on_fail 4 build_abe newlib
		;;
	esac
	run_step skip_on_fail 5 build_abe stage2
	run_step skip_on_fail 6 build_abe gdb
	run_step skip_on_fail 7 build_abe qemu
	;;
    tcwg_gnu*_cross_check_*)
	run_step skip_on_fail -8 build_abe binutils
	run_step skip_on_fail -7 build_abe stage1
	run_step skip_on_fail x clean_sysroot
	case "${rr[components]}" in
	    *glibc*)
		run_step skip_on_fail -6 build_abe linux
		run_step skip_on_fail -5 build_abe glibc
		;;
	    *newlib*)
		run_step skip_on_fail -5 build_abe newlib
		;;
	esac
	run_step skip_on_fail -4 build_abe stage2
	run_step skip_on_fail -3 build_abe gdb
	run_step skip_on_fail -2 build_abe qemu
	run_step skip_on_fail -1 build_abe dejagnu
	run_step skip_on_fail 0 build_abe "${rr[ci_project]#*_cross_}" -- "${runtestflags[@]}"
	;;
    tcwg_gnu_native_build)
	run_step skip_on_fail 0 true
	run_step skip_on_fail 1 build_abe binutils
	run_step skip_on_fail 2 build_abe gcc
	run_step skip_on_fail x clean_sysroot
	run_step skip_on_fail 4 build_abe linux
	run_step skip_on_fail 5 build_abe glibc
	run_step skip_on_fail 6 build_abe gdb
	;;
    tcwg_gnu_native_check_*|tcwg_gnu_native_fast_check_*)
	component="${rr[ci_project]#tcwg_gnu_native_}"
	component="${component#fast_}"

	run_step skip_on_fail -8 build_abe binutils
	run_step skip_on_fail -7 build_abe gcc
	run_step skip_on_fail x clean_sysroot
	run_step skip_on_fail -5 build_abe linux
	run_step skip_on_fail -4 build_abe glibc
	run_step skip_on_fail -3 build_abe gdb
	run_step skip_on_fail -1 build_abe dejagnu
	run_step skip_on_fail 0 build_abe "$component" -- "${runtestflags[@]}"
	;;
    tcwg_bootstrap_build)
	run_step skip_on_fail 0 true
	run_step skip_on_fail 1 build_abe ${type_of_test}
	;;
    tcwg_bootstrap_check)
	run_step skip_on_fail -2 build_abe ${type_of_test#check_}
	run_step skip_on_fail -1 build_abe dejagnu
	run_step skip_on_fail 0 build_abe ${type_of_test} -- "${runtestflags[@]}"
	;;
    tcwg_binutils_build|tcwg_gcc_build|tcwg_gdb_build)
	run_step skip_on_fail 0 true
	run_step skip_on_fail 1 build_abe "${rr[components]}"
	;;
    tcwg_binutils_check|tcwg_gcc_check|tcwg_gdb_check)
	run_step skip_on_fail -2 build_abe "${rr[components]}"
	run_step skip_on_fail -1 build_abe dejagnu
	run_step skip_on_fail 0 build_abe "check_${rr[components]}" \
		 -- "${runtestflags[@]}"
	;;
    tcwg_glibc_build)
	run_step skip_on_fail 0 init_abe_sysroot
	# ABE tries to copy gcc runtime libraries on glibc install, which
	# fails when we don't build gcc.  Workaround by not installing glibc.
	run_step skip_on_fail 1 build_abe glibc -- --disable install
	;;
    tcwg_glibc_check)
	run_step skip_on_fail -3 init_abe_sysroot
	run_step skip_on_fail -2 build_abe glibc -- --disable install
	run_step skip_on_fail -1 build_abe dejagnu
	run_step skip_on_fail 0 build_abe check_glibc \
		 --check_patch linaro-local/dejagnu-sum \
		 -- "${runtestflags[@]}" --disable install
	;;
    tcwg_gnu_woa_build)
	run_step skip_on_fail 0 true
	run_step skip_on_fail 1 build_abe binutils
	#run_step skip_on_fail 2 build_abe stage1
	#run_step skip_on_fail x clean_sysroot
	#run_step skip_on_fail 3 build_abe newlib
	#run_step skip_on_fail 4 build_abe stage2
	#run_step skip_on_fail 5 build_abe gdb
	;;
    tcwg_gnu_woa_check_*)
	run_step skip_on_fail -8 build_abe binutils
	#run_step skip_on_fail -7 build_abe stage1
	#run_step skip_on_fail x clean_sysroot
	#run_step skip_on_fail -6 build_abe newlib
	#run_step skip_on_fail -5 build_abe stage2
	#run_step skip_on_fail -4 build_abe gdb
	run_step skip_on_fail -1 build_abe dejagnu
	run_step skip_on_fail 0 build_abe "${rr[ci_project]#tcwg_gnu_woa_}" -- "${runtestflags[@]}"
	;;
esac
run_step reset_on_fail x check_regression
run_step stop_on_fail x update_baseline

trap "" EXIT