path: root/tcwg_kernel-build.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh
. $scripts/round-robin.sh

convert_args_to_variables "$@"

obligatory_variables rr[ci_config]

# Execution mode: baseline, bisect, jenkins-full
rr[mode]="${rr[mode]-baseline}"

# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[ci_project]="${rr[ci_project]-tcwg_kernel}"
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
rr[reset_baseline]="${rr[reset_baseline]-false}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"

# {toolchain_name}-{toolchain_ver}-{target}-{linux}-{linux_config}
IFS=- read -a ci_config <<EOF
${rr[ci_config]}
EOF
rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
rr[release]=${rr[release]-${ci_config[1]}}
rr[target]=${rr[target]-${ci_config[2]}}
rr[linux_config]=${rr[linux_config]-${ci_config[4]}}

case "${rr[toolchain]}" in
    llvm) rr[components]="llvm linux" ;;
    gnu) rr[components]="binutils gcc linux" ;;
    *) assert false ;;
esac

# Use baseline branches by default.
for c in ${rr[components]}; do
    rr[${c}_branch]=${rr[${c}_branch]-baseline}
    obligatory_variables rr[${c}_url]
done

start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"

if $verbose2; then set -x; fi

trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT

# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "${rr[mode]}" in
    "baseline")
	default_finish_at="update_baseline"
	;;
    "bisect")
	case "${rr[toolchain]}:$(print_single_updated_component)" in
	    gnu:binutils) default_start_at="build_abe-binutils" ;;
	    gnu:gcc) default_start_at="build_abe-stage1" ;;
	    gnu:linux) default_start_at="build_linux" ;;
	    llvm:llvm) default_start_at="build_llvm" ;;
	    llvm:linux) default_start_at="build_linux" ;;
	    *) assert false ;;
	esac
	default_finish_at="check_regression"
	;;
    "jenkins-full") ;;
esac
if [ x"$start_at" = x"default" ]; then
    start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
    finish_at="$default_finish_at"
fi

run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"

# Build Linux kernel
build_linux_1 ()
{
    (
    set -euf -o pipefail

    clone_repo linux

    rm -rf $(pwd)/bin
    mkdir $(pwd)/bin

    local bin cc ld_opt

    case "${rr[toolchain]}" in
	llvm)
	    bin="$(pwd)/llvm-install/bin"
	    cc="$bin/clang"
	    ;;
	gnu)
	    bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin"
	    cc="$bin/$(print_gnu_target ${rr[target]})-gcc"
	    ;;
    esac

    case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in
        llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;;
        llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;;
        *) ld_opt="" ;;
    esac

    # Use binutils, etc from $bin
    export PATH="$bin:$PATH"

    # Use ccache only when bisecting linux (or preparing to).
    # Otherwise the compiler is new in every build and we would
    # only clobber ccache volume.
    local ccache=""
    if [ x"${rr[mode]}" != x"jenkins-full" \
	  -a x"$(print_single_updated_component)" = x"linux" ]; then
	ccache="ccache"
    fi
    cat > $(pwd)/bin/${rr[target]}-cc <<EOF
#!/bin/sh
exec $ccache $cc "\$@"
EOF
    chmod +x $(pwd)/bin/${rr[target]}-cc

    # Define make variables.
    local opts="CC=$(pwd)/bin/${rr[target]}-cc $ld_opt SUBLEVEL=0 EXTRAVERSION=-bisect"
    if [ x"${rr[target]}" != x"$(uname -m)" ]; then
	opts="$opts ARCH=$(print_kernel_target ${rr[target]})"
	opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-"
	opts="$opts HOSTCC=gcc"
    fi

    cd linux

    make $opts distclean

    make $opts ${rr[linux_config]}
    sed -i -e 's:CONFIG_LOCALVERSION_AUTO=y:# CONFIG_LOCALVERSION_AUTO is not set:' .config
    set +f; rm -f localversion*; set -f
    make $opts oldconfig

    ccache -z
    KBUILD_BUILD_TIMESTAMP=0 make $opts -j$(nproc --all) -s -k &
    local res=0 && wait $! || res=$?
    ccache -s
    return $res
    )
}

# Build linux and count number successfully built .o files in linux
build_linux ()
{
    (
    set -euf -o pipefail

    build_linux_1 &
    local res=0 && wait $! || res=$?

    # Number of .o files created is the main success metric.
    local linux_n_obj
    linux_n_obj=$(find linux -name "*.o" | wc -l)
    echo "linux_n_obj:" >> ${rr[top_artifacts]}/results
    echo "$linux_n_obj" >> ${rr[top_artifacts]}/results

    if [ $res = 0 ]; then
	echo "linux build successful:" >> ${rr[top_artifacts]}/results
	echo "all" >> ${rr[top_artifacts]}/results
    fi

    return $res
    )
}

# Boot linux kernel
boot_linux ()
{
    (
    set -euf -o pipefail

    local image cpu
    case ${rr[target]} in
	aarch64)
	    image=linux/arch/arm64/boot/Image.gz
	    cpu="-cpu cortex-a53"
	    ;;
	arm)
	    image=linux/arch/arm/boot/zImage
	    cpu=""
	    ;;
	*) assert false ;;
    esac
    timeout 10s qemu-system-${rr[target]} \
	    -kernel $image -machine virt $cpu -m 512 \
	    -serial mon:stdio -display none \
	    -append "console=ttyAMA0 panic=-1" -no-reboot

    echo "linux boot successful:" >> ${rr[top_artifacts]}/results
    echo "boot" >> ${rr[top_artifacts]}/results
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
{
    (
    set -euf -o pipefail

    local linux_n_obj
    linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results)

    # Assume worst for non-existent baseline.
    local base_linux_n_obj="-10"
    if [ -f base-artifacts/results ]; then
	base_linux_n_obj=$(tail -n1 base-artifacts/results)
    fi

    case "$linux_n_obj:$base_linux_n_obj" in
	boot:*) return 0 ;;
	*:boot) return 1 ;;
	all:*) return 0 ;;
	*:all) return 1 ;;
	*)
	    if ! [ "$linux_n_obj" -ge "-10" ]; then
		# Something is very wrong with result (e.g., it's not a number).
		return 1
	    fi
	    if ! [ "$base_linux_n_obj" -ge "-10" ]; then
		# Something is very wrong with result (e.g., it's not a number).
		return 0
	    fi

	    if [ $linux_n_obj -ge $base_linux_n_obj ]; then
		return 0
	    else
		return 1
	    fi
	    ;;
    esac
    )
}

run_step stop_on_fail -10 reset_artifacts
case "${rr[toolchain]}" in
    gnu)
	run_step stop_on_fail x prepare_abe
	run_step skip_on_fail -9 build_abe binutils
	run_step skip_on_fail -8 build_abe stage1
	;;
    llvm)
	run_step skip_on_fail -1 build_llvm
	;;
esac
run_step skip_on_fail x build_linux
run_step skip_on_fail x boot_linux
run_step reset_on_fail x check_regression
run_step stop_on_fail x update_baseline
run_step stop_on_fail x push_baseline

trap "" EXIT