path: root/tcwg_kernel-build.sh

#!/bin/bash

set -ef -o pipefail

scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh

convert_args_to_variables "$@"

# Execution mode: baseline, jenkins-full, bisect, continue.
mode="${mode-baseline}"

# Toolchain to use: gnu or llvm
obligatory_variables rr toolchain

case "$mode" in
    "jenkins-full")
	obligatory_variables current_project current_rev
	;;
esac

# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
current_project="${current_project-none}"
current_rev="${current_rev-default}"

linux_config="${linux_config-defconfig}"
reset_baseline="${reset_baseline-false}"
start_at="${start_at-default}"
finish_at="${finish_at-default}"
target="${target-native}"
top_artifacts="${top_artifacts-$(pwd)/artifacts}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"

set -u

if $verbose2; then set -x; fi

trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $top_artifacts/failures" EXIT

if [ x"$target" = x"native" ]; then
    target=$(uname -m)
fi

# We are not going to push any results for a one-off build.
if [ x"$current_project" = x"none" ]; then
    git_read_only=true
else
    git_read_only=false
fi

# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "$mode" in
    "baseline")
	default_finish_at="update_baseline"
	git_read_only=true
	;;
    "bisect")
	case "$toolchain:$current_project" in
	    gnu:binutils) default_start_at="build_abe-binutils" ;;
	    gnu:gcc) default_start_at="build_abe-gcc" ;;
	    gnu:linux) default_start_at="count_linux_objs" ;;
	    llvm:llvm) default_start_at="build_llvm" ;;
	    llvm:linux) default_start_at="count_linux_objs" ;;
	esac
	default_finish_at="check_regression"
	git_read_only=true
	;;
    "continue")
	# Developer mode.
	default_start_at="prepare_abe"
	git_read_only=true
	;;
    "jenkins-full") ;;
esac

if [ x"$start_at" = x"default" ]; then
    start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
    finish_at="$default_finish_at"
fi

if [ x"$mode" != x"continue" ]; then
    base_artifacts_opt="base-artifacts"
else
    base_artifacts_opt=""
fi

run_step_init "$start_at" "$finish_at" "$top_artifacts" "$base_artifacts_opt" "$verbose"

# Pretty name for PROJECT's version
#rr[PROJECT_version]

# PROJECT's git url
#rr[PROJECT_url]

# PROJECT's git revision parsable by git rev-parse.
#rr[PROJECT_rev]

# Baseline branch name for current configuration.  These branches are
# present in all above git repos.
rr[baseline_branch]="linaro-local/ci/tcwg_kernel/$toolchain-${rr[${toolchain}_version]}-$target-${rr[linux_version]}-$linux_config"

# PROJECT that we are testing in this build.  Use ${rr[current_rev]} for
# this project, and ${rr[baseline_branch]} for all other projects.
rr[current_project]="$current_project"

# Git revision of ${rr[current_project]} to test.
rr[current_rev]="$current_rev"

# Helper variable, only used in asserts.
rr[git_read_only]="$git_read_only"

# Linux config to build in build_linux.
rr[linux_config]="$linux_config"

# Run mode: bisect or non-bisect.  In bisect mode we do a couple things
# slightly differently (e.g., don't touch repo in clone_repo() ).
rr[mode]="$mode"

# Ignore failures in check_regression, which forces update_baseline().
rr[reset_baseline]="$reset_baseline"

# Target architecture to build for: arm or aarch64.
rr[target]="$target"

# Toolchain to use: gnu or llvm.
rr[toolchain]="$toolchain"

# Top-level artifacts directory.
rr[top_artifacts]="$top_artifacts"

# Reset artifacts to an empty state.  ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
reset_artifacts ()
{
    (
    set -euf -o pipefail

    # Clean ${rr[top_artifacts]} but preserve
    # - ${rr[top_artifacts]}/console.log and $run_step_artifacts/console.log, which
    # are being written to by run_step().
    # - ${rr[top_artifacts]}/jenkins/*, which is cleaned by tcwg_kernel.yaml.
    find ${rr[top_artifacts]} -maxdepth 2 \
	 ! -path ${rr[top_artifacts]} ! -path ${rr[top_artifacts]}/console.log \
	 ! -path $run_step_artifacts ! -path $run_step_artifacts/console.log \
	 ! -path ${rr[top_artifacts]}/jenkins ! -path "${rr[top_artifacts]}/jenkins/*" \
	 -print0 | xargs -0 rm -rf

    # Set success metric to worst possible (-10).
    echo "Starting build" > ${rr[top_artifacts]}/results
    echo "-10" >> ${rr[top_artifacts]}/results

    # Clone base-artifacts so that run_step can rsync artifacts for skipped
    # steps.
    # TODO: base-artifacts repo is starting to get big, so we should
    # fetch only the $baseline branch, instead of all branches.
    clone_or_update_repo base-artifacts ${rr[baseline_branch]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git
    )
}

# Clone repository for $project
# $1: Repo / project name
clone_repo ()
{
    (
    set -euf -o pipefail
    local project="$1"

    if [ x"${rr[mode]}" = x"bisect" ]; then
	# Don't touch anything in bisect mode.
	return 0
    fi

    local branch
    # Select the branch to build.
    # ${rr[current_rev]} specifies branch for ${rr[current_project]}, and everything else
    # uses baseline branch.
    if [ x"$project" = x"${rr[current_project]}" ]; then
	if [ x"${rr[current_rev]}" = x"default" ]; then
	    branch=${rr[${current_project}_rev]}
	elif [ x"${rr[current_rev]}" = x"baseline" ]; then
	    branch="refs/remotes/baseline/${rr[baseline_branch]}"
	else
	    branch=${rr[current_rev]}
	fi
    else
	branch="refs/remotes/baseline/${rr[baseline_branch]}"
    fi

    # Decide on whether to use read-only or read-write mode for
    # refs/remotes/baseline.  We use read-only wherever possible to allow
    # developers without ssh keys on Linaro git servers to reproduce builds
    # in --mode "baseline".
    local read_only="true"
    if [ x"$project" = x"${rr[current_project]}" ]; then
	read_only="${rr[git_read_only]}"
    fi

    # Clone origin remote
    clone_or_update_repo_no_checkout $project ${rr[${project}_url]} auto > /dev/null
    # Add baseline remote
    git_init_linaro_local_remote $project baseline $read_only
    # Checkout, now that we have both origin and baseline remotes ready.
    clone_or_update_repo $project $branch ${rr[${project}_url]} > /dev/null

    cd $project

    git rev-parse HEAD > $run_step_artifacts/revision
    if [ x"$project" = x"${rr[current_project]}" ]; then
	local baseline_rev bad_rev

	baseline_rev=$(git rev-parse refs/remotes/baseline/${rr[baseline_branch]})
	bad_rev=$(git rev-parse HEAD)
	# Prepare for failure.  If build fails we will bisect sha1 for
	# ${rr[current_rev]} and sha1 for $baseline_rev.
	cat > ${rr[top_artifacts]}/trigger-bisect-on-failure <<EOF
current_project=${rr[current_project]}
baseline_rev=$baseline_rev
bad_rev=$bad_rev
EOF
	git rev-list --count HEAD ^$baseline_rev > ${rr[top_artifacts]}/distance-to-baseline
    fi
    )
}

# Prepare ABE tree for [partial] GNU builds
prepare_abe ()
{
    (
    set -euf -o pipefail

    clone_or_update_repo abe tested https://git-us.linaro.org/toolchain/abe.git > /dev/null

    cd abe

    # Add ccache wrappers.
    rm -rf $(pwd)/bin
    mkdir $(pwd)/bin

    cat > $(pwd)/bin/gcc <<EOF
#!/bin/sh
exec ccache /usr/bin/gcc "\$@"
EOF
    chmod +x $(pwd)/bin/gcc

    cat > $(pwd)/bin/g++ <<EOF
#!/bin/sh
exec ccache /usr/bin/g++ "\$@"
EOF
    chmod +x $(pwd)/bin/g++

    export PATH=$(pwd)/bin:$PATH

    ./configure --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref
    )
}

# Build ABE component
# $1: Component -- ABE component to build.
build_abe_1 ()
{
    (
    set -euf -o pipefail

    local component="$1"

    local project stage
    case "$component" in
	stage1)
	    project=gcc
	    stage="--stage 1"
	    ;;
	stage2)
	    project=gcc
	    stage="--stage 2"
	    ;;
	*)
	    project=$component
	    stage=""
	    ;;
    esac

    clone_repo $project

    # Don't use ABE's repo clone functions and setup abe/snapshots/ directory
    # to have the right entries.
    local git_dir git_path
    git_dir="$(basename ${rr[${project}_url]})"
    git_path="abe/snapshots/$git_dir"
    rm -rf $git_path $git_path~master
    ln -s $(pwd)/$project $git_path
    ln -s $(pwd)/$project $git_path~master

    cd abe

    local gnu_target
    gnu_target=$(print_gnu_target ${rr[target]})

    # Remove previous build directories and .stamp files.
    # We rely on ccache for fast rebuilds.
    set +f; rm -rf builds/x86_64-unknown-linux-gnu/$gnu_target/$git_dir~master*; set -f

    export PATH=$(pwd)/bin:$PATH

    if [ x"$component" != x"stage2" ]; then
	# TODO: Fix install_sysroot logic in ABE.
	# ABE tries to install sysroot even for partial builds, e.g.,
	# with "--build binutils".  Workaround by patching ABE.
	sed -i -e "s/do_install_sysroot/:/" lib/control.sh
    else
	git checkout -- lib/control.sh
    fi

    ccache -z
    # Run "./abe.sh --build $project".
    ./abe.sh \
	--build $project \
	--target $gnu_target \
	--extraconfigdir config/master \
	--disable update \
	$project=${rr[${project}_url]}~master \
	$stage
    ccache -s
    )
}

# Build binutils or GCC using ABE
# $1: Component -- Binutils or stage1 GCC.
build_abe ()
{
    (
    set -euf -o pipefail

    local component="$1"

    # Attempt to build GCC only if binutils build (in previous step) was OK.
    if [ x"$component" != x"binutils" -a ! -f $run_step_prev_artifacts/build-ok ]; then
	return
    fi

    build_abe_1 $component &
    local res=0; wait $! || res=$?

    # Update artifacts on success.
    if [ $res = 0 ]; then
	echo "Finished build_abe $component" >> ${rr[top_artifacts]}/results
	case "$component" in
	    binutils) echo "-9" >> ${rr[top_artifacts]}/results ;;
	    stage1) echo "-8" >> ${rr[top_artifacts]}/results ;;
	    linux) echo "-7" >> ${rr[top_artifacts]}/results ;;
	    glibc) echo "-6" >> ${rr[top_artifacts]}/results ;;
	    stage2) echo "-5" >> ${rr[top_artifacts]}/results ;;
	    *) assert false ;;
	esac

	touch $run_step_artifacts/build-ok
    fi
    )
}

# Build LLVM
build_llvm_1 ()
{
    (
    set -euf -o pipefail

    clone_repo llvm

    # Copy only components from the monorepo that are required for kernel build
    rsync -a --del --exclude /tools/clang llvm/llvm/ llvm-src/
    rsync -a --del llvm/clang/ llvm-src/tools/clang/

    # Setup ccache and ninja wrappers.
    rm -rf $(pwd)/bin
    mkdir $(pwd)/bin

    cat > $(pwd)/bin/cc <<EOF
#!/bin/sh
exec ccache /usr/bin/gcc "\$@"
EOF
    chmod +x $(pwd)/bin/cc

    cat > $(pwd)/bin/c++ <<EOF
#!/bin/sh
exec ccache /usr/bin/g++ "\$@"
EOF
    chmod +x $(pwd)/bin/c++

    if [ -f /usr/local/bin/ninja.bin ]; then
	# Use ninja configuration from llvm buildbots to avoid running out of RAM.
	cat > $(pwd)/bin/ninja <<EOF
#!/bin/sh
exec /usr/local/bin/ninja.bin -m 30 -M 50 -D 5000 "\$@"
EOF
	chmod +x $(pwd)/bin/ninja
    fi

    export PATH=$(pwd)/bin:$PATH

    # Remove stale build directories.  We rely on ccache for fast rebuilds.
    rm -rf llvm-build llvm-install
    mkdir -p llvm-build
    cd llvm-build

    cmake -G Ninja  ../llvm-src -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install -DLLVM_TARGETS_TO_BUILD=$(print_llvm_target ${rr[target]})
    ccache -z
    ninja
    ninja install
    ccache -s
    )
}

# Build LLVM
build_llvm ()
{
    (
    set -euf -o pipefail

    build_llvm_1 &
    local res=0; wait $! || res=$?

    # Update artifacts on success.
    if [ $res = 0 ]; then
	echo "Finished build_llvm" >> ${rr[top_artifacts]}/results
	echo "-1" >> ${rr[top_artifacts]}/results
	touch $run_step_artifacts/build-ok
    fi
    )
}

# Build Linux kernel
build_linux ()
{
    (
    set -euf -o pipefail

    clone_repo linux

    rm -rf $(pwd)/bin
    mkdir $(pwd)/bin

    local bin cc

    case "${rr[toolchain]}" in
	llvm)
	    bin="$(pwd)/llvm-install/bin"
	    cc="$bin/clang"
	    ;;
	gnu)
	    bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin"
	    cc="$bin/$(print_gnu_target ${rr[target]})-gcc"
	    ;;
    esac

    # Use binutils, etc from $bin
    export PATH="$bin:$PATH"

    # Use ccache only when bisecting linux.  Otherwise the compiler is
    # new in every build and we would only clobber ccache volume.
    local ccache=""
    if [ x"${rr[mode]}" = x"bisect" -a x"${rr[current_project]}" = x"linux" ]; then
	ccache="ccache"
    fi
    cat > $(pwd)/bin/${rr[target]}-cc <<EOF
#!/bin/sh
exec $ccache $cc "\$@"
EOF
    chmod +x $(pwd)/bin/${rr[target]}-cc

    # Define make variables.
    local opts="CC=$(pwd)/bin/${rr[target]}-cc"
    if [ x"${rr[target]}" != x"$(uname -m)" ]; then
	opts="$opts ARCH=$(print_kernel_target ${rr[target]})"
	opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-"
	opts="$opts HOSTCC=gcc"
    fi

    cd linux

    make $opts distclean

    make $opts ${rr[linux_config]}

    ccache -z
    make $opts -j$(nproc --all) -s -k &
    local res=0 && wait $! || res=$?
    ccache -s
    return $res
    )
}

# Count number successfully built .o files in linux (and build linux)
count_linux_objs ()
{
    (
    set -euf -o pipefail

    # Run only if compiler was successfully built in the previous step.
    if [ -f $run_step_prev_artifacts/build-ok ]; then
	build_linux &
	local res=0; wait $! || res=$?

	# Number of .o files created is the main success metric.
	echo "linux_n_obj:" >> ${rr[top_artifacts]}/results
	if [ $res != 0 ]; then
	    local linux_n_obj
	    linux_n_obj=$(find linux -name "*.o" | wc -l)
	    echo "$linux_n_obj" >> ${rr[top_artifacts]}/results
	else
	    echo "all" >> ${rr[top_artifacts]}/results
	fi
    fi
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
{
    (
    set -euf -o pipefail

    local linux_n_obj
    linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results)

    if [ x"$linux_n_obj" = x"all" ]; then
	# Ideal result, no need to compare to baseline.
	return 0
    fi

    if ! [ "$linux_n_obj" -ge "-10" ]; then
	# Something is very wrong with result (e.g., it's not a number).
	return 1
    fi

    # Assume worst for non-existent baseline.
    local base_linux_n_obj="-10"

    if [ -f base-artifacts/results ]; then
	base_linux_n_obj=$(tail -n1 base-artifacts/results)
	if [ x"$base_linux_n_obj" = x"all" ]; then
	    # Current build is not ideal, so make sure baseline "all" is
	    # better than the current result.
	    base_linux_n_obj=$(($linux_n_obj+1))
	elif ! [ "$base_linux_n_obj" -ge "-10" ]; then
	    # Something is very wrong with baseline result
	    # (e.g., it's not a number).
	    return 0
	fi
    fi

    if [ $linux_n_obj -ge $base_linux_n_obj ]; then
	return 0
    else
	return 1
    fi
    )
}

# Check if current build regressed compared to the baseline
# (unless ${rr[reset_baseline]} is set).
check_regression ()
{
    (
    set -euf -o pipefail

    if ! ${rr[reset_baseline]} && ! no_regression_p; then
	# Fail.
	false
    fi
    )
}

# Commit current result and artifacts to the baseline repository
update_baseline ()
{
    (
    set -euf -o pipefail

    local amend=""
    local prev_head=""

    # We discard baseline entries for results worse or same than
    # the current one, but keep entries for results that are better
    # (so that we have a record of pending regressions).
    while no_regression_p; do
	if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then
	    prev_head=$(git -C base-artifacts rev-parse HEAD)
	    git -C base-artifacts reset --hard HEAD^
	else
	    # We got to the beginning of git history, so amend the current
	    # commit.  The initial state of baseline is "empty" branch,
	    # which we treat as worst possible in no_regression_p().
	    amend="--amend"
	    prev_head=""
	    break
	fi
    done

    # For every regression we want to keep artifacts for the first-bad
    # build, so discard one less than needed.
    if [ x"$prev_head" != x"" ]; then
	git -C base-artifacts reset --hard $prev_head
    fi

    # Rsync current artifacts.
    # !!! From this point on logs and other artifacts won't be included
    # in base-artifacts.git repo.
    rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/

    local build_rev
    build_rev=$(git -C ${rr[current_project]} rev-parse HEAD)

    cd base-artifacts
    git add .
    git commit $amend -m "${rr[current_project]}-$build_rev: $(tail -n1 ${rr[top_artifacts]}/results)

$(cat ${rr[top_artifacts]}/results)"
    )
}

# Push to ${rr[current_project]} baseline branch and to base-artifacts repo.
push_baseline ()
{
    (
    set -euf -o pipefail

    assert ! ${rr[git_read_only]}

    git_init_linaro_local_remote base-artifacts baseline false
    git_push base-artifacts baseline ${rr[baseline_branch]}
    git_push ${rr[current_project]} baseline ${rr[baseline_branch]}
    )
}

run_step reset_artifacts
case "${rr[toolchain]}" in
    gnu)
	run_step prepare_abe
	run_step build_abe binutils
	run_step build_abe stage1
	;;
    llvm)
	run_step build_llvm
	;;
esac
run_step count_linux_objs
run_step check_regression
run_step update_baseline
run_step push_baseline

trap "" EXIT