tcwg_kernel-build.sh: Move round-robin helpers to round-robin.sh

Change-Id: I6753fae4d94add85b5729ea1b121395ae5c679f0

1 files changed, 6 insertions, 406 deletions

diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index b349386a..8382e7de 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -92,347 +92,22 @@ fi
 
 run_step_init "$start_at" "$finish_at" "$top_artifacts" "$base_artifacts_opt" "$verbose"
 
-# Pretty name for PROJECT's version
-#rr[PROJECT_version]
-
-# PROJECT's git url
-#rr[PROJECT_url]
-
-# PROJECT's git revision parsable by git rev-parse.
-#rr[PROJECT_rev]
-
-# Baseline branch name for current configuration.  These branches are
-# present in all above git repos.
 rr[baseline_branch]="linaro-local/ci/tcwg_kernel/$toolchain-${rr[${toolchain}_version]}-$target-${rr[linux_version]}-$linux_config"
-
-# PROJECT that we are testing in this build.  Use ${rr[current_rev]} for
-# this project, and ${rr[baseline_branch]} for all other projects.
 rr[current_project]="$current_project"
-
-# Git revision of ${rr[current_project]} to test.
 rr[current_rev]="$current_rev"
-
-# Helper variable, only used in asserts.
 rr[git_read_only]="$git_read_only"
-
-# Linux config to build in build_linux.
-rr[linux_config]="$linux_config"
-
-# Run mode: bisect or non-bisect.  In bisect mode we do a couple things
-# slightly differently (e.g., don't touch repo in clone_repo() ).
 rr[mode]="$mode"
-
-# Ignore failures in check_regression, which forces update_baseline().
 rr[reset_baseline]="$reset_baseline"
-
-# Target architecture to build for: arm or aarch64.
 rr[target]="$target"
-
-# Toolchain to use: gnu or llvm.
-rr[toolchain]="$toolchain"
-
-# Top-level artifacts directory.
 rr[top_artifacts]="$top_artifacts"
 
-# Reset artifacts to an empty state.  ${rr[top_artifacts]}/results is the most
-# important artifact, since it records the metric of how successful the build
-# is.
-reset_artifacts ()
-{
-    (
-    set -euf -o pipefail
-
-    # Clean ${rr[top_artifacts]} but preserve
-    # - ${rr[top_artifacts]}/console.log and $run_step_artifacts/console.log, which
-    # are being written to by run_step().
-    # - ${rr[top_artifacts]}/jenkins/*, which is cleaned by tcwg_kernel.yaml.
-    find ${rr[top_artifacts]} -maxdepth 2 \
-	 ! -path ${rr[top_artifacts]} ! -path ${rr[top_artifacts]}/console.log \
-	 ! -path $run_step_artifacts ! -path $run_step_artifacts/console.log \
-	 ! -path ${rr[top_artifacts]}/jenkins ! -path "${rr[top_artifacts]}/jenkins/*" \
-	 -print0 | xargs -0 rm -rf
-
-    # Set success metric to worst possible (-10).
-    echo "Starting build" > ${rr[top_artifacts]}/results
-    echo "-10" >> ${rr[top_artifacts]}/results
-
-    # Clone base-artifacts so that run_step can rsync artifacts for skipped
-    # steps.
-    # TODO: base-artifacts repo is starting to get big, so we should
-    # fetch only the $baseline branch, instead of all branches.
-    clone_or_update_repo base-artifacts ${rr[baseline_branch]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git
-    )
-}
-
-# Clone repository for $project
-# $1: Repo / project name
-clone_repo ()
-{
-    (
-    set -euf -o pipefail
-    local project="$1"
-
-    if [ x"${rr[mode]}" = x"bisect" ]; then
-	# Don't touch anything in bisect mode.
-	return 0
-    fi
-
-    local branch
-    # Select the branch to build.
-    # ${rr[current_rev]} specifies branch for ${rr[current_project]}, and everything else
-    # uses baseline branch.
-    if [ x"$project" = x"${rr[current_project]}" ]; then
-	if [ x"${rr[current_rev]}" = x"default" ]; then
-	    branch=${rr[${current_project}_rev]}
-	elif [ x"${rr[current_rev]}" = x"baseline" ]; then
-	    branch="refs/remotes/baseline/${rr[baseline_branch]}"
-	else
-	    branch=${rr[current_rev]}
-	fi
-    else
-	branch="refs/remotes/baseline/${rr[baseline_branch]}"
-    fi
-
-    # Decide on whether to use read-only or read-write mode for
-    # refs/remotes/baseline.  We use read-only wherever possible to allow
-    # developers without ssh keys on Linaro git servers to reproduce builds
-    # in --mode "baseline".
-    local read_only="true"
-    if [ x"$project" = x"${rr[current_project]}" ]; then
-	read_only="${rr[git_read_only]}"
-    fi
-
-    # Clone origin remote
-    clone_or_update_repo_no_checkout $project ${rr[${project}_url]} auto > /dev/null
-    # Add baseline remote
-    git_init_linaro_local_remote $project baseline $read_only
-    # Checkout, now that we have both origin and baseline remotes ready.
-    clone_or_update_repo $project $branch ${rr[${project}_url]} > /dev/null
-
-    cd $project
-
-    git rev-parse HEAD > $run_step_artifacts/revision
-    if [ x"$project" = x"${rr[current_project]}" ]; then
-	local baseline_rev bad_rev
-
-	baseline_rev=$(git rev-parse refs/remotes/baseline/${rr[baseline_branch]})
-	bad_rev=$(git rev-parse HEAD)
-	# Prepare for failure.  If build fails we will bisect sha1 for
-	# ${rr[current_rev]} and sha1 for $baseline_rev.
-	cat > ${rr[top_artifacts]}/trigger-bisect-on-failure <<EOF
-current_project=${rr[current_project]}
-baseline_rev=$baseline_rev
-bad_rev=$bad_rev
-EOF
-	git rev-list --count HEAD ^$baseline_rev > ${rr[top_artifacts]}/distance-to-baseline
-    fi
-    )
-}
-
-# Prepare ABE tree for [partial] GNU builds
-prepare_abe ()
-{
-    (
-    set -euf -o pipefail
-
-    clone_or_update_repo abe tested https://git-us.linaro.org/toolchain/abe.git > /dev/null
-
-    cd abe
-
-    # Add ccache wrappers.
-    rm -rf $(pwd)/bin
-    mkdir $(pwd)/bin
-
-    cat > $(pwd)/bin/gcc <<EOF
-#!/bin/sh
-exec ccache /usr/bin/gcc "\$@"
-EOF
-    chmod +x $(pwd)/bin/gcc
-
-    cat > $(pwd)/bin/g++ <<EOF
-#!/bin/sh
-exec ccache /usr/bin/g++ "\$@"
-EOF
-    chmod +x $(pwd)/bin/g++
-
-    export PATH=$(pwd)/bin:$PATH
-
-    ./configure --with-git-reference-dir=/home/tcwg-buildslave/snapshots-ref
-    )
-}
-
-# Build ABE component
-# $1: Component -- ABE component to build.
-build_abe_1 ()
-{
-    (
-    set -euf -o pipefail
-
-    local component="$1"
-
-    local project stage
-    case "$component" in
-	stage1)
-	    project=gcc
-	    stage="--stage 1"
-	    ;;
-	stage2)
-	    project=gcc
-	    stage="--stage 2"
-	    ;;
-	*)
-	    project=$component
-	    stage=""
-	    ;;
-    esac
-
-    clone_repo $project
-
-    # Don't use ABE's repo clone functions and setup abe/snapshots/ directory
-    # to have the right entries.
-    local git_dir git_path
-    git_dir="$(basename ${rr[${project}_url]})"
-    git_path="abe/snapshots/$git_dir"
-    rm -rf $git_path $git_path~master
-    ln -s $(pwd)/$project $git_path
-    ln -s $(pwd)/$project $git_path~master
-
-    cd abe
-
-    local gnu_target
-    gnu_target=$(print_gnu_target ${rr[target]})
-
-    # Remove previous build directories and .stamp files.
-    # We rely on ccache for fast rebuilds.
-    set +f; rm -rf builds/x86_64-unknown-linux-gnu/$gnu_target/$git_dir~master*; set -f
-
-    export PATH=$(pwd)/bin:$PATH
-
-    if [ x"$component" != x"stage2" ]; then
-	# TODO: Fix install_sysroot logic in ABE.
-	# ABE tries to install sysroot even for partial builds, e.g.,
-	# with "--build binutils".  Workaround by patching ABE.
-	sed -i -e "s/do_install_sysroot/:/" lib/control.sh
-    else
-	git checkout -- lib/control.sh
-    fi
-
-    ccache -z
-    # Run "./abe.sh --build $project".
-    ./abe.sh \
-	--build $project \
-	--target $gnu_target \
-	--extraconfigdir config/master \
-	--disable update \
-	$project=${rr[${project}_url]}~master \
-	$stage
-    ccache -s
-    )
-}
-
-# Build binutils or GCC using ABE
-# $1: Component -- Binutils or stage1 GCC.
-build_abe ()
-{
-    (
-    set -euf -o pipefail
-
-    local component="$1"
-
-    # Attempt to build GCC only if binutils build (in previous step) was OK.
-    if [ x"$component" != x"binutils" -a ! -f $run_step_prev_artifacts/build-ok ]; then
-	return
-    fi
-
-    build_abe_1 $component &
-    local res=0; wait $! || res=$?
-
-    # Update artifacts on success.
-    if [ $res = 0 ]; then
-	echo "Finished build_abe $component" >> ${rr[top_artifacts]}/results
-	case "$component" in
-	    binutils) echo "-9" >> ${rr[top_artifacts]}/results ;;
-	    stage1) echo "-8" >> ${rr[top_artifacts]}/results ;;
-	    linux) echo "-7" >> ${rr[top_artifacts]}/results ;;
-	    glibc) echo "-6" >> ${rr[top_artifacts]}/results ;;
-	    stage2) echo "-5" >> ${rr[top_artifacts]}/results ;;
-	    *) assert false ;;
-	esac
-
-	touch $run_step_artifacts/build-ok
-    fi
-    )
-}
-
-# Build LLVM
-build_llvm_1 ()
-{
-    (
-    set -euf -o pipefail
-
-    clone_repo llvm
-
-    # Copy only components from the monorepo that are required for kernel build
-    rsync -a --del --exclude /tools/clang llvm/llvm/ llvm-src/
-    rsync -a --del llvm/clang/ llvm-src/tools/clang/
-
-    # Setup ccache and ninja wrappers.
-    rm -rf $(pwd)/bin
-    mkdir $(pwd)/bin
-
-    cat > $(pwd)/bin/cc <<EOF
-#!/bin/sh
-exec ccache /usr/bin/gcc "\$@"
-EOF
-    chmod +x $(pwd)/bin/cc
-
-    cat > $(pwd)/bin/c++ <<EOF
-#!/bin/sh
-exec ccache /usr/bin/g++ "\$@"
-EOF
-    chmod +x $(pwd)/bin/c++
-
-    if [ -f /usr/local/bin/ninja.bin ]; then
-	# Use ninja configuration from llvm buildbots to avoid running out of RAM.
-	cat > $(pwd)/bin/ninja <<EOF
-#!/bin/sh
-exec /usr/local/bin/ninja.bin -m 30 -M 50 -D 5000 "\$@"
-EOF
-	chmod +x $(pwd)/bin/ninja
-    fi
-
-    export PATH=$(pwd)/bin:$PATH
-
-    # Remove stale build directories.  We rely on ccache for fast rebuilds.
-    rm -rf llvm-build llvm-install
-    mkdir -p llvm-build
-    cd llvm-build
-
-    cmake -G Ninja  ../llvm-src -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install -DLLVM_TARGETS_TO_BUILD=$(print_llvm_target ${rr[target]})
-    ccache -z
-    ninja
-    ninja install
-    ccache -s
-    )
-}
-
-# Build LLVM
-build_llvm ()
-{
-    (
-    set -euf -o pipefail
+# Linux config to build in build_linux.
+rr[linux_config]="$linux_config"
 
-    build_llvm_1 &
-    local res=0; wait $! || res=$?
+# Toolchain to use: gnu or llvm.
+rr[toolchain]="$toolchain"
 
-    # Update artifacts on success.
-    if [ $res = 0 ]; then
-	echo "Finished build_llvm" >> ${rr[top_artifacts]}/results
-	echo "-1" >> ${rr[top_artifacts]}/results
-	touch $run_step_artifacts/build-ok
-    fi
-    )
-}
+. $scripts/round-robin.sh
 
 # Build Linux kernel
 build_linux ()
@@ -519,6 +194,7 @@ count_linux_objs ()
     )
 }
 
+
 # Exit with code 0 if no regression compared to base-artifacts/results.
 no_regression_p ()
 {
@@ -562,82 +238,6 @@ no_regression_p ()
     )
 }
 
-# Check if current build regressed compared to the baseline
-# (unless ${rr[reset_baseline]} is set).
-check_regression ()
-{
-    (
-    set -euf -o pipefail
-
-    if ! ${rr[reset_baseline]} && ! no_regression_p; then
-	# Fail.
-	false
-    fi
-    )
-}
-
-# Commit current result and artifacts to the baseline repository
-update_baseline ()
-{
-    (
-    set -euf -o pipefail
-
-    local amend=""
-    local prev_head=""
-
-    # We discard baseline entries for results worse or same than
-    # the current one, but keep entries for results that are better
-    # (so that we have a record of pending regressions).
-    while no_regression_p; do
-	if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then
-	    prev_head=$(git -C base-artifacts rev-parse HEAD)
-	    git -C base-artifacts reset --hard HEAD^
-	else
-	    # We got to the beginning of git history, so amend the current
-	    # commit.  The initial state of baseline is "empty" branch,
-	    # which we treat as worst possible in no_regression_p().
-	    amend="--amend"
-	    prev_head=""
-	    break
-	fi
-    done
-
-    # For every regression we want to keep artifacts for the first-bad
-    # build, so discard one less than needed.
-    if [ x"$prev_head" != x"" ]; then
-	git -C base-artifacts reset --hard $prev_head
-    fi
-
-    # Rsync current artifacts.
-    # !!! From this point on logs and other artifacts won't be included
-    # in base-artifacts.git repo.
-    rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/
-
-    local build_rev
-    build_rev=$(git -C ${rr[current_project]} rev-parse HEAD)
-
-    cd base-artifacts
-    git add .
-    git commit $amend -m "${rr[current_project]}-$build_rev: $(tail -n1 ${rr[top_artifacts]}/results)
-
-$(cat ${rr[top_artifacts]}/results)"
-    )
-}
-
-# Push to ${rr[current_project]} baseline branch and to base-artifacts repo.
-push_baseline ()
-{
-    (
-    set -euf -o pipefail
-
-    assert ! ${rr[git_read_only]}
-
-    git_init_linaro_local_remote base-artifacts baseline false
-    git_push base-artifacts baseline ${rr[baseline_branch]}
-    git_push ${rr[current_project]} baseline ${rr[baseline_branch]}
-    )
-}
-
 run_step reset_artifacts
 case "${rr[toolchain]}" in
     gnu)