path: root/tcwg_aosp-build.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh
# shellcheck source=round-robin.sh
. $scripts/round-robin.sh

convert_args_to_variables "$@"

obligatory_variables rr[ci_project] rr[ci_config]
declare -A rr

# tcwg_aosp-code_size-{aosp_modules}
IFS=- read -a ci_project <<EOF
${rr[ci_project]}
EOF
aosp_modules=("${aosp_modules[@]-${ci_project[2]}}")

# {aosp_target}-{aosp_ver}
IFS=- read -a ci_config <<EOF
${rr[ci_config]}
EOF
aosp_target="${aosp_target-${ci_config[0]}}"

# Execution mode: build or bisect
rr[mode]="${rr[mode]-build}"

# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
rr[update_baseline]="${rr[update_baseline]-ignore}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"

rr[components]="aosp_superproject llvm toolchain_superproject"

# Use baseline branches by default.
for c in ${rr[components]}; do
    rr[${c}_git]=${rr[${c}_git]-baseline}
done

start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"

if $verbose2; then set -x; fi

trap print_traceback EXIT

# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "${rr[mode]}" in
    "bisect")
	case "$(print_single_updated_component)" in
	    toolchain_superproject) default_start_at="build_aosp_toolchain" ;;
	    llvm) default_start_at="build_shadow_llvm" ;;
	    aosp_superproject) default_start_at="build_aosp" ;;
	    *) assert false ;;
	esac
	;;
esac
if [ x"$start_at" = x"default" ]; then
    start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
    finish_at="$default_finish_at"
fi

run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"

# Clone AOSP repo
clone_aosp ()
{
    (
    set -euf -o pipefail

    local dir="$1"
    local superproject_rev="$2"
    local supermanifest="$3"

    mkdir -p "$dir"
    cd "$dir"

    # Remove locks from previous runs
    find .repo -type f -name "shallow.lock" -delete || true
    # Clean AOSP checkout
    .repo/repo/repo forall -c "git reset --hard; git clean -fdx" \
		    >/dev/null &
    if ! wait $!; then
	# Wipe AOSP checkout on clean errors
	cd ..
	rm -rf "$dir"
	mkdir "$dir"
	cd "$dir"
    fi

    # Resync on superproject change
    if ! [ -f repo_synced ] \
	    || [ x"$(cat repo_synced)" != x"$superproject_rev" ]; then
	rm -f repo_synced

	local manifest_url manifest_branch manifest_rev
	# Add a newline after supermanifest file to avoid "read" exiting with
	# "1" due to EOF.
	read manifest_url manifest_branch manifest_rev \
	      < <(cat "$supermanifest"; echo)

	manifest_url="https://android.googlesource.com/$manifest_url"

	rm -rf ./.repo/repo
	repo init --partial-clone --clone-filter=blob:limit=10M \
	     --use-superproject -u "$manifest_url" -b "$manifest_branch"

	# Use manifest specified in superproject.  We then use --nmu option
	# to "repo sync" to avoid manifest update.
	git -C .repo/manifests fetch "$manifest_url" "$manifest_rev"
	git -C .repo/manifests checkout FETCH_HEAD

	# FIXME:
	# Repo doesn't [yet] support fetching custom revisions of
	# superproject.  We workaround that by hacking command line
	# for "git fetch branch:branch" to "git fetch SHA1:branch".
	# This hack is effective only when we have $manifest_branch
	# specified in "repo init", so don't remove "-b" option in "repo init".
	sed -i -e "s#\[self._branch + \":\" + self._branch\]#\['$superproject_rev' + \":\" + self._branch\]#" ./.repo/repo/git_superproject.py

	./.repo/repo/repo sync --nmu -j"$(nproc --all)" -q &
	if ! wait $!; then
	    ./.repo/repo/repo sync --nmu -j1 --fail-fast
	fi

	# FIXME:
	# Verify that "repo sync" fetched the desired revision of superproject.
	local super_git super_rev
	super_git=$(find ./.repo/exp-superproject/ -name "*-superproject.git")
	super_rev=$(git -C "$super_git" rev-parse "$manifest_branch")
	assert_with_msg "Could not hack superproject repo" \
			[ "$super_rev" = "$superproject_rev" ]

	echo "$superproject_rev" > repo_synced
    fi
    )
}

# Build AOSP's LLVM
build_aosp_toolchain ()
{
    (
    set -euf -o pipefail

    clone_repo toolchain_superproject

    clone_aosp llvm-toolchain "$(get_current_git toolchain_superproject_rev)" \
	       "$(pwd)/toolchain_superproject/.supermanifest"

    cd llvm-toolchain

    assert_with_msg "Missing repo_synced file" [ -f repo_synced ]

    # Check if sources were updated and rebuild AOSP toolchain on re-syncs.
    if [ -f out/stage2/cmake_invocation.sh ] \
       && [ out/stage2/cmake_invocation.sh -ot repo_synced ]; then
	# Re-generate cmake recipe
	rm out/stage2/cmake_invocation.sh
    fi

    # Reproduce AOSP toolchain to get the cmake recipe
    if ! [ -f out/stage2/cmake_invocation.sh ]; then
	rm -rf out

	local tf_loc
	tf_loc="$(pip show tensorflow | grep Location | cut -d' ' -f2)" || true
	[ -d "$tf_loc" ] && export TENSORFLOW_INSTALL="$tf_loc/tensorflow"

	python toolchain/llvm_android/build.py --no-build lldb,windows \
	       --skip-runtimes --skip-tests --skip-package --mlgo
    fi
    )
}

# Build LLVM
build_shadow_llvm ()
{
    (
    set -euf -o pipefail

    clone_repo llvm

    cd llvm-toolchain

    local cc cxx ninja
    cc=$(cat out/stage2/cmake_invocation.sh \
	     | grep -e " -DCMAKE_C_COMPILER=" \
	     | sed -e "s/.* -DCMAKE_C_COMPILER=\([^ ]*\).*/\1/")
    cxx=$(cat out/stage2/cmake_invocation.sh \
	      | grep -e " -DCMAKE_CXX_COMPILER=" \
	      | sed -e "s/.* -DCMAKE_CXX_COMPILER=\([^ ]*\).*/\1/")
    ninja=$(cat out/stage2/cmake_invocation.sh \
	      | grep -e " -DCMAKE_MAKE_PROGRAM=" \
	      | sed -e "s/.* -DCMAKE_MAKE_PROGRAM=\([^ ]*\).*/\1/")
    cd ..

    local workspace
    workspace=$(pwd)

    # ${workspace:?}/bin is to avoid shellcheck warning that below never
    # expands to "rm -rf /bin"
    rm -rf "${workspace:?}/bin"
    mkdir "$workspace/bin"

    cat > "$workspace/bin/cc" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache $cc "\$@"
EOF
    chmod +x "$workspace/bin/cc"

    cat > "$workspace/bin/c++" <<EOF
#!/bin/sh
CCACHE_BASEDIR=$workspace exec ccache $cxx "\$@"
EOF
    chmod +x "$workspace/bin/c++"

    rm -rf llvm-install

    cd llvm-toolchain/out

    cp stage2/cmake_invocation.sh ./


    sed -i \
	-e "s#/llvm-toolchain/out/llvm-project/llvm #/llvm/llvm #" \
	-e "s#/llvm-toolchain/out/stage2-install #/llvm-install #" \
	-e "s# -DCMAKE_C_COMPILER=[^ ]* # -DCMAKE_C_COMPILER=$workspace/bin/cc #" \
	-e "s# -DCMAKE_CXX_COMPILER=[^ ]* # -DCMAKE_CXX_COMPILER=$workspace/bin/c++ #" \
	cmake_invocation.sh

    ccache -z

    rm -rf stage3
    mkdir stage3
    cd stage3

    set +e
    source ../cmake_invocation.sh
    local res=$?
    set -e
    if [ $res != 0 ] || ! $ninja; then
	# Attempt to workaround failures in past LLVM versions.
	cd ..

	# Workaround failures to find BOLT, which seems to be unused anyway.
	sed -i -e "s/bolt;//" cmake_invocation.sh

	# Workaround failure to link x86_64's libc++.so against non-PIC
	# libc++abi.a.  Add -fPIC to x86_64 runtime's CFLAGS.
	sed -i -e "s/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_C_FLAGS=/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_C_FLAGS=-fPIC /g" \
	    cmake_invocation.sh
	sed -i -e "s/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_CXX_FLAGS=/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_CXX_FLAGS=-fPIC /g" \
	    cmake_invocation.sh

	rm -rf stage3
	mkdir stage3
	cd stage3
	source ../cmake_invocation.sh
	$ninja
    fi
    $ninja install

    ccache -s
    )
}

# Build AOSP
build_aosp ()
{
    (
    set -euf -o pipefail

    # Clone AOSP superproject
    clone_repo aosp_superproject

    # Clone AOSP
    clone_aosp aosp "$(get_current_git aosp_superproject_rev)" \
	       "$(pwd)/aosp_superproject/.supermanifest"

    # Remove old binaries
    (set +f; rm -rf shadow*)
    rm -rf aosp/out

    # Install build wrappers
    local prebuilts_clang_bin
    while read -r prebuilts_clang_bin; do
	$scripts/wrappers/install-wrappers.sh \
	    $prebuilts_clang_bin llvm-install/bin aosp shadow
    done < <(find aosp/prebuilts/clang/host/linux-x86/ \
		  -maxdepth 2 -name "bin" -type d)

    $scripts/wrappers/install-wrappers.sh \
	aosp/build/soong/scripts llvm-install/bin aosp shadow strip.sh

    # Build AOSP
    (
	cd aosp
	set -e +ufx +o pipefail
        local release=trunk_staging
	source build/envsetup.sh
	echo "RUN: lunch aosp_${aosp_target}-${release}-user"
	lunch "aosp_${aosp_target}-${release}-user"
	echo "RUN: m ${aosp_modules[*]}"
	if [ x"${aosp_modules[*]}" = x"aosp" ]; then
	    m
	else
	    m "${aosp_modules[@]}"
	fi
    )
    )
}

# Check shadow build
process_shadow_data ()
{
    (
    set -euf -o pipefail

    local success_code="$2"

    if ! [ -f shadow.errors ]; then
	success_code=$(($success_code + 1))
	echo "# shadow build has no errors" >> $run_step_top_artifacts/results
	echo "$success_code" >> $run_step_top_artifacts/results
    else
	echo "# shadow build has errors" >> $run_step_top_artifacts/results
	cp shadow.errors $run_step_top_artifacts/
	exit 1
    fi

    if [ -f shadow.size ]; then
	cp shadow.size $run_step_top_artifacts/size.csv
	success_code=$(($success_code + 1))
	echo "# shadow.size present" >> $run_step_top_artifacts/results
	echo "$success_code" >> $run_step_top_artifacts/results
    fi
    )
}

# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
{
    (
    set -euf -o pipefail

    no_build_regression_p "$@"

    if ! [ -f base-artifacts/size.csv ]; then
	return 0
    elif ! [ -f $run_step_top_artifacts/size.csv ]; then
	return 1
    fi

    # Generate results-vs-prev/results.csv
    mkdir -p $run_step_top_artifacts/results-vs-prev
    $scripts/../bmk-scripts/csvs2table.py \
	--relative "$(pwd)/base-artifacts/size.csv" \
	$run_step_top_artifacts/size.csv \
	> $run_step_top_artifacts/results-vs-prev/results.csv

    # Read result lines from <(tail -n +2 ...) below.
    # "-n +2" is to skip the header line.
    local -a arr
    local rel_text text1 text2 metric total1=0 total2=0
    while IFS=, read -a arr; do
	binary=${arr[0]}
	rel_text=${arr[2]}
	text1=${arr[3]}
	text2=${arr[4]}

	if [ x"$rel_text" = x"n/a" ]; then
	    case "$text1":"$text2" in
		"-1:-1") ;;
		"-1":*)
		    echo "-1,$binary now builds successfully" \
			 >> $run_step_artifacts/binary.improvements
		    ;;
		*:"-1")
		    echo "1,$binary now fails to build" \
			 >> $run_step_artifacts/binary.regressions
		    ;;
	    esac
	    continue
	fi

	metric=$(($rel_text - 100))

	if [ $metric -lt 0 ]; then
	    echo "$metric,$binary reduced in size by ${metric}% from $text1 to $text2" \
		 >> $run_step_artifacts/binary.improvements
	elif [ $metric -gt 0 ]; then
	    echo "$metric,$binary increased in size by ${metric}% from $text1 to $text2" \
		 >> $run_step_artifacts/binary.regressions
	fi

	total1=$(($total1 + $text1))
	total2=$(($total2 + $text2))
    done < <(tail -n +2 $run_step_top_artifacts/results-vs-prev/results.csv)

    if [ $total1 != 0 ]; then
	metric=$(((100 * $total2) / $total1 - 100))
    else
	# Corner case when baseline results have no intersection with
	# the current results.  This happens when AOSP build fails and
	# size.csv only has the header.
	metric=0
    fi
    if [ $metric -lt 0 ]; then
	echo "$metric,AOSP reduced in size by ${metric}% from $total1 to $total2" \
	     >> $run_step_artifacts/aosp.improvements
    elif [ $metric -gt 0 ]; then
	echo "$metric,AOSP increased in size by ${metric}% from $total1 to $total2" \
	     >> $run_step_artifacts/aosp.regressions
    fi

    local primary_change="" regression=false
    if [ -f $run_step_artifacts/aosp.regressions ]; then
        primary_change=$run_step_artifacts/aosp.regressions
	regression=true
    elif [ -f $run_step_artifacts/binary.regressions ]; then
        primary_change=$run_step_artifacts/binary.regressions
	regression=true
    elif [ -f $run_step_artifacts/aosp.improvements ]; then
        primary_change=$run_step_artifacts/aosp.improvements
    elif [ -f $run_step_artifacts/binary.improvements ]; then
        primary_change=$run_step_artifacts/binary.improvements
    fi

    if [ x"$primary_change" != x"" ]; then
	sort -gr -o "$primary_change" "$primary_change"
	if $regression; then
	    head -n1 "$primary_change" \
		| sed -e "s/^/# /" > $run_step_artifacts/results.regressions
	fi
    fi

    case "${rr[ci_project]}" in
	"tcwg_aosp-build"*)
	    # We have passed no_build_regression_p() check above,
	    # so succeed.
	    return 0
	    ;;
    esac

    local -a changed_components
    IFS=" " read -r -a changed_components <<< "$(print_changed_components)"
    case " ${changed_components[*]} " in
	*" llvm "*) ;;
	*)
	    # Code-size builds without changed LLVM we declare as
	    # non-regressing.  Code-size changes are due to changes in AOSP,
	    # not in the compiler.
	    return 0
	    ;;
    esac

    if $regression; then
	return 1
    fi
    return 0
    )
}

rr[breakup_changed_components]="breakup_changed_components llvm"

case "${rr[ci_project]}" in
    "tcwg_aosp-build"*)
	run_step stop_on_fail 0 reset_artifacts
	run_step skip_on_fail 1 build_aosp_toolchain
	run_step skip_on_fail 2 build_shadow_llvm
	run_step skip_on_fail 3 build_aosp
	run_step skip_on_fail x process_shadow_data -- 3
	;;
    *)
	run_step stop_on_fail -10 reset_artifacts
	run_step skip_on_fail -3 build_aosp_toolchain
	run_step skip_on_fail -2 build_shadow_llvm
	run_step skip_on_fail -1 build_aosp
	run_step skip_on_fail x process_shadow_data -- -1
	;;
esac
run_step reset_on_fail x check_regression

trap "" EXIT