diff options
Diffstat (limited to 'tcwg_aosp-build.sh')
| -rwxr-xr-x | tcwg_aosp-build.sh | 498 |
1 files changed, 498 insertions, 0 deletions
diff --git a/tcwg_aosp-build.sh b/tcwg_aosp-build.sh new file mode 100755 index 00000000..06556823 --- /dev/null +++ b/tcwg_aosp-build.sh @@ -0,0 +1,498 @@ +#!/bin/bash + +set -euf -o pipefail + +scripts=$(dirname $0) +# shellcheck source=jenkins-helpers.sh +. $scripts/jenkins-helpers.sh +# shellcheck source=round-robin.sh +. $scripts/round-robin.sh + +convert_args_to_variables "$@" + +obligatory_variables rr[ci_project] rr[ci_config] +declare -A rr + +# tcwg_aosp-code_size-{aosp_modules} +IFS=- read -a ci_project <<EOF +${rr[ci_project]} +EOF +aosp_modules=("${aosp_modules[@]-${ci_project[2]}}") + +# {aosp_target}-{aosp_ver} +IFS=- read -a ci_config <<EOF +${rr[ci_config]} +EOF +aosp_target="${aosp_target-${ci_config[0]}}" + +# Execution mode: build or bisect +rr[mode]="${rr[mode]-build}" + +# Set custom revision for one of the projects, and use baseline revisions +# for all other projects. +rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}" +rr[update_baseline]="${rr[update_baseline]-ignore}" +rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}" + +rr[components]="aosp_superproject llvm toolchain_superproject" + +# Use baseline branches by default. +for c in ${rr[components]}; do + rr[${c}_git]=${rr[${c}_git]-baseline} +done + +# Artifacts version +rr[major]=4 +rr[minor]=0 + +start_at="${start_at-default}" +finish_at="${finish_at-default}" +verbose="${verbose-true}" +verbose2="${verbose2-false}" + +if $verbose2; then set -x; fi + +trap print_traceback EXIT + +# Set start and finish steps for different modes. +default_start_at="" +default_finish_at="" +case "${rr[mode]}" in + "bisect") + case "$(print_single_updated_component)" in + toolchain_superproject) default_start_at="build_aosp_toolchain" ;; + llvm) default_start_at="build_shadow_llvm" ;; + aosp_superproject) default_start_at="build_aosp" ;; + *) assert false ;; + esac + ;; +esac +if [ x"$start_at" = x"default" ]; then + start_at="$default_start_at" +fi +if [ x"$finish_at" = x"default" ]; then + finish_at="$default_finish_at" +fi + +run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose" + +# Clone AOSP repo +clone_aosp () +{ + ( + set -euf -o pipefail + + local dir="$1" + local superproject_rev="$2" + local supermanifest="$3" + + mkdir -p "$dir" + cd "$dir" + + # Remove locks from previous runs + find .repo -type f -name "shallow.lock" -delete || true + # Clean AOSP checkout + .repo/repo/repo forall -c "git reset --hard; git clean -fdx" \ + >/dev/null & + if ! wait $!; then + # Wipe AOSP checkout on clean errors + cd .. + rm -rf "$dir" + mkdir "$dir" + cd "$dir" + fi + + # Resync on superproject change + if ! [ -f repo_synced ] \ + || [ x"$(cat repo_synced)" != x"$superproject_rev" ]; then + rm -f repo_synced + + local manifest_url manifest_branch manifest_rev + # Add a newline after supermanifest file to avoid "read" exiting with + # "1" due to EOF. + read manifest_url manifest_branch manifest_rev \ + < <(cat "$supermanifest"; echo) + + manifest_url="https://android.googlesource.com/$manifest_url" + + rm -rf ./.repo/repo + repo init --partial-clone --clone-filter=blob:limit=10M \ + --use-superproject -u "$manifest_url" -b "$manifest_branch" + + # Use manifest specified in superproject. We then use --nmu option + # to "repo sync" to avoid manifest update. + git -C .repo/manifests fetch "$manifest_url" "$manifest_rev" + git -C .repo/manifests checkout FETCH_HEAD + + # FIXME: + # Repo doesn't [yet] support fetching custom revisions of + # superproject. We workaround that by hacking command line + # for "git fetch branch:branch" to "git fetch SHA1:branch". + # This hack is effective only when we have $manifest_branch + # specified in "repo init", so don't remove "-b" option in "repo init". + sed -i -e "s#\[self._branch + \":\" + self._branch\]#\['$superproject_rev' + \":\" + self._branch\]#" ./.repo/repo/git_superproject.py + + ./.repo/repo/repo sync --nmu -j"$(nproc --all)" -q & + if ! wait $!; then + ./.repo/repo/repo sync --nmu -j1 --fail-fast + fi + + # FIXME: + # Verify that "repo sync" fetched the desired revision of superproject. + local super_git super_rev + super_git=$(find ./.repo/exp-superproject/ -name "*-superproject.git") + super_rev=$(git -C "$super_git" rev-parse "$manifest_branch") + assert_with_msg "Could not hack superproject repo" \ + [ "$super_rev" = "$superproject_rev" ] + + echo "$superproject_rev" > repo_synced + fi + ) +} + +# Build AOSP's LLVM +build_aosp_toolchain () +{ + ( + set -euf -o pipefail + + clone_repo toolchain_superproject + + clone_aosp llvm-toolchain "$(get_current_git toolchain_superproject_rev)" \ + "$(pwd)/toolchain_superproject/.supermanifest" + + cd llvm-toolchain + + assert_with_msg "Missing repo_synced file" [ -f repo_synced ] + + # Check if sources were updated and rebuild AOSP toolchain on re-syncs. + if [ -f out/stage2/cmake_invocation.sh ] \ + && [ out/stage2/cmake_invocation.sh -ot repo_synced ]; then + # Re-generate cmake recipe + rm out/stage2/cmake_invocation.sh + fi + + # Reproduce AOSP toolchain to get the cmake recipe + if ! [ -f out/stage2/cmake_invocation.sh ]; then + rm -rf out + + local tf_loc + tf_loc="$(pip show tensorflow | grep Location | cut -d' ' -f2)" || true + [ -d "$tf_loc" ] && export TENSORFLOW_INSTALL="$tf_loc/tensorflow" + + python toolchain/llvm_android/build.py --no-build lldb,windows \ + --skip-runtimes --skip-tests --skip-package --mlgo + fi + ) +} + +# Build LLVM +build_shadow_llvm () +{ + ( + set -euf -o pipefail + + clone_repo llvm + + cd llvm-toolchain + + local cc cxx ninja + cc=$(cat out/stage2/cmake_invocation.sh \ + | grep -e " -DCMAKE_C_COMPILER=" \ + | sed -e "s/.* -DCMAKE_C_COMPILER=\([^ ]*\).*/\1/") + cxx=$(cat out/stage2/cmake_invocation.sh \ + | grep -e " -DCMAKE_CXX_COMPILER=" \ + | sed -e "s/.* -DCMAKE_CXX_COMPILER=\([^ ]*\).*/\1/") + ninja=$(cat out/stage2/cmake_invocation.sh \ + | grep -e " -DCMAKE_MAKE_PROGRAM=" \ + | sed -e "s/.* -DCMAKE_MAKE_PROGRAM=\([^ ]*\).*/\1/") + cd .. + + local workspace + workspace=$(pwd) + + # ${workspace:?}/bin is to avoid shellcheck warning that below never + # expands to "rm -rf /bin" + rm -rf "${workspace:?}/bin" + mkdir "$workspace/bin" + + cat > "$workspace/bin/cc" <<EOF +#!/bin/sh +CCACHE_BASEDIR=$workspace exec ccache $cc "\$@" +EOF + chmod +x "$workspace/bin/cc" + + cat > "$workspace/bin/c++" <<EOF +#!/bin/sh +CCACHE_BASEDIR=$workspace exec ccache $cxx "\$@" +EOF + chmod +x "$workspace/bin/c++" + + rm -rf llvm-install + + cd llvm-toolchain/out + + cp stage2/cmake_invocation.sh ./ + + + sed -i \ + -e "s#/llvm-toolchain/out/llvm-project/llvm #/llvm/llvm #" \ + -e "s#/llvm-toolchain/out/stage2-install #/llvm-install #" \ + -e "s# -DCMAKE_C_COMPILER=[^ ]* # -DCMAKE_C_COMPILER=$workspace/bin/cc #" \ + -e "s# -DCMAKE_CXX_COMPILER=[^ ]* # -DCMAKE_CXX_COMPILER=$workspace/bin/c++ #" \ + cmake_invocation.sh + + ccache -z + + rm -rf stage3 + mkdir stage3 + cd stage3 + + set +e + source ../cmake_invocation.sh + local res=$? + set -e + if [ $res != 0 ] || ! $ninja; then + # Attempt to workaround failures in past LLVM versions. + cd .. + + # Workaround failures to find BOLT, which seems to be unused anyway. + sed -i -e "s/bolt;//" cmake_invocation.sh + + # Workaround failure to link x86_64's libc++.so against non-PIC + # libc++abi.a. Add -fPIC to x86_64 runtime's CFLAGS. + sed -i -e "s/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_C_FLAGS=/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_C_FLAGS=-fPIC /g" \ + cmake_invocation.sh + sed -i -e "s/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_CXX_FLAGS=/ '-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_CXX_FLAGS=-fPIC /g" \ + cmake_invocation.sh + + rm -rf stage3 + mkdir stage3 + cd stage3 + source ../cmake_invocation.sh + $ninja + fi + $ninja install + + ccache -s + ) +} + +# Build AOSP +build_aosp () +{ + ( + set -euf -o pipefail + + # Clone AOSP superproject + clone_repo aosp_superproject + + # Clone AOSP + clone_aosp aosp "$(get_current_git aosp_superproject_rev)" \ + "$(pwd)/aosp_superproject/.supermanifest" + + # Remove old binaries + (set +f; rm -rf shadow*) + rm -rf aosp/out + + # Install build wrappers + local prebuilts_clang_bin + while read -r prebuilts_clang_bin; do + $scripts/wrappers/install-wrappers.sh \ + $prebuilts_clang_bin llvm-install/bin aosp shadow + done < <(find aosp/prebuilts/clang/host/linux-x86/ \ + -maxdepth 2 -name "bin" -type d) + + $scripts/wrappers/install-wrappers.sh \ + aosp/build/soong/scripts llvm-install/bin aosp shadow strip.sh + + # Build AOSP + ( + cd aosp + set -e +ufx +o pipefail + local release=trunk_staging + source build/envsetup.sh + echo "RUN: lunch aosp_${aosp_target}-${release}-user" + lunch "aosp_${aosp_target}-${release}-user" + echo "RUN: m ${aosp_modules[*]}" + if [ x"${aosp_modules[*]}" = x"aosp" ]; then + m + else + m "${aosp_modules[@]}" + fi + ) + ) +} + +# Check shadow build +process_shadow_data () +{ + ( + set -euf -o pipefail + + local success_code="$2" + + if ! [ -f shadow.errors ]; then + success_code=$(($success_code + 1)) + echo "# shadow build has no errors" >> $run_step_top_artifacts/results + echo "$success_code" >> $run_step_top_artifacts/results + else + echo "# shadow build has errors" >> $run_step_top_artifacts/results + cp shadow.errors $run_step_top_artifacts/ + exit 1 + fi + + if [ -f shadow.size ]; then + cp shadow.size $run_step_top_artifacts/size.csv + success_code=$(($success_code + 1)) + echo "# shadow.size present" >> $run_step_top_artifacts/results + echo "$success_code" >> $run_step_top_artifacts/results + fi + ) +} + +# Exit with code 0 if no regression compared to base-artifacts/results. +no_regression_p () +{ + ( + set -euf -o pipefail + + no_build_regression_p "$@" + + if ! [ -f base-artifacts/size.csv ]; then + return 0 + elif ! [ -f $run_step_top_artifacts/size.csv ]; then + return 1 + fi + + # Generate results-vs-prev/results.csv + mkdir -p $run_step_top_artifacts/results-vs-prev + $scripts/../bmk-scripts/csvs2table.py \ + --relative "$(pwd)/base-artifacts/size.csv" \ + $run_step_top_artifacts/size.csv \ + > $run_step_top_artifacts/results-vs-prev/results.csv + + # Read result lines from <(tail -n +2 ...) below. + # "-n +2" is to skip the header line. + local -a arr + local rel_text text1 text2 metric total1=0 total2=0 + while IFS=, read -a arr; do + binary=${arr[0]} + rel_text=${arr[2]} + text1=${arr[3]} + text2=${arr[4]} + + if [ x"$rel_text" = x"n/a" ]; then + case "$text1":"$text2" in + "-1:-1") ;; + "-1":*) + echo "-1,$binary now builds successfully" \ + >> $run_step_artifacts/binary.improvements + ;; + *:"-1") + echo "1,$binary now fails to build" \ + >> $run_step_artifacts/binary.regressions + ;; + esac + continue + fi + + metric=$(($rel_text - 100)) + + if [ $metric -lt 0 ]; then + echo "$metric,$binary reduced in size by ${metric}% from $text1 to $text2" \ + >> $run_step_artifacts/binary.improvements + elif [ $metric -gt 0 ]; then + echo "$metric,$binary increased in size by ${metric}% from $text1 to $text2" \ + >> $run_step_artifacts/binary.regressions + fi + + total1=$(($total1 + $text1)) + total2=$(($total2 + $text2)) + done < <(tail -n +2 $run_step_top_artifacts/results-vs-prev/results.csv) + + if [ $total1 != 0 ]; then + metric=$(((100 * $total2) / $total1 - 100)) + else + # Corner case when baseline results have no intersection with + # the current results. This happens when AOSP build fails and + # size.csv only has the header. + metric=0 + fi + if [ $metric -lt 0 ]; then + echo "$metric,AOSP reduced in size by ${metric}% from $total1 to $total2" \ + >> $run_step_artifacts/aosp.improvements + elif [ $metric -gt 0 ]; then + echo "$metric,AOSP increased in size by ${metric}% from $total1 to $total2" \ + >> $run_step_artifacts/aosp.regressions + fi + + local primary_change="" regression=false + if [ -f $run_step_artifacts/aosp.regressions ]; then + primary_change=$run_step_artifacts/aosp.regressions + regression=true + elif [ -f $run_step_artifacts/binary.regressions ]; then + primary_change=$run_step_artifacts/binary.regressions + regression=true + elif [ -f $run_step_artifacts/aosp.improvements ]; then + primary_change=$run_step_artifacts/aosp.improvements + elif [ -f $run_step_artifacts/binary.improvements ]; then + primary_change=$run_step_artifacts/binary.improvements + fi + + if [ x"$primary_change" != x"" ]; then + sort -gr -o "$primary_change" "$primary_change" + if $regression; then + head -n1 "$primary_change" \ + | sed -e "s/^/# /" > $run_step_artifacts/results.regressions + fi + fi + + case "${rr[ci_project]}" in + "tcwg_aosp-build"*) + # We have passed no_build_regression_p() check above, + # so succeed. + return 0 + ;; + esac + + local -a changed_components + IFS=" " read -r -a changed_components <<< "$(print_changed_components)" + case " ${changed_components[*]} " in + *" llvm "*) ;; + *) + # Code-size builds without changed LLVM we declare as + # non-regressing. Code-size changes are due to changes in AOSP, + # not in the compiler. + return 0 + ;; + esac + + if $regression; then + return 1 + fi + return 0 + ) +} + +rr[breakup_changed_components]="breakup_changed_components llvm" + +case "${rr[ci_project]}" in + "tcwg_aosp-build"*) + run_step stop_on_fail 0 reset_artifacts + run_step skip_on_fail 1 build_aosp_toolchain + run_step skip_on_fail 2 build_shadow_llvm + run_step skip_on_fail 3 build_aosp + run_step skip_on_fail x process_shadow_data -- 3 + ;; + *) + run_step stop_on_fail -10 reset_artifacts + run_step skip_on_fail -3 build_aosp_toolchain + run_step skip_on_fail -2 build_shadow_llvm + run_step skip_on_fail -1 build_aosp + run_step skip_on_fail x process_shadow_data -- -1 + ;; +esac +run_step reset_on_fail x check_regression + +trap "" EXIT |