diff options
Diffstat (limited to 'tcwg_kernel-build.sh')
-rwxr-xr-x | tcwg_kernel-build.sh | 183 |
1 files changed, 93 insertions, 90 deletions
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index bb923767..53efe1b2 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -11,38 +11,32 @@ scripts=$(dirname $0) convert_args_to_variables "$@" obligatory_variables rr[ci_config] +declare -A rr -# Execution mode: baseline, bisect, jenkins-full -# shellcheck disable=SC2154 -rr[mode]="${rr[mode]-baseline}" +# Execution mode: build or bisect +rr[mode]="${rr[mode]-build}" # Set custom revision for one of the projects, and use baseline revisions # for all other projects. -# shellcheck disable=SC2154 rr[ci_project]="${rr[ci_project]-tcwg_kernel}" -# shellcheck disable=SC2154 rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}" -# shellcheck disable=SC2154 -rr[update_baseline]="${rr[update_baseline]-update}" -# shellcheck disable=SC2154 +rr[update_baseline]="${rr[update_baseline]-ignore}" rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}" # {toolchain_name}-{toolchain_ver}-{target}-{linux}-{linux_config} IFS=- read -a ci_config <<EOF ${rr[ci_config]} EOF -# shellcheck disable=SC2154 rr[toolchain]=${rr[toolchain]-${ci_config[0]}} -# shellcheck disable=SC2154 -rr[release]=${rr[release]-${ci_config[1]}} -# shellcheck disable=SC2154 rr[target]=${rr[target]-${ci_config[2]}} -# shellcheck disable=SC2154 -rr[linux_config]=${rr[linux_config]-${ci_config[4]}} +linux_config=${linux_config-${ci_config[4]}} + +# Artifacts version +rr[major]=4 +rr[minor]=0 case "${rr[toolchain]}" in llvm) - # shellcheck disable=SC2154 rr[components]="binutils llvm linux qemu" ;; gnu) rr[components]="binutils gcc linux qemu" ;; @@ -67,21 +61,16 @@ trap print_traceback EXIT default_start_at="" default_finish_at="" case "${rr[mode]}" in - "baseline") - default_finish_at="update_baseline" - ;; "bisect") - case "${rr[toolchain]}:$(print_single_updated_component)" in - gnu:binutils) default_start_at="build_abe-binutils" ;; - gnu:gcc) default_start_at="build_abe-stage1" ;; - llvm:llvm) default_start_at="build_llvm" ;; - *:linux) default_start_at="build_linux" ;; - *:qemu) default_start_at="build_abe-qemu" ;; + case "$(print_single_updated_component)" in + binutils) default_start_at="build_abe-binutils" ;; + gcc) default_start_at="build_abe-stage1" ;; + llvm) default_start_at="build_kernel_llvm" ;; + linux) default_start_at="build_linux" ;; + qemu) default_start_at="build_abe-qemu" ;; *) assert false ;; esac - default_finish_at="check_regression" ;; - "jenkins-full") ;; esac if [ x"$start_at" = x"default" ]; then start_at="$default_start_at" @@ -92,6 +81,30 @@ fi run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose" +# Build LLVM for building Linux kernel +build_kernel_llvm () +{ + ( + set -euf -o pipefail + + local host target extra_targets="" + + host=$(print_llvm_target native) + target=$(print_llvm_target ${rr[target]}) + + if [ x"$target" != x"$host" ]; then + # We need $host target for HOSTCC + extra_targets="$extra_targets;$host" + fi + if [ x"$target" = x"AArch64" ]; then + # We need 32-bit ARM target to compile vdso32 + extra_targets="$extra_targets;ARM" + fi + + build_llvm "clang;lld" "$extra_targets" + ) +} + # Build Linux kernel build_linux_1 () { @@ -106,9 +119,10 @@ build_linux_1 () rm -rf "$(pwd)"/bin mkdir "$(pwd)"/bin - local abe_bin cc ld_opt="" + local gnu_host abe_bin cc ld hostcc ld_opt="" - abe_bin="$(pwd)/abe/builds/destdir/x86_64-pc-linux-gnu/bin" + gnu_host=$(print_gnu_target native) + abe_bin="$(pwd)/abe/builds/destdir/$gnu_host/bin" # Use binutils $abe_bin for both GNU and LLVM builds. # For LLVM builds at the very least we need target assembler, since, # otherwise, clang will try to use /usr/bin/as for cases that integrated @@ -122,19 +136,25 @@ build_linux_1 () export PATH="$llvm_bin:$PATH" cc="$llvm_bin/clang" - ld_opt="LD=$llvm_bin/ld.lld" + ld="$llvm_bin/ld.lld" + hostcc="$cc" + + ld_opt="LD=$ld" ;; gnu) cc="$abe_bin/$(print_gnu_target ${rr[target]})-gcc" + ld="$abe_bin/$(print_gnu_target ${rr[target]})-ld" + hostcc=gcc ;; esac + echo -n "Compiler: "; $cc --version | head -n1 + echo -n "Linker: "; $ld --version | head -n1 # Use ccache only when bisecting linux or qemu (or preparing to). # Otherwise the compiler is new in every build and we would # only clobber ccache volume. local ccache="" case "${rr[mode]}:$(print_single_updated_component)" in - "baseline:") ccache="ccache" ;; "bisect:linux") ccache="ccache" ;; "bisect:qemu") ccache="ccache" ;; esac @@ -150,14 +170,15 @@ EOF if [ x"${rr[target]}" != x"$(uname -m)" ]; then opts="$opts ARCH=$(print_kernel_target ${rr[target]})" opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-" - opts="$opts HOSTCC=gcc" + opts="$opts HOSTCC=$hostcc" + echo -n "Host cc: "; $cc --version | head -n1 fi cd linux make $opts distclean - make $opts ${rr[linux_config]} + make $opts $linux_config sed -i -e 's:CONFIG_LOCALVERSION_AUTO=y:# CONFIG_LOCALVERSION_AUTO is not set:' .config set +f; rm -f localversion*; set -f make $opts oldconfig @@ -213,8 +234,9 @@ boot_linux () *) assert false ;; esac - local qemu - qemu="$(pwd)/abe/builds/hosttools/x86_64-pc-linux-gnu/bin/qemu-system-${rr[target]}" + local gnu_host qemu + gnu_host=$(print_gnu_target native) + qemu="$(pwd)/abe/builds/hosttools/$gnu_host/bin/qemu-system-${rr[target]}" timeout --foreground 60s "$qemu" \ -kernel $image -machine virt $cpu -m 512 \ @@ -244,65 +266,48 @@ no_regression_p () base_linux_n_obj=$(grep -v "^#" $ref_artifacts/results | tail -n1) fi - # In log scan for errors below - # - sed -e 's/"[^"]*"//g' -- removes quoted "error: app diagnostics" strings - # - " error:" detects compiler errors from GCC and Clang (including GCC ICEs) - # - "^ERROR:" detects linker errors - # - ": undefined reference" detects missing symbols during linking - # - "] Error " detects GNU make errors - # Then grep for "grep" to exclude other uses of this search. - # shellcheck disable=SC2154 - # (defined by init_step in jenkins-helpers) - cat > $run_step_artifacts/results.regressions <<EOF -# First few build errors in logs: -$(cat $new_artifacts/console.log | sed -e 's/"[^"]*"//g' | grep " error:\|^ERROR:\|: undefined reference\|\] Error " | grep -v "grep" | head | sed -e "s/^/# /") -EOF - + local res="" case "$linux_n_obj:$base_linux_n_obj" in - boot:*) return 0 ;; - *:boot) return 1 ;; - all:*) return 0 ;; - *:all) return 1 ;; - *) - if ! [ "$linux_n_obj" -ge "-10" ]; then - # Something is very wrong with result (e.g., it's not a number). - return 1 - fi - if ! [ "$base_linux_n_obj" -ge "-10" ]; then - # Something is very wrong with result (e.g., it's not a number). - return 0 - fi - - if [ $linux_n_obj -ge $base_linux_n_obj ]; then - return 0 - else - return 1 - fi - ;; + boot:*) res=0 ;; + *:boot) res=1 ;; + all:*) res=0 ;; + *:all) res=1 ;; esac - ) -} -# Implement rr[breakup_updated_components] hook. -tcwg_kernel_breakup_updated_components () -{ - ( - set -euf -o pipefail - - # Linux changes tend to cause the most regressions. - # Breakup updated components into linux and the rest of components - # to reduce the number of builds. - if print_updated_components "\n" | grep -q "^linux\$"; then - echo "linux" - print_updated_components "\n" | grep -v "^linux\$" | tr '\n' ' ' | sed -e "s/ \$//g" - echo - else - print_updated_components "\n" + if [ x"$res" != x"0" ]; then + local build_res + + no_build_regression_p "$@" & + build_res=0 && wait $! || build_res=$? + + if [ x"$res" = x"" ]; then + res=$build_res + + # Handle a corner case when a linux commit removes a few + # files from the tree, causing number of object to decrease. + # We do not consider it a regression when the count of object + # files decreases by less than 1%. + if [ x"$res" != x"0" ]; then + if [ x"$(print_single_updated_component)" = x"linux" ] \ + && [ $linux_n_obj -gt 0 ] \ + && [ $linux_n_obj -lt $base_linux_n_obj ]; then + local ratio + ratio=$(($base_linux_n_obj - $linux_n_obj)) + ratio=$((100 * $ratio / $base_linux_n_obj)) + if [ $ratio = 0 ]; then + # Difference in object count is less than 1%. + res=0 + fi + fi + fi + fi fi + + return $res ) } -# shellcheck disable=SC2154 -rr[breakup_updated_components]=tcwg_kernel_breakup_updated_components + +rr[breakup_changed_components]="breakup_changed_components linux" run_step stop_on_fail -10 reset_artifacts run_step stop_on_fail x prepare_abe @@ -312,14 +317,12 @@ case "${rr[toolchain]}" in run_step skip_on_fail -5 build_abe stage1 ;; llvm) - run_step skip_on_fail -5 build_llvm + run_step skip_on_fail -5 build_kernel_llvm ;; esac run_step skip_on_fail -2 build_abe qemu run_step skip_on_fail x build_linux run_step skip_on_fail x boot_linux run_step reset_on_fail x check_regression -run_step stop_on_fail x update_baseline -run_step stop_on_fail x push_baseline trap "" EXIT |