diff options
-rwxr-xr-x | MakeRelease.job | 15 | ||||
-rw-r--r-- | jenkins-helpers.sh | 54 | ||||
-rw-r--r-- | round-robin.sh | 184 | ||||
-rwxr-xr-x | tcwg-cleanup-stale-containers.sh | 8 | ||||
-rwxr-xr-x | tcwg-dev-build.sh | 7 | ||||
-rwxr-xr-x | tcwg_kernel-bisect.sh | 120 | ||||
-rwxr-xr-x | tcwg_kernel-build.sh | 168 |
7 files changed, 348 insertions, 208 deletions
diff --git a/MakeRelease.job b/MakeRelease.job index 83d80f20..6ca48bc3 100755 --- a/MakeRelease.job +++ b/MakeRelease.job @@ -99,10 +99,6 @@ else rsh="ssh $fileserver" fi -if test x"${target}" != x"native" -a x"${target}" != x; then - platform="--target ${target}" -fi - shared="/home/buildslave/workspace/shared/" user_snapshots="${user_workspace}/snapshots" @@ -156,6 +152,13 @@ gcc=${gcc_src:+gcc=${gcc_src}} srcs="${gcc} ${binutils} ${glibc} ${manifest}" logfile=${user_workspace}/MakeRelease-${buildnumber}-$(uname -m).log +# Set ABE's --target setting. No setting means native. +# If manifest file is set, then ABE must use its setting. +target_opt= +if [ x"${target}" != x"native" -a x"${target}" != x -a x"$manifest" = x"" ]; then + target_opt="--target ${target}" +fi + # Build a binary release tarball # Remove logfile if present (for some unknown reason) rm -f ${logfile} @@ -164,7 +167,7 @@ manifests=() tarballs=() # Canadian cross builds require a Linux hosted cross compiler first if test x"${canadian}" = x"true"; then - $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} ${platform} --build all ${libc} ${extra} --tarbin >> ${logfile} + $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} $target_opt --build all ${libc} ${extra} --tarbin >> ${logfile} abe_ret=$? host="--host i686-w64-mingw32" manifests+=($(read_var ${user_workspace}/artifacts1.txt manifest)) @@ -177,7 +180,7 @@ fi # build the mingw32 compiler only if the previous cross-compiler build was # successful. if test ${abe_ret} -eq 0; then - $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} ${platform} ${host} --build all ${libc} ${extra} >> ${logfile} + $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} $target_opt ${host} --build all ${libc} ${extra} >> ${logfile} abe_ret=$? manifests+=($(read_var ${user_workspace}/artifacts2.txt manifest)) fi diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 8a66b2a9..9c0b7ff6 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -1,5 +1,16 @@ #!/bin/bash +# Print absolute path to a file or directory +# $1: Path (must exist) +abs_path () +{ + ( + set -euf -o pipefail + + echo "$(cd $(dirname "$1") && pwd)/$(basename "$1")" + ) +} + # Assert that $@ returns success. assert () { @@ -216,9 +227,12 @@ print_tester_label_for_target () # doesn't support KVM. Test on APM builders for now. echo "tcwg-apm_64-build" ;; - aarch64-linux*) echo "tcwg-tx1_64-test" ;; - armv8l-linux*) echo "tcwg-tx1_32-test" ;; - arm-linux*) echo "tcwg-tk1_32-test" ;; + # We allocate all TK1/TX1 boards to benchmarking, so use APMs + # for cross-testing. This means we no longer test on armv7 + # hardware. + aarch64-linux*) echo "tcwg-apm_64-test" ;; + armv8l-linux*) echo "tcwg-apm_32-test" ;; + arm-linux*) echo "tcwg-apm_32-test" ;; esac ) } @@ -405,6 +419,9 @@ clone_or_update_repo_no_checkout () # Also, prune all loose objects to avoid "git gc --auto" failing # and creating .git/gc.log, which will stop future "git gc --auto" runs. git -C "$dir" gc --auto --force --prune=all + # Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock + # These occur when builds are aborted during "git remote update" or similar. + find "$dir/.git" -name "*.lock" -delete fi ( @@ -739,6 +756,8 @@ manifest_out () # variables and arrays. # "--var value" defines shell variable "$var" to "value". # "++arr elem" defines shell array "$arr[@]" and adds "elem" to it. +# "==arr[key] value" defines shell associative array "$arr[@]" and sets +# "${arr[key]}" to "value". # "@@ file" sources file. # "%% file" starts manifest in file. Also see "^^ true". # "^^ true/false %% manifest" whether to reproduce the build using manifest. @@ -1025,18 +1044,13 @@ git_push () # $1: Step to start execution at (or "" to start at the very first step) # $2: Step to finish execution at (or "" to run till the very end) # $3: Top artifact directory -# $4: Top baseline artifact directory (see Note 1) -# $5: Whether to enable "set -x" verbosity for execution steps. -# -# Note 1: Artifacts for steps before $run_step_start_at/$1 will be copied over -# from $base_artifacts/$4 +# $4: Whether to enable "set -x" verbosity for execution steps. run_step_init () { run_step_start_at="$1" run_step_finish_at="$2" run_step_top_artifacts="$3" - run_step_base_artifacts="$4" - run_step_verbose="$5" + run_step_verbose="$4" run_step_count="0" run_step_prev_step="" @@ -1044,12 +1058,6 @@ run_step_init () run_step_status=0 run_step_artifacts="" - if [ x"$run_step_base_artifacts" != x"" ]; then - run_step_use_baseline=true - else - run_step_use_baseline=false - fi - # We need absolute paths for $run_step_artifacts, which is constructed from # $run_step_top_artifacts. mkdir -p "$run_step_top_artifacts" @@ -1066,9 +1074,8 @@ run_step_init () # 2. artifact handling -- create/clean artifact directories per step. # Also, copy baseline artifacts for steps before START_AT to simulate # skipped steps. -# Step commands have $run_step_artifacts and $run_step_prev_artifacts -# pointing to artifact directories for current and previous step -# respectively. +# Step commands have $run_step_artifacts pointing to artifact directory +# for current step. # 3. logging -- dump stdout and and stderr output of step commands # into per-step console.log files, and, also, into the top-level # console.log file. @@ -1105,10 +1112,8 @@ run_step () if [ x"$pretty_step" = x"$run_step_start_at" \ -o x"$run_step_start_at" = x"" -a x"$run_step_prev_step" = x"" ]; then run_step_active=true - run_step_use_baseline=false fi - run_step_prev_artifacts=$run_step_artifacts run_step_artifacts=$run_step_top_artifacts/$run_step_count-$pretty_step rm -rf "$run_step_artifacts" @@ -1152,13 +1157,6 @@ run_step () assert false esac fi - elif $run_step_use_baseline; then - echo "COPYING BASE-ARTIFACTS for ${step[@]}" - local base_artifacts - base_artifacts="$run_step_base_artifacts/$(basename $run_step_artifacts)" - if [ -d "$base_artifacts" ]; then - rsync -a --del "$base_artifacts/" "$run_step_artifacts/" - fi else echo "SKIPPING ${step[@]}" fi diff --git a/round-robin.sh b/round-robin.sh index cfbfe117..b9c12e99 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -7,6 +7,11 @@ #rr[PROJECT_url] # PROJECT's git branch or SHA1 revision parsable by git rev-parse. +# A special value "baseline" means that PROJECT is not being updated +# in this build, and its baseline branch should be used. +# In a successful build "update_baseline" step will update baseline +# branches of all PROJECTs to the current values, thus setting +# a baseline for the next build. #rr[PROJECT_branch] # PROJECT's git SHA1 revision. These are mostly used in manifests. @@ -16,13 +21,6 @@ # be present in all above git repos (if ${rr[init_configuration]} is false). #rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}" -# PROJECT that we are testing in this build. Use ${rr[current_branch]} for -# this project, and ${rr[baseline_branch]} for all other projects. -#rr[current_project]="$current_project" - -# Git branch or SHA1 revision of ${rr[current_project]} to test. -#rr[current_branch]="$current_branch" - # Run mode: bisect or non-bisect. In bisect mode we do a couple things # slightly differently (e.g., don't touch repo in clone_repo() ). #rr[mode]="$mode" @@ -40,6 +38,40 @@ # to initialize baseline branches in git repos. #rr[init_configuration]=false +# Print round-robin components that are being updated in this build +# (the ones using non-baseline branches). +print_updated_components () +{ + ( + set -euf -o pipefail + + local delim="" + local c + for c in ${rr[components]}; do + if [ x"${rr[${c}_branch]}" != x"baseline" ]; then + printf "%s%s" "$delim" "$c" + delim=${1- } + fi + done + ) +} + +# Print the single round-robin component being updated in this build. +# Print nothing if multiple components are being updated. +print_single_updated_component () +{ + ( + set -euf -o pipefail + + local -a updated_components + updated_components=($(print_updated_components)) + + if [ ${#updated_components[@]} = 1 ]; then + echo "${updated_components[0]}" + fi + ) +} + # Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most # important artifact, since it records the metric of how successful the build # is. @@ -91,24 +123,15 @@ clone_repo () fi local branch - # Select the branch to build. - # ${rr[current_branch]} specifies branch for ${rr[current_project]}, - # and everything else uses baseline branch. - if [ x"$project" = x"${rr[current_project]}" ]; then - # Use the tip of the branch tracked in the current configuration. - if [ x"${rr[current_branch]}" = x"default" ]; then - branch=${rr[${rr[current_project]}_branch]} - elif [ x"${rr[current_branch]}" = x"baseline" ]; then - branch="refs/remotes/baseline/${rr[baseline_branch]}" - else - branch=${rr[current_branch]} - fi - elif ${rr[init_configuration]-false}; then - branch="${rr[${project}_branch]}" - else + + # Resolve "baseline" branch specifier. + if [ x"${rr[${project}_branch]}" = x"baseline" ]; then branch="refs/remotes/baseline/${rr[baseline_branch]}" + else + branch="${rr[${project}_branch]}" fi + # Allow manifest override branch="${rr[${project}_rev]-$branch}" # Decide on whether to use read-only or read-write mode for @@ -116,7 +139,7 @@ clone_repo () # developers without ssh keys on Linaro git servers to reproduce builds # in --mode "baseline". local read_only="true" - if [ x"$project" = x"${rr[current_project]}" -a \ + if [ x"${rr[${project}_branch]}" != x"baseline" -a \ x"${rr[mode]}" = x"jenkins-full" ]; then read_only=false fi @@ -134,20 +157,6 @@ clone_repo () cat <<EOF | manifest_out rr[${project}_rev]=$cur_rev EOF - - if [ x"$project" = x"${rr[current_project]}" ] \ - && ! ${rr[init_configuration]-false}; then - local baseline_rev - baseline_rev=$(git_rev_parse_long $project ${rr[baseline_branch]} baseline) - # Prepare for failure. If build fails we will bisect sha1 for - # ${rr[current_branch]} and sha1 for ${rr[baseline_branch]}. - cat > ${rr[top_artifacts]}/trigger-bisect-on-failure <<EOF -current_project=${rr[current_project]} -baseline_rev=$baseline_rev -bad_rev=$cur_rev -EOF - git -C $project rev-list --count HEAD ^$baseline_rev > ${rr[top_artifacts]}/distance-to-baseline - fi ) } @@ -298,8 +307,9 @@ build_llvm () clone_repo llvm # Copy only components from the monorepo that are required for kernel build - rsync -a --del --exclude /tools/clang llvm/llvm/ llvm-src/ + rsync -a --del --exclude /tools/clang --exclude /tools/lld llvm/llvm/ llvm-src/ rsync -a --del llvm/clang/ llvm-src/tools/clang/ + rsync -a --del llvm/lld/ llvm-src/tools/lld/ # Setup ccache and ninja wrappers. rm -rf $(pwd)/bin @@ -349,6 +359,32 @@ check_regression () set -euf -o pipefail if ! ${rr[reset_baseline]} && ! no_regression_p; then + local single_component=$(print_single_updated_component) + + if [ x"$single_component" = x"" -o x"${rr[mode]}" = x"bisect" ]; then + local c + for c in $(print_updated_components); do + local c2 + for c2 in ${rr[components]}; do + if [ x"$c" = x"$c2" ]; then + echo "${c2}_branch=${rr[${c}_branch]}" + else + echo "${c2}_branch=baseline" + fi + done >> ${rr[top_artifacts]}/trigger-build-$c + done + else + local baseline_rev cur_rev + baseline_rev=$(git_rev_parse_long $single_component ${rr[baseline_branch]} baseline) + cur_rev=$(git -C $single_component rev-parse HEAD) + + cat > ${rr[top_artifacts]}/trigger-bisect <<EOF +current_project=$single_component +baseline_branch=$baseline_rev +bad_branch=$cur_rev +EOF + fi + # Fail. false fi @@ -368,42 +404,84 @@ update_baseline () # the current one, but keep entries for results that are better # (so that we have a record of pending regressions). while no_regression_p; do + prev_head="" if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then - prev_head=$(git -C base-artifacts rev-parse HEAD) + # For every regression we want to keep artifacts for the first-bad + # build, so reset to the most relevant regression (marked by reset-baseline). + if [ -f base-artifacts/reset-baseline ] && ! ${rr[reset_baseline]}; then + prev_head=$(git -C base-artifacts rev-parse HEAD) + fi git -C base-artifacts reset --hard HEAD^ else # We got to the beginning of git history, so amend the current # commit. The initial state of baseline is "empty" branch, # which we treat as worst possible in no_regression_p(). amend="--amend" - prev_head="" break fi done - # For every regression we want to keep artifacts for the first-bad - # build, so discard one less than needed. if [ x"$prev_head" != x"" ]; then git -C base-artifacts reset --hard $prev_head fi - # Rsync current artifacts. + # Rsync current artifacts. Make sure to use -I rsync option since + # quite often size and timestamp on artifacts/results will be the same + # as on base-artifacts/results due to "git reset --hard HEAD^" below. + # This caused rsync's "quick check" heuristic to skip "results" file. # !!! From this point on, logs and other artifacts won't be included # in base-artifacts.git repo (though they will be uploaded to jenkins). - rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ + rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ - local build_rev - build_rev=$(git -C ${rr[current_project]} rev-parse HEAD) + local rev_count + if [ x"$amend" = x"" ]; then + rev_count=$(git -C base-artifacts rev-list --count HEAD) + else + rev_count="0" + fi - cd base-artifacts - git add . - git commit $amend -m "${rr[current_project]}-$build_rev: $(tail -n1 ${rr[top_artifacts]}/results) + local msg_title="$rev_count" + + if ${rr[reset_baseline]}; then + # Create a marker for builds that reset baselines (these are builds + # for bisected regressions). + touch base-artifacts/reset-baseline + msg_title="$msg_title: first-bad" + else + msg_title="$msg_title: good" + fi + + local single_component + single_component=$(print_single_updated_component) + if [ x"$single_component" != x"" ]; then + local single_rev + single_rev=$(git -C $single_component rev-parse HEAD) + msg_title="$msg_title: $single_component-$single_rev" + else + msg_title="$msg_title: $(print_updated_components "-")" + fi + msg_title="$msg_title: $(tail -n1 ${rr[top_artifacts]}/results)" + + git -C base-artifacts add . + git -C base-artifacts commit $amend -m "$msg_title $(cat ${rr[top_artifacts]}/results)" + + # We saw strange behavior with base-artifacts/results not being updated + # in the rsync above. This should be fixed by "-I" rsync option, but + # keep below asserts just in case. + if [ x"$(diff -up ${rr[top_artifacts]}/results base-artifacts/results)" != x"" ] \ + || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then + cd base-artifacts + git status + cat results + ls -la + assert false + fi ) } -# Push to ${rr[current_project]} baseline branch and to base-artifacts repo. +# Push to baseline branches and to base-artifacts repo. push_baseline () { ( @@ -411,6 +489,10 @@ push_baseline () git_init_linaro_local_remote base-artifacts baseline false git_push base-artifacts baseline ${rr[baseline_branch]} - git_push ${rr[current_project]} baseline ${rr[baseline_branch]} + + local c + for c in $(print_updated_components); do + git_push $c baseline ${rr[baseline_branch]} + done ) } diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh index 86fc04b8..cc748c5c 100755 --- a/tcwg-cleanup-stale-containers.sh +++ b/tcwg-cleanup-stale-containers.sh @@ -148,7 +148,7 @@ status=$(($status|(2*$res))) rm_volumes=($($DOCKER volume ls -q -f dangling=true)) # Filter-out named volumes like host-home and home-$USER. Leave only volumes # named like a sha1 hash. -rm_volumes=($(echo "${rm_volumes[@]}" | grep "^[a-f0-9]\{64\}\$" | cat)) +rm_volumes=($(echo "${rm_volumes[@]}" | tr " " "\n" | grep "^[a-f0-9]\{64\}\$" | cat)) if [ ${#rm_volumes[@]} != 0 ]; then echo "Removing dangling volumes" @@ -200,6 +200,10 @@ fi if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then res=0; killall --older-than ${cleanup_ssh_agent_hours}h -u $USER ssh-agent & wait $! || res=$? + if [ $res != 0 ]; then + echo "WARNING: could not kill stale ssh-agent processes" + status=$(($status|16)) + fi fi # Check if we have more containers than max_containers @@ -207,7 +211,7 @@ nb_containers=$($DOCKER ps -a | wc -l) if [ ${max_containers} -gt 0 -a ${nb_containers} -gt ${max_containers} ]; then echo "ERROR: Too many containers left after cleanup: ${nb_containers} (max: ${max_containers})" - status=$(($status|16)) + status=$(($status|32)) fi exit $status diff --git a/tcwg-dev-build.sh b/tcwg-dev-build.sh index cf24ab60..ff4afacc 100755 --- a/tcwg-dev-build.sh +++ b/tcwg-dev-build.sh @@ -8,6 +8,7 @@ convert_args_to_variables "$@" abe_branch="${abe_branch-tested}" dry_run="${dry_run-false}" +manifest="${manifest-}" release_name="${release_name-default}" target="${target-aarch64-linux-gnu}" version="${version-default}" @@ -23,6 +24,11 @@ if [ x"$release_name" = x"default" ]; then release_name="$(date +%Y%m%d-%H_%M_%S)" fi +manifest_opt= +if [ -f "$manifest" ]; then + manifest_opt="--manifest $(abs_path "$manifest")" +fi + manifest_validation_opt= if [ x"$target" = x"native" ]; then manifest_validation_opt="--manifest_validation false" @@ -35,6 +41,7 @@ $scripts/MakeRelease.job \ --target $target \ --toolchainconfig $version \ --workspace `pwd` \ + $manifest_opt \ $manifest_validation_opt \ ${binutils:+--binutils "$binutils"} \ ${gcc:+--gcc "$gcc"} diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index 45a6aaf0..136dc24c 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -16,7 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*" convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" -obligatory_variables bad_rev baseline_rev build_script rr[ci_project] +obligatory_variables bad_branch baseline_branch build_script current_project BUILD_URL="${BUILD_URL:-$(pwd)}" reproduce_bisect="${reproduce_bisect:-false}" @@ -25,13 +25,7 @@ reproduce_bisect="${reproduce_bisect:-false}" convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@" $reproduce_bisect || manifest_pop -obligatory_variables rr[current_project] rr[ci_config] - -current_project=${rr[current_project]} - -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then - obligatory_variables toolchain -fi +obligatory_variables rr[ci_project] rr[ci_config] verbose="${verbose:-true}" @@ -43,38 +37,41 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/fai rebase_workaround=false -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ] \ - && [ x"$current_project" = x"linux" -a x"${rr[linux_version]}" = x"next" ]; then - # Workaround linux-next/master rebasing on top of linux-next/stable. - # Search for regressions against linux-mainline:master (aka linux-next:stable). - clone_or_update_repo $current_project stable ${rr[linux_url]} - # Just in case linux-next:stable has advanced between the build and bisect jobs, - # use merge base between linux-next:stable and $bad_rev. - linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}" - cat <<EOF | manifest_out +case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in + tcwg_kernel/*-next-*:linux) + # Workaround linux-next/master rebasing on top of linux-next/stable. + # Search for regressions against linux-mainline:master (aka linux-next:stable). + clone_or_update_repo $current_project stable ${rr[linux_url]} + # Just in case linux-next:stable has advanced between the build and bisect jobs, + # use merge base between linux-next:stable and $bad_branch. + bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}" + linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}" + cat <<EOF | manifest_out declare -g linux_next_stable=$linux_next_stable EOF - echo "Rebase workaround: forcing baseline_rev to $linux_next_stable" - baseline_rev=$linux_next_stable - rebase_workaround=true -fi + echo "Rebase workaround: forcing baseline_rev to $linux_next_stable" + baseline_rev=$linux_next_stable + rebase_workaround=true + ;; +esac # Build baseline that we are going to re-use to speed-up bisection. # (This also confirms that infrastructure is OK.) -echo "Testing baseline_rev $baseline_rev (should be success)" +echo "Testing baseline_branch $baseline_branch (should be success)" $build_script \ ^^ $reproduce_bisect \ %% $rel_artifacts/manifests/build-baseline.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ ==rr[mode] "baseline" \ - ==rr[current_branch] "$baseline_rev" \ + ==rr[${current_project}_branch] "$baseline_branch" \ ==rr[reset_baseline] true \ ==rr[top_artifacts] "$rel_artifacts/build-baseline" \ --verbose "$verbose" -assert ! [ -f $artifacts/failures ] - -cd $current_project +baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}" +cat <<EOF | manifest_out +declare -g baseline_rev=$baseline_rev +EOF ln -f -s "build-baseline" "$artifacts/build-$baseline_rev" ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh" @@ -124,6 +121,7 @@ $build_script \ %% $rel_artifacts/manifests/build-\$rev.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ ==rr[mode] bisect \ + ==rr[${current_project}_branch] \$rev \ ==rr[top_artifacts] $rel_artifacts/build-\$rev \ --verbose "$verbose" & res=0 && wait \$! || res=\$? @@ -138,6 +136,13 @@ fi EOF chmod +x $artifacts/test.sh +bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}" +cat <<EOF | manifest_out +declare -g bad_rev=$bad_rev +EOF + +cd $current_project + git checkout --detach $bad_rev $artifacts/test.sh & res=0 && wait $! || res=$? @@ -147,14 +152,30 @@ if [ x"$res" = x"0" ]; then if $rebase_workaround; then echo "Rebase workaround: no regression between $baseline_rev and $bad_rev" sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name + case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in + tcwg_kernel/llvm-*-next-*:linux) + cat > $artifacts/trigger-build-rebase <<EOF +llvm_branch=baseline +EOF + ;; + tcwg_kernel/gnu-*-next-*:linux) + cat > $artifacts/trigger-build-rebase <<EOF +binutils_branch=baseline +gcc_branch=baseline +EOF + ;; + *) assert false ;; + esac + cat >> $artifacts/trigger-build-rebase <<EOF +linux_branch=$baseline_rev +reset_baseline=true +EOF else echo "WARNING: build for bad_rev $bad_rev succeeded" - # Build job had a spurious failure. Re-try. - cat > $artifacts/trigger-2-build-master <<EOF -current_project=$current_project -current_branch=$bad_rev -EOF sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name + # Retry build with default parameters + cat > $artifacts/trigger-build-retry <<EOF +EOF fi echo > $artifacts/jenkins/mail-recipients.txt trap "" EXIT @@ -204,10 +225,10 @@ if [ x"$res" = x"0" ]; then # We need to be careful to avoid re-trigger loops, so verify that # last_good is an ancestor of bad_rev. assert git merge-base --is-ancestor $last_good $bad_rev - cat > $artifacts/trigger-0-bisect-again <<EOF + cat > $artifacts/trigger-bisect <<EOF current_project=$current_project -baseline_rev=$baseline_rev -bad_rev=$last_good +baseline_branch=$baseline_rev +bad_branch=$last_good EOF # Don't send any emails. echo > $artifacts/jenkins/mail-recipients.txt @@ -271,6 +292,10 @@ Could not identify regression in $current_project for CI configuration ${rr[ci_c EOF fi +# In log scan for errors below +# - ": error:" detects compiler errors from GCC and Clang +# - "^ERROR:" detects linker errors +# - "] Error " detects GNU make errors cat >> $artifacts/jenkins/mail-body.txt <<EOF Configuration details: $(cat $artifacts/manifests/build-baseline.sh | grep '_url]\|_branch]' | grep -v '="no_') @@ -282,7 +307,7 @@ to (for $bad_name == $bad_sha1) $(cat $artifacts/build-$bad_sha1/results) First few errors in logs of $bad_name: -$(grep ": error:" $artifacts/build-$bad_sha1/console.log | head) +$(grep ": error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head) Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/ Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/ @@ -337,29 +362,22 @@ fi # Set mail recipients last to preserve catch-error value from .yaml file. # Email developers. CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org" -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then - case "$toolchain:$current_project" in - gnu:linux) ;; - gnu:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;; - llvm:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;; - llvm:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;; - esac -fi +case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in + tcwg_kernel/gnu-*:linux) ;; + tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;; + tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;; + tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;; +esac cat > $artifacts/jenkins/mail-recipients.txt <<EOF $CI_MAIL_RECIPIENTS EOF # Reset baseline to the regressed commit so that we will catch subsequent # regressions (worse than $bad_rev). -cat > $artifacts/trigger-1-reset-baseline <<EOF -current_project=$current_project -current_branch=$first_bad -reset_baseline=true -EOF +cp $artifacts/build-$first_bad/trigger-build-$current_project $artifacts/trigger-build-1-reset +echo "reset_baseline=true" >> $artifacts/trigger-build-1-reset # Trigger master build now instead of waiting for next timed SCM trigger. -cat > $artifacts/trigger-2-build-master <<EOF -current_project=$current_project -EOF +cp $artifacts/build-$bad_rev/trigger-build-$current_project $artifacts/trigger-build-2-default trap "" EXIT diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index ab169e5b..9485e0ba 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -4,26 +4,19 @@ set -euf -o pipefail scripts=$(dirname $0) . $scripts/jenkins-helpers.sh +. $scripts/round-robin.sh convert_args_to_variables "$@" obligatory_variables rr[ci_config] -# Execution mode: baseline, bisect, continue, jenkins-full +# Execution mode: baseline, bisect, jenkins-full rr[mode]="${rr[mode]-baseline}" -case "${rr[mode]}" in - "jenkins-full") - obligatory_variables rr[current_project] rr[current_branch] - ;; -esac - # Set custom revision for one of the projects, and use baseline revisions # for all other projects. rr[ci_project]="${rr[ci_project]-tcwg_kernel}" rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}" -rr[current_branch]="${rr[current_branch]-default}" -rr[current_project]="${rr[current_project]-none}" rr[reset_baseline]="${rr[reset_baseline]-false}" rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}" @@ -32,9 +25,22 @@ IFS=- read -a ci_config <<EOF ${rr[ci_config]} EOF rr[toolchain]=${rr[toolchain]-${ci_config[0]}} +rr[release]=${rr[release]-${ci_config[1]}} rr[target]=${rr[target]-${ci_config[2]}} rr[linux_config]=${rr[linux_config]-${ci_config[4]}} +case "${rr[toolchain]}" in + llvm) rr[components]="llvm linux" ;; + gnu) rr[components]="binutils gcc linux" ;; + *) assert false ;; +esac + +# Use baseline branches by default. +for c in ${rr[components]}; do + rr[${c}_branch]=${rr[${c}_branch]-baseline} + obligatory_variables rr[${c}_url] +done + start_at="${start_at-default}" finish_at="${finish_at-default}" verbose="${verbose-true}" @@ -47,29 +53,21 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT # Set start and finish steps for different modes. default_start_at="" default_finish_at="" -base_artifacts_opt="base-artifacts" case "${rr[mode]}" in "baseline") default_finish_at="update_baseline" ;; "bisect") - case "${rr[toolchain]}:${rr[current_project]}" in + case "${rr[toolchain]}:$(print_single_updated_component)" in gnu:binutils) default_start_at="build_abe-binutils" ;; gnu:gcc) default_start_at="build_abe-stage1" ;; - gnu:linux) default_start_at="count_linux_objs" ;; + gnu:linux) default_start_at="build_linux" ;; llvm:llvm) default_start_at="build_llvm" ;; - llvm:linux) default_start_at="count_linux_objs" ;; + llvm:linux) default_start_at="build_linux" ;; + *) assert false ;; esac default_finish_at="check_regression" ;; - "continue") - # Developer mode. - case "${rr[toolchain]}" in - "gnu") default_start_at="prepare_abe" ;; - "llvm") default_start_at="build_llvm" ;; - esac - base_artifacts_opt="" - ;; "jenkins-full") ;; esac if [ x"$start_at" = x"default" ]; then @@ -79,12 +77,10 @@ if [ x"$finish_at" = x"default" ]; then finish_at="$default_finish_at" fi -run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose" - -. $scripts/round-robin.sh +run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose" # Build Linux kernel -build_linux () +build_linux_1 () { ( set -euf -o pipefail @@ -94,7 +90,7 @@ build_linux () rm -rf $(pwd)/bin mkdir $(pwd)/bin - local bin cc + local bin cc ld_opt case "${rr[toolchain]}" in llvm) @@ -107,6 +103,12 @@ build_linux () ;; esac + case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in + llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;; + llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;; + *) ld_opt="" ;; + esac + # Use binutils, etc from $bin export PATH="$bin:$PATH" @@ -114,7 +116,8 @@ build_linux () # Otherwise the compiler is new in every build and we would # only clobber ccache volume. local ccache="" - if [ x"${rr[mode]}" != x"jenkins-full" -a x"${rr[current_project]}" = x"linux" ]; then + if [ x"${rr[mode]}" != x"jenkins-full" \ + -a x"$(print_single_updated_component)" = x"linux" ]; then ccache="ccache" fi cat > $(pwd)/bin/${rr[target]}-cc <<EOF @@ -124,7 +127,7 @@ EOF chmod +x $(pwd)/bin/${rr[target]}-cc # Define make variables. - local opts="CC=$(pwd)/bin/${rr[target]}-cc SUBLEVEL=0 EXTRAVERSION=-bisect" + local opts="CC=$(pwd)/bin/${rr[target]}-cc $ld_opt SUBLEVEL=0 EXTRAVERSION=-bisect" if [ x"${rr[target]}" != x"$(uname -m)" ]; then opts="$opts ARCH=$(print_kernel_target ${rr[target]})" opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-" @@ -144,34 +147,61 @@ EOF KBUILD_BUILD_TIMESTAMP=0 make $opts -j$(nproc --all) -s -k & local res=0 && wait $! || res=$? ccache -s - cp include/config/kernel.release $run_step_artifacts/ return $res ) } -# Count number successfully built .o files in linux (and build linux) -count_linux_objs () +# Build linux and count number successfully built .o files in linux +build_linux () { ( set -euf -o pipefail - if true; then - build_linux & - local res=0; wait $! || res=$? - - # Number of .o files created is the main success metric. - echo "linux_n_obj:" >> ${rr[top_artifacts]}/results - if [ $res != 0 ]; then - local linux_n_obj - linux_n_obj=$(find linux -name "*.o" | wc -l) - echo "$linux_n_obj" >> ${rr[top_artifacts]}/results - else - echo "all" >> ${rr[top_artifacts]}/results - fi + build_linux_1 & + local res=0 && wait $! || res=$? + + # Number of .o files created is the main success metric. + local linux_n_obj + linux_n_obj=$(find linux -name "*.o" | wc -l) + echo "linux_n_obj:" >> ${rr[top_artifacts]}/results + echo "$linux_n_obj" >> ${rr[top_artifacts]}/results + + if [ $res = 0 ]; then + echo "linux build successful:" >> ${rr[top_artifacts]}/results + echo "all" >> ${rr[top_artifacts]}/results fi + + return $res ) } +# Boot linux kernel +boot_linux () +{ + ( + set -euf -o pipefail + + local image cpu + case ${rr[target]} in + aarch64) + image=linux/arch/arm64/boot/Image.gz + cpu="-cpu cortex-a53" + ;; + arm) + image=linux/arch/arm/boot/zImage + cpu="" + ;; + *) assert false ;; + esac + timeout 10s qemu-system-${rr[target]} \ + -kernel $image -machine virt $cpu -m 512 \ + -serial mon:stdio -display none \ + -append "console=ttyAMA0 panic=-1" -no-reboot + + echo "linux boot successful:" >> ${rr[top_artifacts]}/results + echo "boot" >> ${rr[top_artifacts]}/results + ) +} # Exit with code 0 if no regression compared to base-artifacts/results. no_regression_p () @@ -182,37 +212,34 @@ no_regression_p () local linux_n_obj linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results) - if [ x"$linux_n_obj" = x"all" ]; then - # Ideal result, no need to compare to baseline. - return 0 - fi - - if ! [ "$linux_n_obj" -ge "-10" ]; then - # Something is very wrong with result (e.g., it's not a number). - return 1 - fi - # Assume worst for non-existent baseline. local base_linux_n_obj="-10" - if [ -f base-artifacts/results ]; then base_linux_n_obj=$(tail -n1 base-artifacts/results) - if [ x"$base_linux_n_obj" = x"all" ]; then - # Current build is not ideal, so make sure baseline "all" is - # better than the current result. - base_linux_n_obj=$(($linux_n_obj+1)) - elif ! [ "$base_linux_n_obj" -ge "-10" ]; then - # Something is very wrong with baseline result - # (e.g., it's not a number). - return 0 - fi fi - if [ $linux_n_obj -ge $base_linux_n_obj ]; then - return 0 - else - return 1 - fi + case "$linux_n_obj:$base_linux_n_obj" in + boot:*) return 0 ;; + *:boot) return 1 ;; + all:*) return 0 ;; + *:all) return 1 ;; + *) + if ! [ "$linux_n_obj" -ge "-10" ]; then + # Something is very wrong with result (e.g., it's not a number). + return 1 + fi + if ! [ "$base_linux_n_obj" -ge "-10" ]; then + # Something is very wrong with result (e.g., it's not a number). + return 0 + fi + + if [ $linux_n_obj -ge $base_linux_n_obj ]; then + return 0 + else + return 1 + fi + ;; + esac ) } @@ -227,7 +254,8 @@ case "${rr[toolchain]}" in run_step skip_on_fail -1 build_llvm ;; esac -run_step skip_on_fail x count_linux_objs +run_step skip_on_fail x build_linux +run_step skip_on_fail x boot_linux run_step reset_on_fail x check_regression run_step stop_on_fail x update_baseline run_step stop_on_fail x push_baseline |