diff options
Diffstat (limited to 'round-robin.sh')
| -rw-r--r-- | round-robin.sh | 1434 |
1 files changed, 1019 insertions, 415 deletions
diff --git a/round-robin.sh b/round-robin.sh index feeafae3..06eceae1 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -4,41 +4,56 @@ . "$(dirname $0)"/jenkins-helpers.sh # Round-Robin associative array. +# FIXME: This should be declared when starting a new manifest with +# %%rr[top_artifacts]. declare -gA rr +# Major and minor versions of the manifest. These are used in +# round-robin-baseline.sh to determine when historic result in base-artifacts/ +# needs to be regenerated. +# Mismatch in major numbers means that the historic result may not be +# compatible with the current scripts, and it is OK to drop it. +# Mismatch in minor numbers means that the historic result should be +# compatible with the current scripts, and it should be updated normally. +# In most cases we will be increasing minor numbers to trigger regeneration of +# reports in interesting-commits.git and updating jira cards. +# Mismatch in patch numbers means that the historic results are fully compatible +# with the current scripts. However, the generated notify files are not aligned +# anymore with one of the backend (ex: dashboard), and then need to be regenerated. +#rr[major]="0" +#rr[minor]="0" +#rr[patch]="0" + # PROJECT's git url#branch or url#SHA1 revision parsable by git rev-parse. # A special value "baseline" means that PROJECT is not being updated # in this build, and its baseline branch should be used. -# In a successful build "update_baseline" step will update baseline +# After a successful build round-robin-baseline.sh will update baseline # branches of all PROJECTs to the current values, thus setting # a baseline for the next build. #rr[PROJECT_git] -# PROJECT's git SHA1 revision. These are mostly used in manifests. +# PROJECT's git SHA1 revision. This is used in manifest and overrides +# any branch setting in ${rr[PROJECT_git]}. #rr[PROJECT_rev] -# Baseline branch name for current configuration. These branches should -# be present in all above git repos (if ${rr[init_configuration]} is false). +# Baseline branch name for current configuration. #rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}" -# Run mode: bisect or non-bisect. In bisect mode we do a couple things -# slightly differently (e.g., don't touch repo in clone_repo() ). +# Run mode: build or bisect. In bisect mode we do a couple things +# slightly differently (e.g., don't touch repo in clone_repo() ), and +# it is allowed to have only a single component updated. #rr[mode]="$mode" # How to handle baseline: -# - init: use "empty" results for base-artifacts, which will make current -# build successful. Push our artifacts as the one and only entry. -# - update: update baseline branches of base-artifacts and components' repos +# - onsuccess: update baseline branches of base-artifacts and components' repos # on success (generate trigger files for bisection on failure). -# - reset: ignore failures in check_regression(), which will make current +# - force: ignore failures in check_regression(), which will make current # build successful. Push our artifacts to the top of base-artifacts/. -# - push: push results from current build (whatever they are, regressions are -# ignored) as new commit to base-artifacts, and update baseline -# branches. This is useful for user projects, e.g., to generate -# historical results for several toolchain versions. -# - rebase: treat results of this build as historically eldest, and -# rebase base-artifacts commits on top of this build's artifacts. -#rr[update_baseline]=update/reset/init/rebase +# - init: use "empty" results for base-artifacts, which will make current +# build successful. Push our artifacts as the one and only entry. +# - ignore: Do not affect baseline. Useful for developer testing. +#rr[update_baseline]=onsuccess/force/init/ignore + # Target architecture to build for: arm or aarch64. #rr[target]="$target" @@ -51,51 +66,18 @@ declare -gA rr # shellcheck disable=SC2154 rr[no_regression_p]=no_regression_p -# Hook to break up updated component (see print_updated_components) into +# Hook to break up changed component (see print_changed_components) into # smaller sets: print one set per line. By default, breakup into singletons. # shellcheck disable=SC2154 -rr[breakup_updated_components]=breakup_updated_components - -# Print round-robin components that are being updated in this build -# (the ones using non-baseline branches). -print_updated_components () -{ - ( - set -euf -o pipefail - - local delim="" - local c - for c in ${rr[components]}; do - if [ x"${rr[${c}_git]}" != x"baseline" ]; then - echo -ne "$delim$c" - delim=${1- } - fi - done - echo - ) -} - -# By default, print each component on its own line. -breakup_updated_components () -{ - print_updated_components "\n" -} +rr[breakup_changed_components]=breakup_changed_components -# Print the single round-robin component being updated in this build. -# Print nothing if multiple components are being updated. -print_single_updated_component () -{ - ( - set -euf -o pipefail +# Abe's repository and branch to use for the build. +rr[abe_repo]="https://git-us.linaro.org/toolchain/abe.git" +rr[abe_branch]="master" - local update_components - IFS=" " read -r -a updated_components <<< "$(print_updated_components)" - - if [ ${#updated_components[@]} -eq 1 ]; then - echo "${updated_components[0]}" - fi - ) -} +# Host compiler defaults to /usr/bin/gcc and g++ +rr[host_cc]="/usr/bin/gcc" +rr[host_c++]="/usr/bin/g++" # Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most # important artifact, since it records the metric of how successful the build @@ -113,23 +95,65 @@ reset_artifacts () fresh_dir $run_step_top_artifacts \ $run_step_top_artifacts/console.log \ $run_step_artifacts/console.log \ + $run_step_top_artifacts/manifest.sh \ "$run_step_top_artifacts/jenkins/*" + local branch repo1 repo + branch="${rr[baseline_branch]}" + repo1="${branch#linaro-local/ci/}" + repo="ssh://bkp.tcwglab/home/tcwg-buildslave/base-artifacts/$repo1.git" + + local git_result + git_result=$(git ls-remote --heads "$repo" "refs/heads/$branch" || true) + + if [ "$git_result" = "" ]; then + echo "WARNING: BASELINE IS NOT FOUND; INITIALIZING AN EMPTY BASELINE" + rr[update_baseline]="init" + run_step_patch_env "==rr[update_baseline]" "init" + fi + + if [ x"${rr[update_baseline]}" = x"init" ]; then + branch="empty" + repo="ssh://bkp.tcwglab/home/tcwg-buildslave/base-artifacts/empty.git" + fi + # Clone base-artifacts here so that bisect runs (which skip this step) # don't overwrite it. # base-artifacts repo is big and changes all the time, so we # fetch only the $baseline_branch, instead of all branches. - local single_branch - if [ x"${rr[update_baseline]}" = x"init" ]; then - single_branch=empty + rr[base-artifacts_rev]="${rr[base-artifacts_rev]-$branch}" + clone_or_update_repo base-artifacts "${rr[base-artifacts_rev]}" \ + "$repo" auto "$branch" + + git_annex_download base-artifacts annex + + if [ -d base-artifacts/git/ ]; then + # Copy baseline git_url/git_rev settings into the current build, + # which will then be overwritten in due course by clone_repo() + # of various components. + # Note that we need to copy data for all components to correctly handle + # builds that fail before all their components are checked out. + # Note that we want to iterate over components (rather than rsync + # the whole base-artifacts/git/ directory) to avoid copying data for + # removed components. + local c + for c in ${rr[components]}; do + get_baseline_git ${c}_url | set_current_git ${c}_url + get_baseline_git ${c}_rev | set_current_git ${c}_rev + done else - single_branch=${rr[baseline_branch]} + # We are in "init" baseline build, apparently. "Init" builds should + # have a full set of git data for all components specified on + # the command line, so that get_baseline_git() is not called. + mkdir base-artifacts/git fi - rr[base-artifacts_rev]="${rr[base-artifacts_rev]-$single_branch}" - clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch + # We need rr[major], rr[minor], and rr[baseline_branch] in round-robin-baseline.sh + # when rewriting history, which happens right after reset_artifacts()/run-manifest-start. cat <<EOF | manifest_out -rr[base-artifacts_rev]=$(git -C base-artifacts rev-parse HEAD) +rr[baseline_branch]="${rr[baseline_branch]}" +rr[major]="${rr[major]}" +rr[minor]="${rr[minor]}" EOF ) } @@ -142,38 +166,99 @@ clone_repo () set -euf -o pipefail local project="$1" - if [ x"${rr[mode]}" = x"bisect" ]; then - # Cleanup current checkout in bisect mode. - git_clean "$project" - return 0 - fi - local url branch - if [ x"${rr[${project}_git]}" != x"baseline" ]; then - # Fetch and checkout from the specified repo. - url="${rr[${project}_git]%#*}" - branch="${rr[${project}_git]#*#}" - else - # Fetch and checkout from baseline repo. - url=$(print_baseline_repo "$project" true) - branch="${rr[baseline_branch]}" - fi - - clone_or_update_repo_no_checkout "$project" "$url" auto "" origin \ - > /dev/null + case "${rr[${project}_git]}" in + *"#"*) + # Fetch from specified remote repo. + url="${rr[${project}_git]%#*}" + branch="${rr[${project}_git]#*#}" + ;; + "baseline") + # Fetch from remote repo specified in the baseline. + url=$(get_baseline_git ${project}_url) + branch=$(get_baseline_git ${project}_rev) + ;; + *) + # Use revision in the existing local repo. + # Most likely it is "HEAD" in precommit testing. + url="" + branch="${rr[${project}_git]}" + ;; + esac - # Allow manifest override + # Allow manifest override for $url + url="${rr[${project}_url]-$url}" + # Allow manifest override for $branch branch="${rr[${project}_rev]-$branch}" - git -C $project checkout --detach "$branch" + if [ x"${rr[mode]}" = x"bisect" ]; then + # In bisect mode we rely on round-robin-bisect.sh to arrange + # all source directories, and here we only clean them. + # Note that in bisect mode round-robin-bisec.sh passes "_git" spec + # as url#sha1 so that create_trigger_files() generates trigger-build-* + # suitable for triggering last_good and first_bad builds. + git_clean "$project" + elif [ "$url" = "" ]; then + # In local mode -- clean the project directory. + git_clean "$project" + # Don't use git_checkout(), which prefers remote resolution of refs. + git -C "$project" checkout --detach "$branch" + else + clone_or_update_repo "$project" "$branch" "$url" > /dev/null + fi local cur_rev cur_rev=$(git -C $project rev-parse HEAD) + rr[debug_${project}_date]=$(git -C $project show --no-patch \ + --pretty="%ct # %cr" HEAD) + + # Store git info in the manifest and git data into artifacts. + # Git data in artifacts is then used by subsequent builds to fetch + # baseline commits. + if [ "$url" != "" ]; then + echo "$url" | set_current_git ${project}_url + fi + echo "$cur_rev" | set_current_git ${project}_rev + ) +} - cat <<EOF | manifest_out -rr[${project}_rev]=$cur_rev +# Configure ccache wrappers in "$1". +setup_ccache () +{ + ( + set -euf -o pipefail + local bin="$1" + + local -a ccache_opts=("CCACHE_BASEDIR=$workspace") + if [ -d "$HOME/.ccache" ] && ! touch "$HOME/.ccache" 2>/dev/null; then + # Setup read-only ccache; this is for pre-commit testing. + # Since this is for ephemeral pre-commit, do not bother about + # cleaning up temp directory. + # + # Note that we use "touch" instead of "test -w" to check writability + # of $HOME/.ccache. This is because "test -w" documentation says + # that "test -w" checks for "w" permission, which is not the same + # as writability -- e.g., consider read-only filesystems. + # In practice, "test -w" does seem to check for actual writability, + # but "touch" is more robust. + ccache_opts+=("CCACHE_READONLY=true" "CCACHE_NOSTATS=true" + "CCACHE_TEMPDIR=$(mktemp -d)") + fi + + cat > "$bin/gcc" <<EOF +#!/bin/sh +${ccache_opts[@]} exec ccache ${rr[host_cc]} "\$@" +EOF + chmod +x "$bin/gcc" + cp "$bin/gcc" "$bin/cc" + + cat > "$bin/g++" <<EOF +#!/bin/sh +${ccache_opts[@]} exec ccache ${rr[host_c++]} "\$@" EOF + chmod +x "$bin/g++" + cp "$bin/g++" "$bin/c++" ) } @@ -183,7 +268,14 @@ prepare_abe () ( set -euf -o pipefail - clone_or_update_repo abe tested https://git-us.linaro.org/toolchain/abe.git > /dev/null + clone_or_update_repo abe ${rr[abe_branch]} ${rr[abe_repo]} > /dev/null + + # We use our modified version of GCC's comparison script + clone_or_update_repo gcc-compare-results master \ + https://git.linaro.org/toolchain/gcc-compare-results.git + + local workspace + workspace=$(pwd) cd abe @@ -192,19 +284,15 @@ prepare_abe () rm -rf "$(pwd)/bin" mkdir "$(pwd)/bin" - cat > "$(pwd)/bin/gcc" <<EOF -#!/bin/sh -exec ccache /usr/bin/gcc "\$@" -EOF - chmod +x "$(pwd)/bin/gcc" - cp "$(pwd)/bin/gcc" "$(pwd)/bin/cc" + setup_ccache "$(pwd)/bin" - cat > "$(pwd)/bin/g++" <<EOF + # Disable building documention. Apparently, this is one of + # the most popular ways. + cat > "$(pwd)/bin/makeinfo" <<EOF #!/bin/sh -exec ccache /usr/bin/g++ "\$@" +exec true EOF - chmod +x "$(pwd)/bin/g++" - cp "$(pwd)/bin/g++" "$(pwd)/bin/c++" + chmod +x "$(pwd)/bin/makeinfo" PATH=$(pwd)/bin:$PATH export PATH @@ -213,14 +301,89 @@ EOF ) } -# Build ABE component -# $1: Component -- ABE component to build. +# Create GNU toolchain xfail files +# $1: file to store flaky xfails to. +# $2: file to store baseline xfails to. +build_abe_check_xfails () +{ + ( + set -euf -o pipefail + local flaky_tests="$1" + local baseline_fails="$2" + + local sumfiles="$run_step_top_artifacts/sumfiles" + if [ -f "$sumfiles/flaky.xfail" ]; then + # Add newly-detected flaky tests to the xfails to be used in + # tcwg_gnu-build.sh:no_regression_p(). + # Strictly speaking, this is not necessary, since all tests + # detected as flaky in the current run will show up as PASSed + # test in the final merged .sum files. + echo "# New flaky tests" >> "$flaky_tests" + cat "$sumfiles/flaky.xfail" >> "$flaky_tests" + fi + + # Fetch flaky tests from base-artifacts history. + echo "# Known flaky tests" >> "$flaky_tests" + local history_flaky history_root="" + while read history_flaky; do + if [ "$history_root" = "" ]; then + history_root="$history_flaky" + continue + fi + + (echo; cat "$history_flaky") >> "$flaky_tests" + done < <(get_git_history 0 base-artifacts sumfiles/flaky.xfail) + rm -rf "$history_root" + + # Construct $baseline_fails from base-artifacts/sumfiles/. + # These and $flaky_tests are passed to ABE to speed-up test convergence + # and then to .sum comparison in tcwg_gnu-build.sh:no_regression_p(). + if [ -d base-artifacts/sumfiles ]; then + gcc-compare-results/contrib/testsuite-management/validate_failures.py \ + --build_dir=base-artifacts/sumfiles --produce_manifest \ + --manifest "$baseline_fails" + else + touch "$baseline_fails" + fi + ) +} + +# Build ABE component. Arguments: +# +# build_abe <component> [--build_patch <patch_branch>] [--check_patch <patch_branch>] [--] [ABE arguments]* +# +# Where: +# +# <component> ABE component to build. +# --build_patch <patch_branch> Branch with patch to apply before build. +# --check_patch <patch_branch> Branch with patch to apply before test. +# -- Separates arguments for build_abe from arguments for +# other components. +# +# Any argument not mentioned above is carried over to ABE. build_abe () { ( set -euf -o pipefail local component="$1" + shift + + if [ x"$component" = x"check_gdb" ]; then + # Limit GDB testsuites to single-thread parallelism. + # We've tried running GDB testsuites with 16-thread parallelism, + # but could not shake out flaky tests in the course of several weeks. + # Try stabilizing GDB testsuites with single-thread parallelism. + # If this doesn't work, we'll have to look into dejagnu. + local cpus + cpus=$(cat abe/host.conf | grep "^cpus=" | sed -e "s/^cpus=\(.*\)/\1/") + if [ "$cpus" -gt 1 ]; then + cp abe/host.conf abe/host.conf.orig + sed -i -e "s/^cpus=.*/cpus=1/" abe/host.conf + fi + elif [ -f abe/host.conf.orig ]; then + mv abe/host.conf.orig abe/host.conf + fi local project stage action check check=false @@ -284,6 +447,20 @@ build_abe () ;; esac + local build_patch="" + if [ $# -gt 0 ] && [ "$1" = "--build_patch" ]; then + build_patch="$2" + shift 2 + fi + + local check_patch="" + if [ $# -gt 0 ] && [ "$1" = "--check_patch" ]; then + check_patch="$2" + shift 2 + fi + + # Finished processing arguments for build_abe. Now look for arguments meant + # for various components. while [ $# -gt 0 ]; do if [ x"$1" = x"--" ]; then shift @@ -292,20 +469,83 @@ build_abe () shift done - local patch_branch="" - if [ $# -gt 0 ] && [ x"$1" = x"--patch" ]; then - patch_branch="$2" - shift 2 + local -a rerun_failed_tests=() + local -a send_results=() + if $check; then + # Clean testing results + rm -rf "$run_step_top_artifacts/sumfiles" \ + "$run_step_top_artifacts/00-sumfiles" + mkdir "$run_step_top_artifacts/sumfiles" \ + "$run_step_top_artifacts/00-sumfiles" + + local flaky_tests="$run_step_artifacts/flaky.xfail" + local baseline_fails="$run_step_artifacts/baseline.xfail" + build_abe_check_xfails "$flaky_tests" "$baseline_fails" + + rerun_failed_tests=("--rerun-failed-tests" + "--gcc-compare-results" "$PWD/gcc-compare-results" + "--flaky-failures" "$flaky_tests" + "--expected-failures" "$baseline_fails") + + if [ "${rr[update_baseline]}" = "onsuccess" ]; then + # If we have a chance to commit $new_flaky into base-artifacts.git + # then pretend it is 8 weeks into the future and ignore flaky + # entries that will be expired by then. This, effectively, gives + # us 8 weeks to re-detect/confirm flaky tests without + # no_regression_p() noticing anything. + # We will then set expiration date to "now+12 weeks" (see below) + # for entries in $new_flaky/"sumfiles/flaky.xfail". + local weeks_from_now + weeks_from_now=$(date -d "now+8 weeks" +%Y%m%d) + rerun_failed_tests+=("--failures-expiration-date" "$weeks_from_now") + elif [ "${rr[update_baseline]}" = "force" ]; then + # If we know that we will commit $new_flaky into base-artifacts.git, + # then ignore known flaky entries to try and detect as many + # flaky tests as possible. "Forced" history entries will remain + # in base-artifacts more-or-less forever, while "onsuccess" entries + # are eventually discarded to keep history size reasonable. + # For "update_baseline==init" we have this behavior naturally, + # since there is no history to pull flaky entries from. + echo "# Ignoring known flaky entries" > "$flaky_tests" + fi + + case "${rr[ci_project]}" in + # Don't send results for partial 'make check' + *_fast_check_*) ;; + *) + # Default recipient, overriden in round-robin-notify.sh + send_results=("--send-results-to" "christophe.lyon@linaro.org" + "--send-results-filter" "$(pwd)/abe/scripts/testresults2jenkins.sh") + ;; + esac fi if [ x"$project" = x"gcc" ]; then - # Don't build Go, which has a funky and unstable testsuite. # Stage1 ignores "--set languages=" option, so this applies only # to stage2 builds. - stage="$stage --set languages=c,c++,fortran,lto" - if [ $# != 0 ]; then - stage="$stage $*" - fi + case "${rr[ci_project]}" in + *embed*) + # Do not build fortran for bare-metal configurations. + stage="$stage --set languages=c,c++,lto" + ;; + *_mingw_*) + # FIXME: Only C is supported for aarch64-w64-mingw32. + stage="$stage --set languages=c" + ;; + *_native_*) + # Enable all languages in configuration that is slow anyway. + stage="$stage --set languages=all" + ;; + *) + # Build upstream-default languages (not abe-default languages). + stage="$stage --set languages=default" + ;; + esac + fi + + # Carry over any remaining arguments to ABE. + if [ $# != 0 ]; then + stage="$stage $*" fi action="--build $project" @@ -318,9 +558,20 @@ build_abe () local custom_abe_src_opt="" local git_dir="$project" local n_patches=0 + local patch_repo + + case "$project" in + binutils|gdb) + patch_repo="binutils-gdb" + ;; + *) + patch_repo="$project" + ;; + esac + case "$component" in - # Use our custom sources for everything, but kernel headers and dejagnu. - linux|dejagnu) ;; + # Use our custom sources for everything, but dejagnu. + dejagnu) ;; *) git_dir="$git_dir.git" custom_abe_src_opt="$project=http://git.l.o/$git_dir~master --disable update" @@ -328,12 +579,30 @@ build_abe () if ! $check; then clone_repo $project - if [ x"$patch_branch" != x"" ]; then - git -C $project fetch "https://git.linaro.org/toolchain/$project.git" "refs/heads/$patch_branch" + if [ x"$build_patch" != x"" ]; then + git -C $project fetch \ + "https://git.linaro.org/toolchain/$patch_repo.git" \ + "refs/heads/$build_patch" git -C $project cherry-pick FETCH_HEAD n_patches=1 fi + ( + cd $project + + # Avoid rebuilding of auto-generated C files. Rather than + # try to determine which are auto-generated, touch all of + # them. If a C file is not autogenerated, it does + # no harm to update its timestamp. + git ls-files -z '*.c' | xargs -r -0 touch + + # Touch GCC's auto-generated files to avoid + # non-determenistic behavior. + if [ -x ./contrib/gcc_update ]; then + ./contrib/gcc_update --touch + fi + ) + # Don't use ABE's repo clone functions and setup abe/snapshots/ # directory to have the right entries. local git_path @@ -341,14 +610,75 @@ build_abe () rm -rf $git_path $git_path~master ln -s "$(pwd)/$project" $git_path ln -s "$(pwd)/$project" $git_path~master + else + if [ x"$check_patch" != x"" ]; then + git -C $project fetch \ + "https://git.linaro.org/toolchain/$patch_repo.git" \ + "refs/heads/$check_patch" + git -C $project cherry-pick FETCH_HEAD + n_patches=1 + fi fi ;; esac - cd abe + # FIXME remove debug traces + set +f + stat -c "%Y %n" * abe/snapshots/$git_dir~master abe/snapshots/$git_dir~master/ || true + set -f + + + # In precommit testing, enable maintainer_mode so that we + # regenerate files as needed. Also update $PATH to include the + # right versions of autoconf and automake. + # ${rr[update_baseline]} == "ignore" is an approximate detection + # of precommit testing mode, since this can also be true while we + # are bisecting. When bisecting, we want to keep the sources + # exactly as they were committed, so we don't enable + # maintainer_mode in this case. + # + # FIXME binutils, gdb and gcc are not ready for automatic + # maintainer_mode. Disable for all projects for the moment. + local maintainer_mode=false + if [ "${rr[update_baseline]}" = "ignore" ] \ + && [ "${rr[mode]}" != "bisect" ] \ + && [ "$project" != "binutils" ] \ + && [ "$project" != "gcc" ] \ + && [ "$project" != "gdb" ] \ + && false; then + maintainer_mode=true + fi + + # FIXME Test only with fast_* projects for the moment (they have + # no precommit mode, so enable maintainer_mode in the "normal" + # jobs. + if [ "${rr[mode]}" != "bisect" ]; then + case "${rr[ci_project]}" in + *_fast_check_*) + maintainer_mode=true + ;; + esac + fi - local gnu_target - gnu_target=$(print_gnu_target ${rr[target]}) + if $maintainer_mode; then + stage="$stage --enable maintainer_mode" + # No need to export PATH, it is already exported by parent processes + PATH=/usr/local/automake-1.15.1/bin:/usr/local/autoconf-2.69/bin:$PATH + fi + + # Build docs in build-only jobs, disable them in "check" ones + case "${rr[ci_project]}" in + *_build) + # Remove the fake makeinfo we created in prepare_abe(), so + # that we can check that docs can be built. + rm -f "$(pwd)/abe/bin/makeinfo" + ;; + *) + stage="$stage --disable make_docs" + ;; + esac + + cd abe # Remove previous build directories and .stamp files. # (we rely on ccache for fast rebuilds) @@ -362,79 +692,250 @@ build_abe () PATH=$(pwd)/bin:$PATH export PATH - if [ x"$component" != x"stage2" ]; then - # TODO: Fix install_sysroot logic in ABE. - # ABE tries to install sysroot even for partial builds, e.g., - # with "--build binutils". Workaround by patching ABE. - sed -i -e "s/do_install_sysroot/:/" lib/control.sh - else - git checkout -- lib/control.sh - fi - - if true; then - # WORKAROUND abe patches being blocked on proper testing. - # Append, not overwrite runtestflags in abe.sh - sed -i -e 's/override_runtestflags=.*/override_runtestflags="$override_runtestflags $setting"/' abe.sh - fi - ccache -z local target_opt="" if [ x"${rr[target]}" != x"native" ]; then - target_opt="--target $gnu_target" - - # Disable TSan execution tests when using QEMU. - # QEMU can't handle TSan's shadow memory and it sends host machine - # into swap. - if $check && [ x"$project" = x"gcc" ]; then - assert_with_msg "ERROR: Testing is not using QEMU" \ - ! echo "$stage" | grep -q -e "--testcontainer" - sed -i \ - -e 's/set dg-do-what-default run/set dg-do-what-default link' \ - ../gcc/gcc/testsuite/lib/tsan-dg.exp - fi + target_opt="--target $(print_gnu_target ${rr[target]})" fi # Run "./abe.sh --build $project". - ./abe.sh \ - $action \ - $target_opt \ - --extraconfigdir config/master \ - $custom_abe_src_opt \ - $stage & + # shellcheck disable=SC2206 + local -a abe_cmd=( + ./abe.sh + $action + $target_opt + --extraconfigdir config/master + $custom_abe_src_opt + "${rerun_failed_tests[@]}" + "${send_results[@]}" + $stage) + TESTRESULTS_PREFIX=$run_step_artifacts/testresults- "${abe_cmd[@]}" & res=0 && wait $! || res=$? + # FIXME remove debug traces + set +f + stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true + find ../$project -newer builds/*/*/$project-*-configure.stamp || true + find ../$project -newer builds/*/*/$project-*-build.stamp || true + set -f + + # If a build without check failed, re-run it with --disable + # parallel so that errors are easier to extract from the logs. + if ! $check && [ $res -ne 0 ]; then + # ABE skips the configure step if $builddir/config.status + # exists. Remove this file when maintainer-mode is enabled, so + # that files are regenerated in case something went wrong + # during the parallel build. We cannot use ABE's --force + # because it also implies --enable-update. + if $maintainer_mode; then + rm -f builds/*/*/$project-*/config.status + fi + + abe_cmd+=(--disable parallel) + "${abe_cmd[@]}" | ts abe-debug-build: & + res=0 && wait $! || res=$? + + # FIXME remove debug traces + set +f + stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true + set -f + + if [ $res -eq 0 ]; then + # Parallel build failed, single-threaded one passed: + # consider this "OK" on aarch32, but a failure on other + # targets where we do not expect memory exhaustion. + if [ "$(getconf LONG_BIT)" = "32" ]; then + echo "WARNNING: Parallel build failed, single-threaded one passed." + echo "WARNNING: Considering this build as succesful (likely a transient memory exhaustion caused by a highly parallel build)" + else + if $maintainer_mode; then + echo "WARNNING: Parallel build failed, single-threaded one passed. This error is ignored when maintainer-mode is enabled." + else + echo "ERROR: Parallel build failed, single-threaded one passed." + res=1 + fi + fi + fi + fi + # Revert patches if applied. if [ -d ../$project ]; then - git -C ../$project reset --hard HEAD~$n_patches + # FIXME remove debug traces + set +f + stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true + git -C ../$project status + git -C ../$project diff HEAD~$n_patches + set -f + + git -C ../$project reset -q --hard HEAD~$n_patches + + # FIXME remove debug traces + set +f + stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true + find ../$project -newer builds/*/*/$project-*-configure.stamp || true + find ../$project -newer builds/*/*/$project-*-build.stamp || true + set -f fi - # If abe failed to build component, return exit status. - if [ x"$res" != x"0" ]; then - return $res - fi ccache -s + # Save logs generated in the current step to artifacts. + # Note that the logs generated in the previous steps will have .log.xz + # extension, and would not match in "find". + local log + while IFS= read -r -d '' log; do + rm -f "$log.xz" + ( + xz "$log" + cp "$log.xz" "$run_step_artifacts/" + ) & + done < <(find builds/ \( -name "make-*.log" -o -name "check-*.log" \) \ + -print0) + + # FIXME remove debug traces + set +f + stat -c "%Y %n" * snapshots/$git_dir~master snapshots/$git_dir~master/ || true + set -f + if $check; then - local sum log - rm -rf ${rr[top_artifacts]}/sumfiles - mkdir -p ${rr[top_artifacts]}/sumfiles + local sum sumfiles while IFS= read -r -d '' sum; do - cp "$sum" ${rr[top_artifacts]}/sumfiles/ - log="${sum%.sum}.log" + case "$res:$sum" in + 0:*".sum") + # Only store sum files in definitive directory if abe + # succeeded. + sumfiles="$run_step_top_artifacts/sumfiles" + ;; + *) + # Store *.sum.N files and .log.xz files in 00-sumfiles/, + # so that these non-essential big files are eventually + # removed from base-artifacts.git history by + # "git filter-repo" in round-robin-baseline.sh's + # trim_base_artifacts(). + sumfiles="$run_step_top_artifacts/00-sumfiles" + ;; + esac + + # Remove WORKSPACE prefix instead of making a plain copy + sed "s|Running .*/snapshots/|Running |" \ + < "$sum" > "$sumfiles/$(basename "$sum")" + + log="${sum/.sum/.log}" # Testsuite logs grow 50-400MB in size, so compress them to save - # disk space on ci.linaro.org. + # disk space on ci.linaro.org and in base-artifacts.git. # Delete previous XZ'ed log, which can occur when we are bisecting # QEMU and not rebuilding compiler (therefore not cleaning compiler # build/test directory). + # Process logs in parallel; "wait" below waits for all to + # finish. rm -f "$log.xz" ( xz "$log" - cp "$log.xz" ${rr[top_artifacts]}/sumfiles/ + cp "$log.xz" "$run_step_top_artifacts/00-sumfiles/" ) & - done < <(find builds/ -name "*.sum" -print0) - # Wait for logs to compress - wait + done < <(find builds/ \( -name '*.sum' -o -name '*.sum.[0-9]*' \) \ + -print0) + + # ABE re-wrote the $flaky_tests file to contain entries only + # for the new flaky tests detected in this run. + if [ -s "$flaky_tests" ]; then + sumfiles="$run_step_top_artifacts/sumfiles" + if [ "$res" != 0 ]; then + sumfiles="$run_step_top_artifacts/00-sumfiles" + fi + + # Mark new flaky entries to expire in 12 weeks. + # The upside of expiration is that we will have an up-to-date list + # of flaky tests, which we can address. + # The downside is that we will spend a bit more CPU cycles + # to re-detect flaky tests. + local expire + expire=$(date -d "now+12 weeks" +%Y%m%d) + # A lot of thought went into the following nugget: + expire="expire=$expire" + sed -i -e "s#^flaky | #flaky,$expire | #" "$flaky_tests" + + # Move flaky fails to the sumfiles so that they will be + # fetched by next run's get_git_history(). + echo "# From ${BUILD_URL-$(pwd)}:" > "$sumfiles/flaky.xfail" + cat "$flaky_tests" >> "$sumfiles/flaky.xfail" + fi + + if [ $res -eq 0 ] \ + && [ -f $run_step_artifacts/testresults-mail-body.txt ]; then + ( + # Move testresults files, so that it's easier to find them + # later when we want to send them via email. + rm -rf $run_step_top_artifacts/testresults + mkdir $run_step_top_artifacts/testresults + mv $run_step_artifacts/testresults-mail-recipients.txt \ + $run_step_artifacts/testresults-mail-subject.txt \ + $run_step_top_artifacts/testresults/ + # Add a pointer to the build origin, for easier tracking + echo "# From ${BUILD_URL-$(pwd)}:" \ + > $run_step_top_artifacts/testresults/testresults-mail-body.txt + cat $run_step_artifacts/testresults-mail-body.txt \ + >> $run_step_top_artifacts/testresults/testresults-mail-body.txt + rm $run_step_artifacts/testresults-mail-body.txt + ) & + if ! wait $!; then + echo "christophe.lyon@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "${BUILD_URL-}\nERROR: failed to process testresults" \ + >> artifacts/jenkins/error-mail-body.txt + fi + fi + fi + + # Wait for logs to compress. + wait + + return $res + ) +} + +# Print sysroot path under ABE's build tree. +print_abe_sysroot () +{ + ( + set -euf -o pipefail + + local host target sysroot + host=$(print_gnu_target native) + target=$(print_gnu_target ${rr[target]}) + + # FIXME: This is a copy of ugly code from abe/lib/globals.sh: + # init_globals_and_PATH(). Don't ask my why we have different sysroot + # paths for cross and native cases. + sysroot="$(pwd)/abe/builds/destdir/$host" + if [ "$host" != "$target" ]; then + sysroot="$sysroot/$target" + fi + if [ "${rr[target]}" != "woa64" ]; then + # FIXME: WoA toolchain uses mingw CRT, and this is a quick + # fix to make it build. At the moment ABE pretends to use + # newlib library when building mingw GCC. See settings of + # stage2_flags in abe/config/gcc.conf. + sysroot="$sysroot/libc" fi + + echo "$sysroot" + ) +} + +# If we bisect a regression between different major versions of Glibc, +# then we might get a mixed sysroot with several versions of ld-M.N.so and +# other binaries installed side-by-side. Such a sysroot will break +# benchmarking, which requires a single ld-*.so binary to be present. +# Similarly, weird problems can occur if we are bisecting linux +# and re-installing kernel headers one on top another. +# Forcefully delete sysroot before building C library or linux headers. +clean_sysroot () +{ + ( + set -euf -o pipefail + + rm -rf "$(print_abe_sysroot)" ) } @@ -444,28 +945,26 @@ build_llvm () ( set -euf -o pipefail - local use_abe=${1-false} + local projects="${1-clang;lld}" + local extra_targets="${2-}" + local metric_id="${3-}" clone_repo llvm - + if [ x"$metric_id" = x"num_vect_loops" ]; then + wget -O llvm-vect-metric.diff "https://git.linaro.org/toolchain/jenkins-scripts.git/plain/downstream_patches/llvm-vect-metric.diff" + git -C llvm apply "$(pwd)/llvm-vect-metric.diff" + fi sanity_check_pwd + local workspace + workspace=$(pwd) + # Setup ccache and ninja wrappers. # shellcheck disable=SC2115 rm -rf "$(pwd)/bin" mkdir "$(pwd)/bin" - cat > "$(pwd)/bin/cc" <<EOF -#!/bin/sh -exec ccache /usr/bin/gcc "\$@" -EOF - chmod +x "$(pwd)/bin/cc" - - cat > "$(pwd)/bin/c++" <<EOF -#!/bin/sh -exec ccache /usr/bin/g++ "\$@" -EOF - chmod +x "$(pwd)/bin/c++" + setup_ccache "$(pwd)/bin" if [ -f /usr/local/bin/ninja.bin ]; then # Use ninja configuration from llvm buildbots to avoid running out of RAM. @@ -479,20 +978,19 @@ EOF PATH=$(pwd)/bin:$PATH export PATH - local binutils_incdir - # Freshen up build and install directories. We rely on ccache for fast rebuilds. - if $use_abe; then - rsync -a --del abe/builds/destdir/x86_64-pc-linux-gnu/ llvm-install/ - binutils_incdir=$(pwd)/binutils/include - else - rm -rf llvm-install - binutils_incdir=/usr/include - fi - rm -rf llvm-build + # Freshen up build and install directories. We rely on ccache for fast + # rebuilds. + rm -rf llvm-build llvm-install mkdir -p llvm-build cd llvm-build - cmake -G Ninja ../llvm/llvm "-DLLVM_ENABLE_PROJECTS=clang;lld" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install "-DLLVM_TARGETS_TO_BUILD=$(print_llvm_target ${rr[target]})" -DLLVM_BINUTILS_INCDIR=$binutils_incdir + local llvm_targets + llvm_targets="$(print_llvm_target ${rr[target]})${extra_targets}" + + cmake -G Ninja ../llvm/llvm "-DLLVM_ENABLE_PROJECTS=$projects" \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True \ + -DCMAKE_INSTALL_PREFIX=../llvm-install \ + "-DLLVM_TARGETS_TO_BUILD=$llvm_targets" -DCLANG_DEFAULT_LINKER=lld ccache -z ninja ninja install @@ -518,273 +1016,379 @@ no_build_regression_p () fi local build_result_ref build_result_new - build_result_ref=$(grep -v "^#" $ref_artifacts/results | tail -n1) - build_result_new=$(grep -v "^#" $new_artifacts/results | tail -n1) - - if [ $build_result_new -lt $build_result_ref ]; then - # In log scan for errors below - # - sed -e 's/"[^"]*"//g' -- removes quoted "error: app diagnostics" strings - # - " error:" detects compiler errors from GCC and Clang (including GCC ICEs) - # - "^ERROR:" detects linker errors - # - ": undefined reference" detects missing symbols during linking - # - "] Error " detects GNU make errors - # Then grep for "grep" to exclude other uses of this search. - cat > $run_step_artifacts/results.regressions <<EOF + build_result_ref=$(grep -v -E "^#|^$" $ref_artifacts/results | tail -n1) + build_result_new=$(grep -v -E "^#|^$" $new_artifacts/results | tail -n1) + + if [ $build_result_new -ge $build_result_ref ]; then + return 0 + fi + + # If we are observing a regression with a negative score, consider results as invalid. + # Setting EXTERNAL_FAIL will avoid any bisect, and the run will be marked as UNSTABLE. + # A side-effect of this is lack of reduction to component when a failure to build + # one of the components will just stop the job in UNSTABLE state and not attempt + # to update other components. + if [ $build_result_new -lt 0 ]; then + return $EXTERNAL_FAIL + fi + + local last_log + last_log=$(find $new_artifacts/ -name console.log.xz | sort -g | tail -n1) + # In log scan for errors below + # - sed -e 's/"[^"]*"//g' -- removes quoted "error: app diagnostics" strings + # - " error:" detects compiler errors from GCC and Clang (including GCC ICEs) + # - "^ERROR:" detects linker errors + # - ": undefined reference" detects missing symbols during linking + # - "] Error " detects GNU make errors + # Then grep for "grep" to exclude other uses of this search. + # Do this twice, once with the abe-debug-build: prefix (where we + # disabled build parallelism to make debug easier), once without + # this prefix if for some reason the failure occurred only with + # parallelism enabled. + local debug_log + debug_log=$(mktemp) + + xzcat $last_log | \ + grep abe-debug-build: | \ + sed -e 's/abe-debug-build: //' | \ + sed -e 's/"[^"]*"//g' | \ + grep " error:\|^ERROR:\|: undefined reference\|\] Error " | \ + grep -v "grep" | \ + head | \ + sed -e "s/^/# /" > $debug_log + + cat > $run_step_artifacts/results.regressions <<EOF # First few build errors in logs: -$(cat $new_artifacts/console.log | sed -e 's/"[^"]*"//g' | grep " error:\|^ERROR:\|: undefined reference\|\] Error " | grep -v "grep" | head | sed -e "s/^/# /") EOF - return 1 + + if [ -s $debug_log ]; then + cat $debug_log >> $run_step_artifacts/results.regressions + else + xzcat $last_log | \ + grep -v abe-debug-build: | \ + sed -e 's/"[^"]*"//g' | \ + grep " error:\|^ERROR:\|: undefined reference\|\] Error " | \ + grep -v "grep" | \ + head | \ + sed -e "s/^/# /" >> $run_step_artifacts/results.regressions fi - return 0 + rm -f $debug_log + + return 1 ) } -# Check if current build regressed compared to the baseline -# (unless ${rr[update_baseline]} is set to "reset"). -check_regression () +# Generate trigger-build-* and trigger-bisect files to reduce the regression. +# $1: Directory for trigger-* files. +# $2: Score of the build. +create_trigger_files () { ( set -euf -o pipefail + local trigger_dest="$1" + local score="$2" + + local -a changed_components + IFS=" " read -r -a changed_components <<< "$(print_changed_components)" + + if [ ${#changed_components[@]} -gt 1 ] \ + || { [ x"${rr[mode]}" = x"bisect" ] \ + && ! [ "$score" -lt 0 ] 2>/dev/null; }; then + # If we have several changed components, then trigger individual builds + # for these components to narrow down the problem to a single component. + # Also generate trigger-build-* files as a favor to + # round-robin-bisect.sh script to distinguish "bad" from "skip" builds + # and to provide templates for triggering follow-up builds. + # Note that we can't use "[ "$score" -ge 0 ]" condition above and below + # because it will give wrong answer for non-numeric scores like "all" + # and "boot". "-lt" comparison, however, will produce correct answer, + # albeit, with an occasional harmless error message. + local -a update_components + + while read -a update_components; do + local c update_components2 + update_components2=$(echo "${update_components[@]}" | tr ' ' '-') + + # find the list of components_to_update + local -a components_to_update=() + for c in ${rr[components]}; do + if echo "${update_components[@]}" | tr ' ' '\n' \ + | grep "^$c\$" >/dev/null; then + assert_with_msg "Should never happen for precommit builds" \ + [ "${rr[${c}_git]}" != "HEAD" ] + components_to_update+=("${c}") + fi + done - local score - score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1) - - if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then - echo "ERROR: We have encountered some infrastructure problem (e.g.," - echo " benchmarking boards are offline), andso we can't finish" - echo " the build." - # Exit now and don't update baseline artifacts. - # By not creating trigger-build-* files, we signal - # round-robin-bisect.sh to skip this build/revision. - exit $EXTERNAL_FAIL - fi - - if [ x"${rr[update_baseline]}" = x"rebase" ]; then - # No-one ever used "rebase" mode. It's now obsoleted by custom ci_projects. - local base_artifacts_head base_artifacts_tail - base_artifacts_head=$(git -C base-artifacts rev-parse HEAD) - base_artifacts_tail=$(git -C base-artifacts rev-list --max-parents=0 HEAD) - git -C base-artifacts reset --hard $base_artifacts_tail - - ${rr[no_regression_p]} ${rr[top_artifacts]} base-artifacts & - if wait $!; then - echo "Current results are no better then the all-time-best results. \ -No reason to rebase base-artifacts." - false + # print components_to_update to the trigger file + if [ "${rr[dynamic_components_list]+abc}" ]; then + echo "dynamic_components_list=${components_to_update[*]}" + else + for c in "${components_to_update[@]}"; do + echo "${c}_git=${rr[${c}_git]}" + done + fi > $trigger_dest/trigger-build-$update_components2 + + # Add update_baseline setting to the parameters, so that + # "ignore" builds will trigger "ignore" reduction builds. + # Do not set update_baseline for bisect builds, since these + # trigger-build-* files will then be used as templates to trigger + # last_good and first_bad builds. + if [ x"${rr[mode]}" != x"bisect" ]; then + echo "update_baseline=${rr[update_baseline]}" \ + >> $trigger_dest/trigger-build-$update_components2 + fi + done < <(${rr[breakup_changed_components]}) + elif [ ${#changed_components[@]} = 1 ] \ + && ! [ "$score" -lt 0 ] 2>/dev/null; then + local single_component="${changed_components[0]}" + + # Trigger bisect failures in a single changed component in all steps + # with a positive result. If $score is less-than 0, then + # the regression is not very interesting, so don't bisect. + + # Rather than the current git commit sha1, use the original + # specification which can be a branch name: in case the + # regression has already been fxed in the branch, we won't + # bother running a useless bisection. + local bad_git + bad_git=$(get_current_manifest "{rr[${single_component}_git]}") + + cat > $trigger_dest/trigger-bisect <<EOF +current_project=$single_component +bad_git=$bad_git +EOF + if [ -f $run_step_artifacts/extra-bisect-params ]; then + cat $run_step_artifacts/extra-bisect-params >> $trigger_dest/trigger-bisect fi + fi + ) +} - git -C base-artifacts reset --hard $base_artifacts_head - else - # Generate comparison artifacts for update, reset, init and push modes. - ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} & - if wait $!; then - # All good, no regression - return - fi +# Make sure manifest has all up-to-date rr[] fields that other +# round-robin-*.sh scripts require. This is an easy way to update manifests +# saved in base-artifacts.git to the latest format by rewriting result history. +finalize_manifest () +{ + ( + set -euf -o pipefail - if [ -f $run_step_artifacts/results.regressions ]; then - # Add regression info generated by no_regression_p to top-level - # results file. - cat $run_step_artifacts/results.regressions \ - >> ${rr[top_artifacts]}/results + echo "# Saving rr[] in the manifest" | manifest_out + local field value + while read -r field; do + value=$(get_current_manifest "{rr[$field]-}") + if [ "$value" = "${rr[$field]}" ]; then + continue fi - # We've got a regression. Generate trigger-* files. - local single_component - single_component=$(print_single_updated_component) - local trigger_dest - - if [ x"${rr[update_baseline]}" = x"update" ]; then - # When "updating" baseline place trigger-* files at top level - # where jenkins expects them -- and trigger the followup builds. - trigger_dest="${rr[top_artifacts]}" - else - # We don't want to trigger follow up builds when "pushing" - # baseline. So, for the record, place trigger-* files in - # the step's artifacts directory. - trigger_dest="$run_step_artifacts" + if [ "$value" != "" ]; then + echo "# WARNING: overriding previous rr[$field]=$value" \ + | manifest_out fi - # 1. If $score is less-than 0, then the regression is not very - # interesting, so reduce down to component, but don't bisect. This - # allows non-broken components to advance their revisions. - # 2. Otherwise, trigger builds with narrowed-down list of updated - # components -- by generating trigger-build-* files. Also generate - # these files as a favor to round-robin-bisect.sh script. - # 3. Otherwise, we have narrowed-down regression to a single component, - # so trigger bisect build for this component -- by generating - # trigger-bisect file. Note that the negative "! [ $score -lt 0 ]" - # condition is to handle non-numeric scores like "all" and "boot". - if [ $score -lt 0 ] 2>/dev/null && [ x"${rr[mode]}" = x"bisect" ]; then - # Don't bisect uninteresting regressions. - : - elif [ x"$single_component" = x"" ] || [ x"${rr[mode]}" = x"bisect" ]; then - local -a update_components - - while read -a update_components; do - local c update_components2 - update_components2=$(echo "${update_components[@]}" | tr ' ' '-') - - for c in ${rr[components]}; do - if echo "${update_components[@]}" | tr ' ' '\n' | grep -q "^$c\$"; then - echo "${c}_git=${rr[${c}_git]}" - else - echo "${c}_git=baseline" - fi - done > $trigger_dest/trigger-build-$update_components2 - done < <(${rr[breakup_updated_components]}) - elif ! [ $score -lt 0 ] 2>/dev/null; then - # Bisect failures in all steps after "-1" step. - local cur_rev - cur_rev=$(git -C $single_component rev-parse HEAD) - - cat > $trigger_dest/trigger-bisect <<EOF -current_project=$single_component -bad_git=${rr[${single_component}_git]%#*}#$cur_rev + cat <<EOF | manifest_out +rr[$field]="${rr[$field]}" EOF - if [ -f $run_step_artifacts/extra-bisect-params ]; then - cat $run_step_artifacts/extra-bisect-params >> $trigger_dest/trigger-bisect - fi - fi - - if [ x"${rr[update_baseline]}" = x"update" ]; then - echo "Detected a regression in \"update\" mode!" - false - else - # Compare results to generate logs and other artifacts, - # but we don't really care about regressions. - : - fi - fi + done < <(IFS=$'\n'; echo "${!rr[*]}" | sort | grep -v "^top_artifacts\$") ) } -# Commit current result and artifacts to the baseline repository -update_baseline () + +# Check if current build regressed compared to the baseline. +# +# As inputs we have: +# $score -- last line of artifacts/results +# $res -- exit code of no_regression_p() +# +# Additionally, ${rr[update_baseline]} and ${rr[mode]} affect how the rest +# of the build will proceed -- via create_trigger_files(). +# +# Decision matrix: +# +# 1. score >= 0 && res == 0: return 0 +# OK; new results good for baseline; +# update baseline if != ignore (e.g., not bisect test or precommit) +# - round-robin-notify.sh compares results in artifacts/ vs +# base-artifacts#HEAD^ +# don't update baseline if == ignore (e.g., bisect test or precommit) +# - round-robin-notify.sh compares results in artifacts/ vs +# base-artifacts#HEAD +# send precommit feedback if precommit testing is active; +# (see [R] below) send regression report if notify==onregression +# push baseline if != ignore (e.g., not precommit build); +# trigger pre-commit testing if SCM build; +# +# 2. score >= 0 && res == E: return E +# sporadic fail; wait for next build; don't trigger anything; +# don't update baseline even with force or init; +# Can happen in, e.g., tcwg_native_build if git server fails while +# cloning repo for one of the components; +# +# 3. score >= 0 && res != {0,E}: return I (or 0 if update_baseline==force) +# regression (or expected regression if update_baseline==force); +# if update_baseline == onsuccess -- trigger reduce or bisect; +# [R] if update_baseline == force -- succeed, return 0, treat as (1). +# if update_baseline == ignore -- nothing +# - if mode==bisect, then create trigger-* files as favor for +# round-robin-bisect.sh script. +# - if pre-commit testing is active, then send notifications +# similar to "update_baseline==force". See [P] below. +# +# 4. score < 0 && res == 0: return E +# should not happen; baseline is bad and current build is no better; +# don't update the baseline or reduce or bisect; send error-mail +# +# 5. score < 0 && res == E: return E +# sporadic fail; wait for next build; don't trigger anything; +# don't update baseline even with force or init. +# +# 6. score < 0 && res != {0,E}: return I +# uninteresting fail; trigger reduce, but no bisect +# don't update baseline even with force or init. +# This happens when the build failed before reaching "interesting" +# phase -- e.g., glibc build failed in a configuration that checks +# gcc testsuite results in a cross-toolchain. In this case we want +# to continue testing and updating baseline when advancing binutils and +# gcc sources, while waiting for glibc build to get fixed. +# For this reason we often have matching tcwg_*_build ci_projects for +# most tcwg_*_check ci_projects. Non-interesting (for tcwg_*_check) failures +# are reduced, bisected and reported by tcwg_*_build projects. +# +# 7. score == -E: return E +# special case indicating a sporadic failure; same as (5) but without +# calling no_regression_p() to get "res". +# +# "E" -- EXTERNAL_FAIL; "I" -- INTERNAL_FAIL +# +# Notes: +# - If score < 0 then we never update baseline -- even with force or init. +# This allows us to always have a reasonable "interesting" baseline, +# which is necessary for pre-commit testing. E.g., we don't want to +# pre-commit test upstream patches against a baseline that doesn't build. +# +# - If we return E -- exit, do nothing, wait for the next timed trigger. +# Hopefully by that time infra fixes itself. +# In the .yaml files we have a choice on how to implement this: +# a. Mark the build as FAILURE, which will prevent any subsequent steps +# from running. Pros: simple, doesn't require conditional-steps; +# cons: the build is shown as "red" failure. +# b. Mark the build as UNSTABLE, and then condition all subsequent steps +# to run only for current-status==SUCCESS, which is what we are doing +# now. Pros: the build is shown as "yellow"; cons: all subsequent +# steps need to be wrapped in conditional-step. +# +# - [P] It's not clear how to send notifications for failed pre-commit +# builds. At the moment we run trigger-followup-builds for failed +# builds and exit, thus avoiding notify-and-push step for failed builds. +# It seems we need to +# a. Split notify-and-push into "notify" and "push" +# b. Move "notify" before trigger-followup-builds and leave "push" +# after trigger-followup-builds. +# c. We need to make sure "notify" does not send anything when it is +# running after a failed build. +# d. Trigger of pre-commit testing needs to happen only after "push", +# so we know that we have a good recent baseline. +# +# - If we return E during precommit testing, the check will be in "pending" +# state, and testing of the patch will be retriggered on the next round. +# +# - If we return N != {0, I, E}, then the exit was due to script error, +# so send error-mail and follow case of returning "E". +# +# - If we returned "0", then build should be marked as SUCCESS. +# +# - If we returned "I", then build should be marked FAILURE +# +# - If we returned "E" or anything else, then build should be marked +# as UNSTABLE. +check_regression () { ( set -euf -o pipefail - local amend="" - local rebase_head rebase_tail - - if [ x"${rr[update_baseline]}" = x"init" ]; then - amend="--amend" - elif [ x"${rr[update_baseline]}" = x"push" ]; then - : - elif [ x"${rr[update_baseline]}" = x"rebase" ]; then - rebase_head=$(git -C base-artifacts rev-parse HEAD) - rebase_tail=$(git -C base-artifacts rev-list --max-parents=0 HEAD) - git -C base-artifacts reset --hard $rebase_tail - amend="--amend" - else - # ${rr[update_baseline]} == update, reset - local prev_head="" - - # We discard baseline entries for results worse or same than - # the current one, but keep entries for results that are better - # (so that we have a record of pending regressions). - while true; do - ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} & - if ! wait $!; then - break - fi + local score + score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1) - prev_head="" - if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then - # For every regression we want to keep artifacts for the first-bad - # build, so reset to the most relevant regression (marked by reset-baseline). - if [ -f base-artifacts/reset-baseline ] \ - && [ x"${rr[update_baseline]}" = x"update" ]; then - prev_head=$(git -C base-artifacts rev-parse HEAD) - fi - git -C base-artifacts reset --hard HEAD^ - else - # We got to the beginning of git history, so amend the current - # commit. The initial state of baseline is "empty" branch, - # which we treat as worst possible in ${rr[no_regression_p]}(). - amend="--amend" - break - fi - done + if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then + echo "ERROR: We have encountered some infrastructure problem (e.g.," + echo " benchmarking boards are offline), and we can't finish" + echo " the build." - if [ x"$prev_head" != x"" ]; then - git -C base-artifacts reset --hard $prev_head - fi + # Exit now and don't update baseline artifacts. + # By not creating trigger-build-* files, we signal + # round-robin-bisect.sh to skip this build/revision. + return $EXTERNAL_FAIL fi - # Rsync current artifacts. Make sure to use -I rsync option since - # quite often size and timestamp on artifacts/results will be the same - # as on base-artifacts/results due to "git reset --hard HEAD^" below. - # This caused rsync's "quick check" heuristic to skip "results" file. - # !!! From this point on, logs and other artifacts won't be included - # in base-artifacts.git repo (though they will be uploaded to jenkins). - rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ - - local rev_count - if [ x"$amend" = x"" ]; then - rev_count=$(git -C base-artifacts rev-list --count HEAD) - else - rev_count="0" - fi + local res - local msg_title="$rev_count: ${rr[update_baseline]}" + # Generate comparison artifacts for update, reset, init and push modes. + ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} & + res=0 && wait $! || res=$? - if [ x"${rr[update_baseline]}" = x"reset" ]; then - # Create a marker for builds that reset baselines (these are builds - # for bisected regressions). - touch base-artifacts/reset-baseline + # Move extra artifacts that no_regression_p generated to $top_artifacts. + if [ -d $run_step_artifacts/top-artifacts ]; then + rsync -a $run_step_artifacts/top-artifacts/ ${rr[top_artifacts]}/ + rm -rf $run_step_artifacts/top-artifacts fi - local single_component - single_component=$(print_single_updated_component) - if [ x"$single_component" != x"" ]; then - local single_rev - single_rev=$(git -C $single_component rev-parse HEAD) - msg_title="$msg_title: $single_component-$single_rev" - else - msg_title="$msg_title: $(print_updated_components "-")" + if [ -f $run_step_artifacts/results.regressions ]; then + # Add regression info generated by no_regression_p to top-level + # results file. + cat $run_step_artifacts/results.regressions \ + >> ${rr[top_artifacts]}/results fi - msg_title="$msg_title: $(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)" - - git -C base-artifacts add . - git -C base-artifacts commit $amend -m "$msg_title -BUILD_URL: ${BUILD_URL-$(pwd)} - -results: -$(cat ${rr[top_artifacts]}/results)" - - if [ x"${rr[update_baseline]}" = x"rebase" ]; then - for rebase_tail in $(git -C base-artifacts rev-list --reverse $rebase_head); do - git -C base-artifacts checkout $rebase_tail -- . - git -C base-artifacts commit -C $rebase_tail - done + rr[no_regression_result]="$res" + finalize_manifest + + if [ $res = 0 ]; then + if [ "$score" -lt 0 ] 2>/dev/null; then + # Case (4) in the comment above. + if [ -d ${rr[top_artifacts]}/jenkins ]; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "${BUILD_URL-}\nERROR: case (4) in check_regression" \ + >> artifacts/jenkins/error-mail-body.txt + fi + return $EXTERNAL_FAIL + else + # All good, no regression + return 0 + fi + elif [ $res = $EXTERNAL_FAIL ]; then + # Comparison failed to produce a meaningful result + return $EXTERNAL_FAIL fi - ) -} -# Push to baseline branches and to base-artifacts repo. -push_baseline () -{ - ( - set -euf -o pipefail + assert_with_msg "no_regression_p should succeed in init baseline mode" \ + [ x"${rr[update_baseline]}" != x"init" ] + + # We've got a regression. Generate trigger-* files. + local trigger_dest + if [ "${rr[update_baseline]}" = "onsuccess" ] \ + || [ "${rr[mode]}" = "bisect" ]; then + # We are seeing a failure, so instead of updating baseline start + # reducing/bisecting the failure. Create trigger-* files at top level + # where jenkins expects them -- and trigger the followup builds. + trigger_dest="${rr[top_artifacts]}" + else + # We don't want to trigger follow up builds when forcing + # the baseline. So, for the record, place trigger-* files in + # the step's artifacts directory. + trigger_dest="$run_step_artifacts" + fi - git_init_linaro_local_remote base-artifacts baseline false - git_push base-artifacts baseline ${rr[baseline_branch]} + create_trigger_files "$trigger_dest" "$score" - if [ x"${rr[update_baseline]}" = x"rebase" ]; then - return + if [ "$score" -lt "0" ]; then + # Case (6) above. + return $INTERNAL_FAIL + elif [ "${rr[update_baseline]}" = "force" ]; then + return 0 fi - local url - local c - for c in $(print_updated_components); do - # Clone (but don't checkout) always-present "empty" branch of - # the baseline repo. This initializes read/write "baseline" remote. - url=$(print_baseline_repo "$c" false) - git_set_remote "$c" baseline "$url" - git_push $c baseline ${rr[baseline_branch]} - done + echo "Detected a regression!" + return $INTERNAL_FAIL ) } |