#!/bin/bash set -ef -o pipefail scripts=$(dirname $0) # shellcheck source=jenkins-helpers.sh . $scripts/jenkins-helpers.sh declare -A rr # Process bisect-only args convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" obligatory_variables bad_git build_script current_project declare bad_git build_script current_project # Relative artifacts are used for generation of manifests and reproduction # instructions. rel_artifacts="${rel_artifacts-artifacts}" artifacts=$(pwd)/$rel_artifacts fresh_dir $artifacts \ "$artifacts/manifest.sh" \ "$artifacts/build-parameters/manifest.sh" \ "$artifacts/jenkins/*" BUILD_URL="${BUILD_URL:-$(pwd)}" replay_log="${replay_log-}" reproduce_bisect="${reproduce_bisect:-false}" # Process build args and record them in build-parameters.sh convert_args_to_variables ^^ $reproduce_bisect %%build_parameters $artifacts/build-parameters "$@" $reproduce_bisect || manifest_pop # Account for "^^ false %%foo bar" options SHIFT_CONVERTED_ARGS=$(($SHIFT_CONVERTED_ARGS-4)) shift "$SHIFT_CONVERTED_ARGS" obligatory_variables build_parameters rr[ci_project] rr[ci_config] declare build_parameters # Process build args and record them in build-parameters.sh convert_args_to_variables ^^ $reproduce_bisect %%baseline_parameters $artifacts/baseline-parameters "$@" $reproduce_bisect || manifest_pop # Account for "^^ false %%foo bar" options SHIFT_CONVERTED_ARGS=$(($SHIFT_CONVERTED_ARGS-4)) shift "$SHIFT_CONVERTED_ARGS" obligatory_variables baseline_parameters declare baseline_parameters verbose="${verbose-true}" set -u if $verbose; then set -x; fi mkdir -p $artifacts/jenkins touch $artifacts/jenkins/build-name trap print_traceback EXIT # Exit with success exit_0 () { # Cleanup bisect/ directory, which has a full [unneeded] copy of the build. chmod -R +rwx bisect/ rm -rf bisect/ trap "" EXIT exit 0 } current_project_dir=$(get_component_dir $current_project) bad_url="${bad_git%#*}" bad_branch="${bad_git#*#}" rebase_workaround=false rebase_workaround_opts=() case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in tcwg_kernel/*-next-*:linux) # Workaround linux-next/master rebasing on top of linux-next/stable. # Search for regressions between linux-next:stable and # linux-next:master. echo "Rebase workaround: forcing linux baseline to linux-next:stable" rebase_workaround=true rebase_workaround_opts+=("==rr[linux_git]" "$bad_url#stable") ;; esac # Build baseline that we are going to re-use to speed-up bisection. # (This also confirms that infrastructure is OK.) echo "Testing baseline revision (expecting success)" $build_script \ ^^ $reproduce_bisect \ %%rr[top_artifacts] "$rel_artifacts/build-baseline" \ @@ $build_parameters/manifest.sh \ @@ $baseline_parameters/manifest.sh \ ==rr[mode] build \ ==rr[update_baseline] force \ --verbose "$verbose" \ "${rebase_workaround_opts[@]}" # Establish results in build-baseline as the baseline to compare test builds # against. $scripts/round-robin-baseline.sh \ @@rr[top_artifacts] "$rel_artifacts/build-baseline" \ __base_artifacts base-artifacts baseline_rev="${baseline_rev-$(git -C $current_project_dir rev-parse HEAD)}" cat </dev/null; then git -C $current_project_dir bisect good $baseline_rev fi # Bisect script. # # With this script we find the first commit that has regressed compared # to baseline, but not, necessarily, the commit that caused regression in # $bad_rev. Consider the scenario: # - rev_10 produced good result "2000" -- this is current baseline # - rev_20 completely broke the build (say, result "10") # - rev_22 fixed the build # - rev_30 regressed the build to result "1000" -- this is the regression we # detected vs "2000" baseline. # # The script will identify rev_20 as the first failing commit, which will # cause the baseline to be reset to rev_20 with metric "10". When we then # rebuild master (at rev_30) we will see a /progression/ from "10" to "1000", # thus missing the regression of "2000" to "1000". # # To catch the "2000" to "1000" regression someone would need to manually # trigger bisect between rev_22 and rev_30. # # We reduce the impact of the above by assigning negative values to result # metric for builds that look very broken, and do not bisect regressions into # the "negative" side. cat > $artifacts/test.sh </dev/null; then exit 1 elif git -C $current_project_dir bisect log | grep "^git bisect skip \$rev\\\$" >/dev/null; then exit 125 elif git -C $current_project_dir bisect log | grep "^git bisect good \$rev\\\$" >/dev/null; then exit 0 fi # Restore known-good baseline state. rsync -a --del ${baseline_exclude[@]} ./bisect/baseline/ ./ $build_script \ ^^ $reproduce_bisect \ %%rr[top_artifacts] $rel_artifacts/build-\$rev \ @@ $rel_artifacts/build-parameters/manifest.sh \ ==rr[mode] bisect \ ==rr[update_baseline] ignore \ ==rr[${current_project}_git] "$bad_url#\$rev" \ --verbose "$verbose" & res=0 && wait \$! || res=\$? git -C $current_project_dir reset -q --hard if [ x"\$res" != x"0" ]; then if [ -f $rel_artifacts/build-\$rev/trigger-build-$current_project ]; then exit 1 else # The build failed due to an uninteresting problem -- a prerequisite # failed to build or benchmarking harness went down. We mark such # revisions "skipped", but up to a point. If we skip more revisions # in a row, than half the number of tests necessary to finish the bisect, # then we mark such "skipped" revision as "bad". # Number of "git bisect skip" in a row n_skips=\$(git -C $current_project_dir bisect log | awk ' BEGIN { n_skips=0 } /git bisect skip/ { n_skips++; next } /git bisect/ { n_skips=0 } END { print n_skips } ') revs_left=\$(git -C $current_project_dir bisect view --pretty=%H | wc -l) # Half the number of steps to finish the bisect n_steps_2=\$(echo "n_steps=l(\$revs_left)/l(2); scale=0; n_steps/2" | bc -l) if [ \$n_steps_2 -lt 2 ]; then # Avoid skipping revisions at the end of the bisect. n_steps_2=2 fi if [ \$n_skips -le \$n_steps_2 ]; then exit 125 else # We had several skips in a row and still have many revisions to bisect. # Mark this one "bad" to progress the bisect. exit 1 fi fi else exit 0 fi EOF chmod +x $artifacts/test.sh # Fetch $bad_branch/$bad_rev from $bad_url prev_rev=$(git -C $current_project_dir rev-parse HEAD) # Note: avoid using clone_or_update_repo(), which, potentially, can delete # and re-clone the repo. Deleting the repo would be bad, since we would # lose bisect state initialized by the above "git bisect" commands. # Note: avoid using clone_or_update_repo(), which calls git_checkout(), which # does a more thorough job in cleaning up the repo directory than # "git reset -q --hard" in ./test.sh. The logic here is that we want # the "bad" build to run in the same environment as the "test" builds. git -C "$current_project_dir" fetch "$bad_url" "$bad_branch" git -C "$current_project_dir" checkout --detach FETCH_HEAD bad_rev="${bad_rev-$(git -C "$current_project_dir" rev-parse HEAD)}" cat <> $artifacts/trigger-build-rebase <build->bisect->reset happens again and again, # then this is a scripting or infrastructure problem, and we detect # it by unusually high ratio of forced builds in CI dashboard. echo "WARNING: build for bad_rev $bad_rev succeeded" sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name cat > $artifacts/trigger-build-reset </dev/null; then git -C $current_project_dir bisect bad $bad_rev ln -f -s "build-$bad_rev" "$artifacts/build-bad" fi # Print first_bad revision (if detected) get_first_bad () { ( # Allow pipefail to handle error exit codes from git bisect log and grep. # Note that child shell inherits settings from parent shell, so we need # excplicitly set "+o pipefail". set -euf +o pipefail git -C $current_project_dir bisect log | tail -n1 \ | grep "^# first bad commit:" \ | sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/" ) } # Print revs tested during bisect. # $1 -- Revision kind -- good/bad/skip. # Print all revisions if no $1 given. print_tested_revs () { ( # Allow pipefail to handle error exit codes from git bisect log and grep. # Note that child shell inherits settings from parent shell, so we need # excplicitly set "+o pipefail". set -euf +o pipefail local kind="${1-[a-z]*}" git -C $current_project_dir bisect log | grep "^git bisect $kind " \ | sed -e "s/^git bisect $kind //" ) } # Print ratio at which commit splits current bisection range, or "-1" if # the commit is outside of bisection range. The ideal split is 50/50 -- # right in the middle. # $1: Commit SHA1 print_sha1_split () { local sha1="$1" local line line=$(grep -n "^$sha1\$" $commits_to_test | cut -d":" -f 1) if [ x"$line" = x"" ]; then echo "-1" return fi # Skip revisions that were already tested. Good revisions are filtered # out in the above $commits_to_test check, and here we filter out # "bad" and "skip" revisions. if git -C $current_project_dir bisect log \ | grep "^git bisect .* $sha1\$" >/dev/null; then echo "-1" return fi line=$((100 * $line / $(cat $commits_to_test | wc -l))) if [ $line -gt 50 ]; then line=$((100 - $line)) fi echo "$line" } # Try to reduce bisection range by testing regressions (and their parents) # identified in other configurations. print_interesting_commit () { ( set -euf -o pipefail # Generate list of commits inside the bisection range. git -C $current_project_dir bisect view --pretty=%H > $commits_to_test # Bisecting linux-next.git regressions is difficult enough due to how # the tree is constructed, so we prefer to not use interesting-commits # when $rebase_workaround is true. This makes linux-next bisects as # natural as they can be. if $rebase_workaround; then return fi # Clone interesting-commits.git repo, which contains a list of SHA1s # that might cut down bisection time. Mostly, these are first_bad and # last_good commits. local icommits="bisect/interesting-commits" clone_or_update_repo $icommits master \ https://git-us.linaro.org/toolchain/ci/interesting-commits.git \ auto master >/dev/null 2>&1 local project_dir project_dir=$icommits/$(interesting_subdir $current_project) if ! [ -d "$project_dir" ]; then return fi # Below loop can generate lots of console noise. set +x # Find an interesting commit that splits bisection range best. local sha1 prev_sha1 best_split=-1 best_sha1="" while read sha1; do while read prev_sha1; do split=$(print_sha1_split "$prev_sha1") if [ $split -gt $best_split ]; then best_split=$split best_sha1=$prev_sha1 fi done < <(echo "$sha1" cd "$project_dir/$sha1" find -name last_good -print0 | xargs -0 cat | sort -u) done < <(cd "$project_dir"; ls) if $verbose; then set -x; fi if [ "$best_sha1" = "" ]; then # We didn't find an interesting sha1, so use a stock recommendation by # git. Note that we want to remain in the "try-interesting-commits" # loop (rather than switching to "git bisect run") in the hopes that # some other job will add a new entry to interesting-commits while # we are testing the stock revisions. best_sha1=$(git -C $current_project_dir bisect next | tail -n1 \ | sed -e "s/^\[\([0-9a-f]\+\)\].*\$/\1/") # Ensure that best_sha1 is indeed a sha1. I could not figure out # how to tell "git bisect next" to print only sha1 without extra # annotations, so have to parse the string with "sed". if ! echo "$best_sha1" | grep '^[0-9a-f]\+$' &>/dev/null; then best_sha1="" fi fi echo "$best_split $best_sha1" ) } commits_to_test=$artifacts/git-logs/commits_to_test IFS=" " read -r split sha1 <<< "$(print_interesting_commit)" # Record commits in the initial bisect range. These are commits_to_test plus # commits that have been tested. commits_in_range=$artifacts/git-logs/commits_in_range cp $commits_to_test $commits_in_range print_tested_revs >> $commits_in_range while [ x"$sha1" != x"" ] \ && [ x"$(get_first_bad $artifacts/trigger-build-1-last-good <> $artifacts/trigger-build-2-reset if [ "$reset_rev" = "$first_bad" ]; then # We have identified a single-commit regression, so notify developers # about it. echo "notify=onregression" >> $artifacts/trigger-build-2-reset fi fi # Trigger master build now instead of waiting for next timed SCM trigger. # Make sure git specification is as it was passed to the bisect # (i.e., master branch, not a specific SHA1). cp $artifacts/build-$bad_rev/trigger-build-$current_project \ $artifacts/trigger-build-3-default sed -i -e "s%^\(${current_project}_git\)=.*\$%\1=$bad_git%" \ $artifacts/trigger-build-3-default # Save BISECT_* logs find "$current_project_dir" -path "$current_project_dir/.git/BISECT_*" -print0 \ | xargs -0 -I@ mv @ $artifacts/git-logs/ if [ x"$first_bad" != x"" ]; then sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name ln -f -s "build-$first_bad" "$artifacts/build-first_bad" ln -f -s "build-$last_good" "$artifacts/build-last_good" fi exit_0