diff options
Diffstat (limited to 'round-robin-bisect.sh')
| -rwxr-xr-x | round-robin-bisect.sh | 835 |
1 files changed, 338 insertions, 497 deletions
diff --git a/round-robin-bisect.sh b/round-robin-bisect.sh index 8ded4ea6..8f883258 100755 --- a/round-robin-bisect.sh +++ b/round-robin-bisect.sh @@ -6,12 +6,7 @@ scripts=$(dirname $0) # shellcheck source=jenkins-helpers.sh . $scripts/jenkins-helpers.sh -# Relative artifacts are used for generation of manifests and reproduction -# instructions. -rel_artifacts=artifacts -artifacts=$(pwd)/$rel_artifacts - -fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*" +declare -A rr # Process bisect-only args convert_args_to_variables "$@" @@ -20,16 +15,39 @@ shift "$SHIFT_CONVERTED_ARGS" obligatory_variables bad_git build_script current_project declare bad_git build_script current_project +# Relative artifacts are used for generation of manifests and reproduction +# instructions. +rel_artifacts="${rel_artifacts-artifacts}" +artifacts=$(pwd)/$rel_artifacts + +fresh_dir $artifacts \ + "$artifacts/manifest.sh" \ + "$artifacts/build-parameters/manifest.sh" \ + "$artifacts/jenkins/*" + BUILD_URL="${BUILD_URL:-$(pwd)}" replay_log="${replay_log-}" reproduce_bisect="${reproduce_bisect:-false}" # Process build args and record them in build-parameters.sh -convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@" +convert_args_to_variables ^^ $reproduce_bisect %%build_parameters $artifacts/build-parameters "$@" $reproduce_bisect || manifest_pop +# Account for "^^ false %%foo bar" options +SHIFT_CONVERTED_ARGS=$(($SHIFT_CONVERTED_ARGS-4)) +shift "$SHIFT_CONVERTED_ARGS" -obligatory_variables rr[ci_project] rr[ci_config] -declare -A rr +obligatory_variables build_parameters rr[ci_project] rr[ci_config] +declare build_parameters + +# Process build args and record them in build-parameters.sh +convert_args_to_variables ^^ $reproduce_bisect %%baseline_parameters $artifacts/baseline-parameters "$@" +$reproduce_bisect || manifest_pop +# Account for "^^ false %%foo bar" options +SHIFT_CONVERTED_ARGS=$(($SHIFT_CONVERTED_ARGS-4)) +shift "$SHIFT_CONVERTED_ARGS" + +obligatory_variables baseline_parameters +declare baseline_parameters verbose="${verbose-true}" @@ -42,6 +60,17 @@ touch $artifacts/jenkins/build-name trap print_traceback EXIT +# Exit with success +exit_0 () { + # Cleanup bisect/ directory, which has a full [unneeded] copy of the build. + chmod -R +rwx bisect/ + rm -rf bisect/ + trap "" EXIT + exit 0 +} + +current_project_dir=$(get_component_dir $current_project) + bad_url="${bad_git%#*}" bad_branch="${bad_git#*#}" @@ -51,101 +80,68 @@ rebase_workaround_opts=() case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in tcwg_kernel/*-next-*:linux) # Workaround linux-next/master rebasing on top of linux-next/stable. - # Search for regressions against linux-mainline:master (aka linux-next:stable). - clone_or_update_repo $current_project stable $bad_url - # Just in case linux-next:stable has advanced between the build and bisect jobs, - # use merge base between linux-next:stable and $bad_git. - bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}" - linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}" - cat <<EOF | manifest_out -declare -g linux_next_stable=$linux_next_stable -EOF - echo "Rebase workaround: forcing linux baseline to $linux_next_stable" + # Search for regressions between linux-next:stable and + # linux-next:master. + echo "Rebase workaround: forcing linux baseline to linux-next:stable" rebase_workaround=true rebase_workaround_opts+=("==rr[linux_git]" - "$bad_url#$linux_next_stable") + "$bad_url#stable") ;; esac # Build baseline that we are going to re-use to speed-up bisection. # (This also confirms that infrastructure is OK.) -echo "Testing baseline (should be success)" +echo "Testing baseline revision (expecting success)" $build_script \ ^^ $reproduce_bisect \ - %% $rel_artifacts/manifests/build-baseline.sh \ - @@ $rel_artifacts/manifests/build-parameters.sh \ - ==rr[mode] baseline \ - ==rr[update_baseline] push \ - ==rr[top_artifacts] "$rel_artifacts/build-baseline" \ + %%rr[top_artifacts] "$rel_artifacts/build-baseline" \ + @@ $build_parameters/manifest.sh \ + @@ $baseline_parameters/manifest.sh \ + ==rr[mode] build \ + ==rr[update_baseline] force \ --verbose "$verbose" \ "${rebase_workaround_opts[@]}" -baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}" +# Establish results in build-baseline as the baseline to compare test builds +# against. +$scripts/round-robin-baseline.sh \ + @@rr[top_artifacts] "$rel_artifacts/build-baseline" \ + __base_artifacts base-artifacts + +baseline_rev="${baseline_rev-$(git -C $current_project_dir rev-parse HEAD)}" cat <<EOF | manifest_out declare -g baseline_rev=$baseline_rev EOF ln -f -s "build-baseline" "$artifacts/build-$baseline_rev" -ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh" - -case "${rr[ci_project]}/${rr[ci_config]}" in - tcwg_gnu/*-check_*|tcwg_cross/*-check_*) - ( - # Build up lists of flaky tests. We do this by comparing - # just-created baseline vs sumfiles in base-artifacts. - fails=$(find $rel_artifacts/build-baseline \ - -path "*-check_regression/fails.sum") - xfail_short="contrib/testsuite-management/flaky/${rr[ci_config]}.xfail" - xfail="gcc-compare-results/$xfail_short" - - if ! [ -f "$fails" ] || ! [ -f "$xfail" ]; then - exit - fi - - ( - echo - echo "# From $BUILD_URL:" - cat "$fails" | sed -e "s/^\([A-Z]\+: \)/flaky \| \1/" - ) >> "$xfail" - - git -C gcc-compare-results add "$xfail_short" - git -C gcc-compare-results commit -m "From $BUILD_URL" - git -C gcc-compare-results review -s - git -C gcc-compare-results push gerrit HEAD:refs/heads/master - ) & - res=0 && wait $! || res=$? - # Ignore any failures in the above. - ;; -esac mkdir -p ./bisect # Save baseline build state, which we restore before individual bisect tests. # This ensures that "bad" bisect tests won't affect "good" ones and vice versa. baseline_exclude=( - --exclude /bisect/ --exclude "/$rel_artifacts/" --exclude "/$current_project/" + --exclude /bisect/ --exclude "/$rel_artifacts/" --exclude "/$current_project_dir/" ) rsync -a --del --delete-excluded "${baseline_exclude[@]}" ./ ./bisect/baseline/ -cd $current_project - mkdir $artifacts/git-logs # Make sure the sources are clean before bisecting -git reset --hard +git -C $current_project_dir reset -q --hard if [ -f "$replay_log" ]; then cp "$replay_log" $artifacts/git-logs/bisect_replay.sh - git bisect replay $artifacts/git-logs/bisect_replay.sh + git -C $current_project_dir bisect replay $artifacts/git-logs/bisect_replay.sh else - git bisect start + git -C $current_project_dir bisect start fi # Hard-link BISECT_LOG inside $artifacts to guarantee its upload to jenkins. -ln -f "$(pwd)"/.git/BISECT_LOG $artifacts/git-logs/bisect_log +ln -f "$(pwd)"/$current_project_dir/.git/BISECT_LOG $artifacts/git-logs/bisect_log -if ! git bisect log | grep -q "^git bisect .* $baseline_rev\$"; then - git bisect good $baseline_rev +if ! git -C $current_project_dir bisect log \ + | grep "^git bisect .* $baseline_rev\$" >/dev/null; then + git -C $current_project_dir bisect good $baseline_rev fi # Bisect script. @@ -175,38 +171,64 @@ cat > $artifacts/test.sh <<EOF set -euf -o pipefail -rev=\$(git rev-parse HEAD) +current_project_dir=\$1 + +rev=\$(git -C $current_project_dir rev-parse HEAD) -if git bisect log | grep -q "^git bisect bad \$rev\\\$"; then +if git -C $current_project_dir bisect log | grep "^git bisect bad \$rev\\\$" >/dev/null; then exit 1 -elif git bisect log | grep -q "^git bisect skip \$rev\\\$"; then +elif git -C $current_project_dir bisect log | grep "^git bisect skip \$rev\\\$" >/dev/null; then exit 125 -elif git bisect log | grep -q "^git bisect good \$rev\\\$"; then +elif git -C $current_project_dir bisect log | grep "^git bisect good \$rev\\\$" >/dev/null; then exit 0 fi -cd .. - # Restore known-good baseline state. rsync -a --del ${baseline_exclude[@]} ./bisect/baseline/ ./ $build_script \ ^^ $reproduce_bisect \ - %% $rel_artifacts/manifests/build-\$rev.sh \ - @@ $rel_artifacts/manifests/build-parameters.sh \ + %%rr[top_artifacts] $rel_artifacts/build-\$rev \ + @@ $rel_artifacts/build-parameters/manifest.sh \ ==rr[mode] bisect \ + ==rr[update_baseline] ignore \ ==rr[${current_project}_git] "$bad_url#\$rev" \ - ==rr[top_artifacts] $rel_artifacts/build-\$rev \ --verbose "$verbose" & res=0 && wait \$! || res=\$? -git -C $current_project reset --hard +git -C $current_project_dir reset -q --hard if [ x"\$res" != x"0" ]; then if [ -f $rel_artifacts/build-\$rev/trigger-build-$current_project ]; then exit 1 else - exit 125 + # The build failed due to an uninteresting problem -- a prerequisite + # failed to build or benchmarking harness went down. We mark such + # revisions "skipped", but up to a point. If we skip more revisions + # in a row, than half the number of tests necessary to finish the bisect, + # then we mark such "skipped" revision as "bad". + + # Number of "git bisect skip" in a row + n_skips=\$(git -C $current_project_dir bisect log | awk ' +BEGIN { n_skips=0 } +/git bisect skip/ { n_skips++; next } +/git bisect/ { n_skips=0 } +END { print n_skips } +') + revs_left=\$(git -C $current_project_dir bisect view --pretty=%H | wc -l) + # Half the number of steps to finish the bisect + n_steps_2=\$(echo "n_steps=l(\$revs_left)/l(2); scale=0; n_steps/2" | bc -l) + if [ \$n_steps_2 -lt 2 ]; then + # Avoid skipping revisions at the end of the bisect. + n_steps_2=2 + fi + if [ \$n_skips -le \$n_steps_2 ]; then + exit 125 + else + # We had several skips in a row and still have many revisions to bisect. + # Mark this one "bad" to progress the bisect. + exit 1 + fi fi else exit 0 @@ -215,9 +237,17 @@ EOF chmod +x $artifacts/test.sh # Fetch $bad_branch/$bad_rev from $bad_url -prev_rev=$(git rev-parse HEAD) -clone_or_update_repo . "$bad_branch" "$bad_url" -bad_rev="${bad_rev-$(git_rev_parse_long . $bad_branch)}" +prev_rev=$(git -C $current_project_dir rev-parse HEAD) +# Note: avoid using clone_or_update_repo(), which, potentially, can delete +# and re-clone the repo. Deleting the repo would be bad, since we would +# lose bisect state initialized by the above "git bisect" commands. +# Note: avoid using clone_or_update_repo(), which calls git_checkout(), which +# does a more thorough job in cleaning up the repo directory than +# "git reset -q --hard" in ./test.sh. The logic here is that we want +# the "bad" build to run in the same environment as the "test" builds. +git -C "$current_project_dir" fetch "$bad_url" "$bad_branch" +git -C "$current_project_dir" checkout --detach FETCH_HEAD +bad_rev="${bad_rev-$(git -C "$current_project_dir" rev-parse HEAD)}" cat <<EOF | manifest_out declare -g bad_rev=$bad_rev EOF @@ -225,150 +255,60 @@ EOF if [ x"$baseline_rev" = x"$bad_rev" ]; then echo "WARNING: Detected regression with no change in sources of $current_project" sed -i -e "s/\$/-no_change/" $artifacts/jenkins/build-name - trap "" EXIT - exit 1 + res=0 +else + # Confirm regression in $bad_rev vs $baseline_rev. + echo "Testing bad revision (expecting failure)" + git -C $current_project_dir checkout --detach $bad_rev + $artifacts/test.sh $current_project_dir & + res=0 && wait $! || res=$? fi -# Confirm regression in $bad_rev vs $baseline_rev. -git checkout --detach $bad_rev -$artifacts/test.sh & -res=0 && wait $! || res=$? # Restore revision previously checked out. Otherwise "git bisect run" # below will not use replay info. -git checkout --detach $prev_rev +git -C $current_project_dir checkout --detach $prev_rev if [ x"$res" = x"0" ]; then if $rebase_workaround; then echo "Rebase workaround: no regression between $baseline_rev and $bad_rev" sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name - project_name="${rr[ci_project]}/${rr[ci_config]}:$current_project" - case $project_name in - tcwg_kernel/llvm-*-next-*:linux) - cat > $artifacts/trigger-build-rebase <<EOF -llvm_git=baseline -EOF - ;; - tcwg_kernel/gnu-*-next-*:linux) - cat > $artifacts/trigger-build-rebase <<EOF -binutils_git=baseline -gcc_git=baseline -EOF - ;; - *) assert_with_msg "Unknown project name: $project_name" false ;; - esac cat >> $artifacts/trigger-build-rebase <<EOF linux_git=$bad_url#$baseline_rev -update_baseline=reset +update_baseline=force EOF else + # Build for $bad_rev is successful, which can be due to a number + # of things: + # - Regressions in speed benchmarking are not 100% stable, + # - Something has changed in the build environment, + # - Underlying hardware has changed, + # - Something entirely different. + # In all these cases we recover by rebuilding from baseline sources + # and updating the baseline. + # If baseline reset->build->bisect->reset happens again and again, + # then this is a scripting or infrastructure problem, and we detect + # it by unusually high ratio of forced builds in CI dashboard. echo "WARNING: build for bad_rev $bad_rev succeeded" sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name - # Regressions in speed benchmarking are not stable, - # so retry with resetting baseline artifacts. - # Retry with default parameters for other cases. - case "${rr[ci_project]}/${rr[ci_config]}" in - tcwg_bmk*/gnu*-O[23]*) - cat > $artifacts/trigger-build-reset <<EOF -binutils_git=baseline -gcc_git=baseline -glibc_git=baseline -update_baseline=reset + cat > $artifacts/trigger-build-reset <<EOF +update_baseline=force EOF - ;; - tcwg_bmk*/llvm-*-O[23]*) - cat > $artifacts/trigger-build-reset <<EOF -binutils_git=baseline -gcc_git=baseline -glibc_git=baseline -llvm_git=baseline -update_baseline=reset -EOF - ;; - esac fi - echo > $artifacts/jenkins/mail-recipients.txt - trap "" EXIT - exit 0 + exit_0 elif [ x"$res" = x"125" ]; then # We have confirmed a regression, but not what we have been triggered # to bisect. echo "WARNING: build for bad_rev $bad_rev showed uninteresting regression" sed -i -e "s/\$/-uninteresting/" $artifacts/jenkins/build-name - echo > $artifacts/jenkins/mail-recipients.txt - trap "" EXIT - exit 0 + exit_0 fi -if ! git bisect log | grep -q "^git bisect .* $bad_rev\$"; then - git bisect bad $bad_rev +if ! git -C $current_project_dir bisect log \ + | grep "^git bisect .* $bad_rev\$" >/dev/null; then + git -C $current_project_dir bisect bad $bad_rev ln -f -s "build-$bad_rev" "$artifacts/build-bad" - ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh" fi -# Clone interesting-commits.git repo, which contains a list of SHA1s -# that might cut down bisection time. Mostly, these are first_bad and -# last_good commits. -interesting_commits="../bisect/interesting-commits" - -interesting_commits_url=https://git-us.linaro.org/toolchain/ci/interesting-commits.git -interesting_commits_branch=linaro-local/ci/${rr[ci_project]} -if ! git ls-remote --heads $interesting_commits_url \ - | grep -q ".* refs/heads/$interesting_commits_branch"; then - interesting_commits_branch=empty -fi -interesting_commits_rev=${interesting_commits_rev-$interesting_commits_branch} -clone_or_update_repo $interesting_commits $interesting_commits_rev $interesting_commits_url auto $interesting_commits_branch -interesting_commits_rev=$(git -C $interesting_commits rev-parse HEAD) -cat <<EOF | manifest_out -declare -g interesting_commits_rev=$interesting_commits_rev -EOF - -# Add SHA1 commit $1 to interesting-commits and push the repo. -# If $2 is "regression" then record current configuration as having regressed -# at this commit. -# Ignore failures (since this is cache handling). -push_interesting_commit () -{ - declare -g push_interesting_commit_result=0 - ( - set -euf -o pipefail - - local sha1="$1" - local kind="$2" - local -a configs - - if ! grep -q "^$sha1" $interesting_commits/$current_project; then - echo "$sha1" >> $interesting_commits/$current_project - fi - - if [ x"$kind" = x"regression" ]; then - mapfile -t configs < <(grep "^$sha1" $interesting_commits/$current_project | sed -e "s/^$sha1 *//") - configs+=("${rr[ci_project]}"/"${rr[ci_config]}") - mapfile -t configs < <(echo "${configs[@]}" | tr ' ' '\n' | sort -u) - sed -i -e "s#^$sha1.*\$#$sha1 ${configs[*]}#" $interesting_commits/$current_project - fi - - git -C $interesting_commits add . - if [ x"$(git -C $interesting_commits status --short)" = x"" ]; then - # No file has changed. We've been here before... - # E.g., this is a re-occuring regression in linux-next. - exit 125 - fi - git -C $interesting_commits commit -m "Add $kind $sha1 from $BUILD_URL - -${configs[*]}" & - local res=0 && wait $! || res=$? - - if [ x"$res" = x"0" ]; then - # Interesting-commits.git do not have .gitreview, so it's - # simpler to push via gitolite. - git_init_linaro_local_remote $interesting_commits baseline false - git_push $interesting_commits baseline linaro-local/ci/${rr[ci_project]} - fi - ) & - wait $! || push_interesting_commit_result=$? -} - # Print first_bad revision (if detected) get_first_bad () { @@ -378,7 +318,8 @@ get_first_bad () # excplicitly set "+o pipefail". set -euf +o pipefail - git bisect log | tail -n1 | grep "^# first bad commit:" \ + git -C $current_project_dir bisect log | tail -n1 \ + | grep "^# first bad commit:" \ | sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/" ) } @@ -395,62 +336,156 @@ print_tested_revs () set -euf +o pipefail local kind="${1-[a-z]*}" - git bisect log | grep "^git bisect $kind " | sed -e "s/^git bisect $kind //" + git -C $current_project_dir bisect log | grep "^git bisect $kind " \ + | sed -e "s/^git bisect $kind //" ) } -# Try to reduce bisection range by testing regressions (and their parents) -# identified in other configurations. -touch $interesting_commits/$current_project -# Generate list of commits inside the bisection range. -commits_to_test=$artifacts/git-logs/commits_to_test -git bisect view --pretty=%H > $commits_to_test +# Print ratio at which commit splits current bisection range, or "-1" if +# the commit is outside of bisection range. The ideal split is 50/50 -- +# right in the middle. +# $1: Commit SHA1 +print_sha1_split () +{ + local sha1="$1" -# Record commits in the bisect range. These are commits_to_test plus -# commits that have been tested. -commits_in_range=$artifacts/git-logs/commits_in_range -cp $commits_to_test $commits_in_range -print_tested_revs >> $commits_in_range + local line + line=$(grep -n "^$sha1\$" $commits_to_test | cut -d":" -f 1) + if [ x"$line" = x"" ]; then + echo "-1" + return + fi + + # Skip revisions that were already tested. Good revisions are filtered + # out in the above $commits_to_test check, and here we filter out + # "bad" and "skip" revisions. + if git -C $current_project_dir bisect log \ + | grep "^git bisect .* $sha1\$" >/dev/null; then + echo "-1" + return + fi + + line=$((100 * $line / $(cat $commits_to_test | wc -l))) + if [ $line -gt 50 ]; then + line=$((100 - $line)) + fi + echo "$line" +} -# This loop can generate lots of console noise. -set +x -while [ x"$(get_first_bad </dev/null)" = x"" ] && read -a arr; do +# Try to reduce bisection range by testing regressions (and their parents) +# identified in other configurations. +print_interesting_commit () +{ ( set -euf -o pipefail - sha1="${arr[0]}" + # Generate list of commits inside the bisection range. + git -C $current_project_dir bisect view --pretty=%H > $commits_to_test - # Ignore commits outside of bisection range. - if ! grep -q "^$sha1\$" $commits_to_test; then - exit 0 + # Bisecting linux-next.git regressions is difficult enough due to how + # the tree is constructed, so we prefer to not use interesting-commits + # when $rebase_workaround is true. This makes linux-next bisects as + # natural as they can be. + if $rebase_workaround; then + return fi - # Skip revisions that were already tested. Good revisions are filtered - # out in the above $commits_to_test check, and here we filter out - # "bad" and "skip" revisions. - if git bisect log | grep -q "^git bisect .* $sha1\$"; then - exit 0 + # Clone interesting-commits.git repo, which contains a list of SHA1s + # that might cut down bisection time. Mostly, these are first_bad and + # last_good commits. + local icommits="bisect/interesting-commits" + clone_or_update_repo $icommits master \ + https://git-us.linaro.org/toolchain/ci/interesting-commits.git \ + auto master >/dev/null 2>&1 + + local project_dir + project_dir=$icommits/$(interesting_subdir $current_project) + + if ! [ -d "$project_dir" ]; then + return fi + # Below loop can generate lots of console noise. + set +x + + # Find an interesting commit that splits bisection range best. + local sha1 prev_sha1 best_split=-1 best_sha1="" + while read sha1; do + while read prev_sha1; do + split=$(print_sha1_split "$prev_sha1") + if [ $split -gt $best_split ]; then + best_split=$split + best_sha1=$prev_sha1 + fi + done < <(echo "$sha1" + cd "$project_dir/$sha1" + find -name last_good -print0 | xargs -0 cat | sort -u) + done < <(cd "$project_dir"; ls) + if $verbose; then set -x; fi - git checkout --detach $sha1 - $artifacts/test.sh & + if [ "$best_sha1" = "" ]; then + # We didn't find an interesting sha1, so use a stock recommendation by + # git. Note that we want to remain in the "try-interesting-commits" + # loop (rather than switching to "git bisect run") in the hopes that + # some other job will add a new entry to interesting-commits while + # we are testing the stock revisions. + best_sha1=$(git -C $current_project_dir bisect next | tail -n1 \ + | sed -e "s/^\[\([0-9a-f]\+\)\].*\$/\1/") + # Ensure that best_sha1 is indeed a sha1. I could not figure out + # how to tell "git bisect next" to print only sha1 without extra + # annotations, so have to parse the string with "sed". + if ! echo "$best_sha1" | grep '^[0-9a-f]\+$' &>/dev/null; then + best_sha1="" + fi + fi + echo "$best_split $best_sha1" + ) +} + +commits_to_test=$artifacts/git-logs/commits_to_test + +IFS=" " read -r split sha1 <<< "$(print_interesting_commit)" + +# Record commits in the initial bisect range. These are commits_to_test plus +# commits that have been tested. +commits_in_range=$artifacts/git-logs/commits_in_range +cp $commits_to_test $commits_in_range +print_tested_revs >> $commits_in_range + +while [ x"$sha1" != x"" ] \ + && [ x"$(get_first_bad </dev/null)" = x"" ]; do + if [ "$split" = "-1" ]; then + echo "Trying $sha1 stock recommendation" + else + echo "Trying $sha1 interesting commit, which splits bisections range" \ + "at ${split}%" + fi + + git -C $current_project_dir checkout --detach $sha1 + $artifacts/test.sh $current_project_dir & res=0 && wait $! || res=$? if [ x"$res" = x"0" ]; then - git bisect good + git -C $current_project_dir bisect good || break elif [ x"$res" = x"125" ]; then - git bisect skip + # It may happen that we get to the point where only skipped commits + # are left to test, and any new "git bisect skip" will return an error. + # In this case break from this loop, and let "git bisect run" below + # handle this case. [We also add "|| break" for good and bad cases + # for symmetry.] + git -C $current_project_dir bisect skip || break else - git bisect bad + git -C $current_project_dir bisect bad || break fi - git bisect view --pretty=%H > $commits_to_test - ) </dev/null -done < <(cat $interesting_commits/$current_project) -if $verbose; then set -x; fi + + IFS=" " read -r split sha1 <<< "$(print_interesting_commit)" +done if [ x"$(get_first_bad)" = x"" ]; then - git bisect run $artifacts/test.sh & + # Run stock "git bisect run" for corner cases like $rebase_workaround. + # Most of the time we have bisected the failure down to first_bad above. + + git -C $current_project_dir bisect run $artifacts/test.sh . & res=0 && wait $! || res=$? if [ x"$res" = x"0" ]; then @@ -459,15 +494,20 @@ if [ x"$(get_first_bad)" = x"" ]; then fi first_bad=$(get_first_bad) -reset_rev="$first_bad" -notify_devs=true -notify_author="" + +if [ x"$first_bad" != x"" ] \ + && ! [ -f $artifacts/build-$first_bad/trigger-build-$current_project ]; then + # First_bad is not a real or "interesting" regression. Perhaps it was + # a "skipped" commit marked as "bad. + first_bad="" +fi + if [ x"$first_bad" != x"" ]; then # "git bisect run" succeeded. Check whether this is an actual regression # or bisection artifact. last_good="" bad_last_good="" - for sha1 in $(git rev-parse $first_bad^@); do + for sha1 in $(git -C $current_project_dir rev-parse $first_bad^@); do # It seems that git-bisect assumes parent commit as "good" on # the basis of one of its children being "good". Therefore we # can have a situation when we have parent P with children C1 and C2, @@ -483,7 +523,7 @@ if [ x"$first_bad" != x"" ]; then if ! grep -q "^$sha1\$" $commits_in_range; then child_tested_good=false for tested_good in $(print_tested_revs good); do - if git merge-base --is-ancestor $sha1 $tested_good; then + if git -C $current_project_dir merge-base --is-ancestor $sha1 $tested_good; then child_tested_good=true break fi @@ -494,8 +534,8 @@ if [ x"$first_bad" != x"" ]; then fi echo "Testing first_bad's parent $sha1 (hoping for success)" - git checkout --detach "$sha1" - $artifacts/test.sh & + git -C $current_project_dir checkout --detach "$sha1" + $artifacts/test.sh $current_project_dir & res=0 && wait $! || res=$? if [ x"$res" = x"0" ]; then last_good=$sha1 @@ -504,30 +544,20 @@ if [ x"$first_bad" != x"" ]; then bad_last_good=$sha1 done - if [ x"$last_good" != x"" ]; then - # We have successfully identified a bad commit with a good parent! - # Add both $last_good and $first_bad to interesting commits. - push_interesting_commit $last_good "last-good" - push_interesting_commit $first_bad "regression" - if [ x"$push_interesting_commit_result" = x"125" ]; then - notify_devs=false - fi - else + if [ x"$last_good" = x"" ]; then assert_with_msg "Broken bisection range" [ x"$bad_last_good" != x"" ] - # All parents of $first_bad tested bad, so retrigger bisection with - # a reduced bisection range. - cat > $artifacts/trigger-bisect <<EOF -current_project=$current_project -bad_git=$bad_url#$bad_last_good -EOF - sed -i -e "s/\$/-retry-bisect/" $artifacts/jenkins/build-name - # Don't send any emails. - echo > $artifacts/jenkins/mail-recipients.txt - trap "" EXIT - exit 0 + # All parents of $first_bad tested bad. This is, essencially, + # same situation as "git bisect" failing, which we handle below. + first_bad="" fi +fi + +reset_rev="" +if [ x"$first_bad" != x"" ]; then + reset_rev="$first_bad" else - # "Git bisect" didn't find the first_bad commit. + # "Git bisect" didn't find the first_bad commit or this commit points to + # an uninteresting regression. # Possible reasons include: # - We have marked the regressing commit as "skip", which can happen # to tcwg_bmk* projects when benchmarking infra has a problem. @@ -536,6 +566,9 @@ else # We want to reset baseline to HEAD in this case, so that we catch most # of the commits that introduced change in the result metric. # + # - We have marked a skipped commit as bad to advance bisect, even though + # the failure was uninteresting. + # # So, to make at least some progress on narrowing down the regression ... # - look for the last commit that tested "good". If it's not $baseline_rev, # then we have narrowed down the bisection range somewhat. Therefore, @@ -543,8 +576,8 @@ else # - one to ADVANCE baseline to the last GOOD commit, and # - another to expose the regression again. # - # - If $baseline_rev is the only good commit, then we have got outselves - # to a tricky situation: we can't find the regression, and can't make + # - If $baseline_rev is the only good commit, then we have got ourselves + # into a tricky situation: we can't find the regression and can't make # a good build, which would advance the baseline. To resolve this, # trigger two builds: # - one to RESET baseline to the last BAD commit, and @@ -557,23 +590,40 @@ else [ x"$last_good" != x"" ] if [ x"$last_good" != x"$baseline_rev" ]; then - # $reset_rev=="" due to $first_bad=="" - cat > $artifacts/trigger-build-1-advance <<EOF -${current_project}_git=$bad_url#$last_good -EOF sed -i -e "s/\$/-advance-baseline/" $artifacts/jenkins/build-name else - reset_rev=$(print_tested_revs bad | tail -n1) + # Reset baseline to the earliest bad revision with a true regression. + # For this we look through tested-bad revisions in reverse order. + for reset_rev in $(print_tested_revs bad | tac); do + if [ -f $artifacts/build-$reset_rev/trigger-build-$current_project ]; then + break + fi + done sed -i -e "s/\$/-reset-baseline/" $artifacts/jenkins/build-name fi - # Don't send any emails. - notify_devs=false fi -# Bisect if officially over. -cd .. +# Bisect is officially over. # Create trigger-build-* files for subsequent builds. + +# Advance the baseline to $last_good revision. Don't trigger unnecessary +# build for $baseline_rev itself. +if [ x"$last_good" != x"$baseline_rev" ]; then + assert_with_msg "last_good should not be empty" [ x"$last_good" != x"" ] + cat > $artifacts/trigger-build-1-last-good <<EOF +${current_project}_git=$bad_url#$last_good +update_baseline=force +EOF + # Note that even though $last_good build should succeed, we are forcing + # it to update the baseline. Otherwise, should the last-good build fail, + # it will start a bisect job, which will trigger some more builds. + # The problem is that, as soon as our first-bad build resets the baseline, + # all those builds triggered by the 2nd bisect will operate "in the past". + # Therefore, we either need to handle builds "from the past", or, easier, + # just force the last-good build to always succeed. +fi + if ! [ -f $artifacts/build-$bad_rev/trigger-build-$current_project ]; then # This can happen *only* when replaying bisects. # Otherwise $bad_rev is always tested second to $baseline_rev. @@ -599,242 +649,33 @@ if [ x"$reset_rev" != x"" ]; then \ # Reset baseline to the regressed commit so that we will catch subsequent # regressions (worse than $bad_rev). cp $artifacts/build-$reset_rev/trigger-build-$current_project \ - $artifacts/trigger-build-1-reset - echo "update_baseline=reset" >> $artifacts/trigger-build-1-reset + $artifacts/trigger-build-2-reset + echo "update_baseline=force" >> $artifacts/trigger-build-2-reset + + if [ "$reset_rev" = "$first_bad" ]; then + # We have identified a single-commit regression, so notify developers + # about it. + echo "notify=onregression" >> $artifacts/trigger-build-2-reset + fi fi # Trigger master build now instead of waiting for next timed SCM trigger. +# Make sure git specification is as it was passed to the bisect +# (i.e., master branch, not a specific SHA1). cp $artifacts/build-$bad_rev/trigger-build-$current_project \ - $artifacts/trigger-build-2-default + $artifacts/trigger-build-3-default +sed -i -e "s%^\(${current_project}_git\)=.*\$%\1=$bad_git%" \ + $artifacts/trigger-build-3-default # Save BISECT_* logs -find "$current_project" -path "$current_project/.git/BISECT_*" -print0 | xargs -0 -I@ mv @ $artifacts/git-logs/ - -# Remove any fail-safe email body -rm -f $artifacts/jenkins/mail-body.txt +find "$current_project_dir" -path "$current_project_dir/.git/BISECT_*" -print0 \ + | xargs -0 -I@ mv @ $artifacts/git-logs/ if [ x"$first_bad" != x"" ]; then - mkdir -p $artifacts/jenkins sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name ln -f -s "build-$first_bad" "$artifacts/build-first_bad" - ln -f -s "build-$first_bad.sh" "$artifacts/manifests/build-first_bad.sh" - - good_name="last_good" - good_sha1="$last_good" - bad_name="first_bad" - bad_sha1="$first_bad" - ln -f -s "build-$last_good" "$artifacts/build-last_good" - ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh" - - occurences="$(cat $current_project/$interesting_commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")" - if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then - git_log_level="medium" - notify_author=$(git -C $current_project log --pretty="%an <%ae>" -n 1 \ - "$first_bad") - else - git_log_level="short" - fi - - cat >> $artifacts/jenkins/mail-body.txt <<EOF -Successfully identified regression in *$current_project* in CI configuration ${rr[ci_project]}/${rr[ci_config]}. So far, this commit has regressed CI configurations: -$occurences - -Culprit: -<cut> -$(git -C $current_project log --pretty=$git_log_level -n 1 $first_bad) -</cut> - -EOF -else - good_name="baseline_rev" - good_sha1="$baseline_rev" - bad_name="bad" - bad_sha1="$bad_rev" - cat >> $artifacts/jenkins/mail-body.txt <<EOF -Could not identify regression in *$current_project* in CI configuration ${rr[ci_project]}/${rr[ci_config]}. See 'Bisect log' in the links below for bisection details. - -EOF -fi - -if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$first_bad" != x"" ]; then - case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in - tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;; - tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;; - tcwg_kernel/llvm-*:linux) jira_card="LLVM-647" ;; - tcwg_kernel/llvm-*:*) jira_card="LLVM-646" ;; - tcwg_bmk_*/gnu*-O[23]*) jira_card="GNU-689" ;; - tcwg_bmk_*/gnu*) jira_card="GNU-686" ;; - tcwg_bmk_*/llvm-*O[23]*) jira_card="LLVM-651" ;; - tcwg_bmk_*/llvm-*) jira_card="LLVM-650" ;; - tcwg_binutils/*) jira_card="GNU-692" ;; - tcwg_cross/*) jira_card="GNU-692" ;; - tcwg_gnu/*) jira_card="GNU-692" ;; - # Catch-all case for when project/config IDs change, so that we - # won't miss notifications. Forward all that to GNU-692. - *) jira_card="GNU-692" ;; - esac - - if $notify_devs && [ x"$jira_card" != x"" ]; then - cat > $artifacts/jenkins/jira-body.txt <<EOF -[$jira_card] -$(cat $artifacts/jenkins/mail-body.txt) - -Details: ${BUILD_URL}artifact/$rel_artifacts/jenkins/mail-body.txt/*view*/ -Even more details: ${BUILD_URL}artifact/$rel_artifacts/ -EOF - if ! [ -f $HOME/.jipdate.yml ]; then - cat > $HOME/.jipdate.yml <<EOF -server: - url: https://linaro.atlassian.net - token: #TCWG_JIRA_TOKEN# -text-editor: False -username: team-toolchain+tcwg-jira@linaro.org -EOF - fi - sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" $HOME/.jipdate.yml - echo y | jipdate.py -f $artifacts/jenkins/jira-body.txt - fi -fi - -cat >> $artifacts/jenkins/mail-body.txt <<EOF -Results regressed to (for $bad_name == $bad_sha1) -$(cat $artifacts/build-$bad_sha1/results) - -from (for $good_name == $good_sha1) -$(cat $artifacts/build-$good_sha1/results) - -EOF - -good_build_artifacts="Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/" -bad_build_artifacts="Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/" - -# Benchmark builds have a results ID that is used to download the results -if test -f "$artifacts/build-$good_sha1/results_id"; then - cat >> $artifacts/jenkins/mail-body.txt <<EOF -$good_build_artifacts -Results ID of $good_name: $(cat $artifacts/build-$good_sha1/results_id) -$bad_build_artifacts -Results ID of $bad_name: $(cat $artifacts/build-$bad_sha1/results_id) -EOF -# Os vs Os LTO builds have two result IDs per job -elif test -f "$artifacts/build-$good_sha1/results_id-1"; then - cat >> $artifacts/jenkins/mail-body.txt <<EOF -$good_build_artifacts -Results IDs of $good_name: -$(cat $artifacts/build-$good_sha1/results_id-1) -$(cat $artifacts/build-$good_sha1/results_id-2) -$bad_build_artifacts -Results ID of $bad_name: -$(cat $artifacts/build-$bad_sha1/results_id-1) -$(cat $artifacts/build-$bad_sha1/results_id-2) -EOF -else - cat >> $artifacts/jenkins/mail-body.txt <<EOF -$good_build_artifacts -$bad_build_artifacts -EOF -fi - -cat >> $artifacts/jenkins/mail-body.txt <<EOF -Build top page/logs: ${BUILD_URL} - -Configuration details: -$(cat $artifacts/manifests/build-baseline.sh | grep '_git]' | grep -v '="no_') - -Reproduce builds: -<cut> -mkdir investigate-$current_project-$bad_sha1 -cd investigate-$current_project-$bad_sha1 - -git clone https://git.linaro.org/toolchain/jenkins-scripts - -mkdir -p $rel_artifacts/manifests -curl -o $rel_artifacts/manifests/build-baseline.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-baseline.sh --fail -curl -o $rel_artifacts/manifests/build-parameters.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-parameters.sh --fail -curl -o $rel_artifacts/test.sh ${BUILD_URL}artifact/$rel_artifacts/test.sh --fail -chmod +x $rel_artifacts/test.sh - -# Reproduce the baseline build (build all pre-requisites) -$build_script @@ $rel_artifacts/manifests/build-baseline.sh - -# Save baseline build state (which is then restored in $rel_artifacts/test.sh) -mkdir -p ./bisect -rsync -a --del --delete-excluded ${baseline_exclude[@]} ./ ./bisect/baseline/ - -cd $current_project - -# Reproduce $bad_name build -git checkout --detach $bad_sha1 -../$rel_artifacts/test.sh - -# Reproduce $good_name build -git checkout --detach $good_sha1 -../$rel_artifacts/test.sh - -cd .. -</cut> - -History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/${rr[ci_project]}/${rr[ci_config]} - -Artifacts: ${BUILD_URL}artifact/$rel_artifacts/ -Build log: ${BUILD_URL}consoleText -EOF - -if [ x"$first_bad" != x"" ]; then - cat >> $artifacts/jenkins/mail-body.txt <<EOF - -Full commit (up to 1000 lines): -<cut> -$(git -C $current_project show --stat --patch $first_bad | head -n 1000) -</cut> -EOF -fi - -# Set mail recipients last to preserve catch-error value from .yaml file. -# Email developers. -CI_MAIL_RECIPIENTS=("tcwg-validation@linaro.org") -case "$notify_author@${rr[ci_project]}/${rr[ci_config]}:$current_project" in - ""@*/*:*) ;; - *@tcwg_gnu/*-check_bootstrap*:*) ;; # We are building up list of flaky tests - *@tcwg_cross/*-check_cross:*) ;; # We are building up list of flaky tests - *@tcwg_gnu/*:*) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - ;; - *@tcwg_cross/*:*) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - ;; - *@tcwg_kernel/llvm-*:linux) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - CI_MAIL_RECIPIENTS+=("clang-built-linux@googlegroups.com") - CI_MAIL_RECIPIENTS+=("arnd@linaro.org") - ;; - *@tcwg_kernel/llvm-*:llvm) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - CI_MAIL_RECIPIENTS+=("clang-built-linux@googlegroups.com") - ;; - *@tcwg_bmk*/*:*) - # Don't notify patch authors until we improve benchmarking email - # reporting. - #CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - ;; - *@*/*:*) - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - ;; -esac -if $notify_devs; then - ( - IFS="," - cat > $artifacts/jenkins/mail-recipients.txt <<EOF -${CI_MAIL_RECIPIENTS[*]} -EOF - ) fi -trap "" EXIT +exit_0 |