diff options
Diffstat (limited to 'tcwg_kernel-bisect.sh')
-rwxr-xr-x | tcwg_kernel-bisect.sh | 169 |
1 files changed, 133 insertions, 36 deletions
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index 136dc24c..f6169710 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -187,21 +187,112 @@ ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh" git bisect start $bad_rev $baseline_rev 2>&1 | tee $artifacts/bisect.log -# "git bisect run" can fail (exit with non-zero) in a number of cases: -# - on trivial bisects (e.g., between HEAD^ and HEAD), -# - when merge-base between baseline and bad is worse than baseline, -# - something else? -# In all these cases we want to reset baseline to HEAD, so that we catch -# most of the commits that introduced change in the result metric. -git bisect run $artifacts/test.sh 2>&1 | tee -a $artifacts/bisect.log & -res=0 && wait $! || res=$? +if ! git bisect log > /dev/null; then + # Bisect ended before it could start: $bad_rev^ == $baseline_rev + first_bad=$bad_rev + last_good=$baseline_rev +else + first_bad="" + last_good="" +fi -if [ x"$res" = x"0" ]; then - first_bad=$(cat .git/BISECT_RUN | head -n 1 | grep "is the first bad commit" | cut -d" " -f 1) - assert [ x"$first_bad" != x"" ] +# Clone interesting-commits.git repo, which contains a list of SHA1s +# that might cut down bisection time. Mostly, these are first_bad and +# last_good commits. +interesting_commits_rev=${interesting_commits_rev-${rr[ci_project]}} +clone_or_update_repo ../interesting-commits $interesting_commits_rev https://git-us.linaro.org/toolchain/ci/interesting-commits.git auto $interesting_commits_rev +cat <<EOF | manifest_out +declare -g interesting_commits_rev=$interesting_commits_rev +EOF + +# Add revisions in $@ to interesting-commits and push the repo. +# Ignore failures (since this is cache handling). +push_interesting_commits () +{ + ( + set -euf -o pipefail + local sha1 + + for sha1 in "$@"; do + if ! grep -q "$sha1" ../interesting-commits/$current_project; then + echo "$sha1" >> ../interesting-commits/$current_project + fi + done + + git -C ../interesting-commits add . + git -C ../interesting-commits commit -m "Add commits from $BUILD_URL: + +$*" & + local res=0 && wait $! || res=$? + + if [ x"$res" = x"0" ]; then + git_init_linaro_local_remote ../interesting-commits baseline false + git_push ../interesting-commits baseline ${rr[ci_project]} + fi + ) || true +} + +# Print first_bad revision (if detected) +get_first_bad () +{ + ( + set -euf -o pipefail + + if [ x"$first_bad" != x"" ]; then + echo "$first_bad" + return + fi + + git bisect log | tail -n1 | grep "^# first bad commit:" \ + | sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/" + ) +} + +# Try to reduce bisection range by testing regressions (and their parents) +# identified in other configurations. +sha1="" +touch ../interesting-commits/$current_project +while [ x"$(get_first_bad </dev/null)" = x"" ] && read sha1; do + ( + set -euf -o pipefail + + # Skip $bad_rev and $baseline_rev, these were already tested. + if [ x"$sha1" = x"$bad_rev" -o x"$sha1" = x"$baseline_rev" ]; then + continue + fi + # Skip revisions outside of bisection range. + if ! git merge-base --is-ancestor $baseline_rev $sha1 \ + || ! git merge-base --is-ancestor $sha1 $bad_rev; then + continue + fi + + git checkout --detach $sha1 + $artifacts/test.sh & + res=0 && wait $! || res=$? + if [ x"$res" = x"0" ]; then + git bisect good + else + git bisect bad + fi + ) </dev/null +done < ../interesting-commits/$current_project + +if [ x"$(get_first_bad)" = x"" ]; then + git bisect run $artifacts/test.sh 2>&1 | tee -a $artifacts/bisect.log & + res=0 && wait $! || res=$? + + if [ x"$res" = x"0" ]; then + assert [ x"$(get_first_bad)" != x"" ] + fi +fi + +first_bad=$(get_first_bad) +if [ x"$first_bad" != x"" ]; then + # "git bisect run" succeeded. Check whether this is an actual regression + # or bisection artifact. res=0 - for last_good in $(git rev-parse $first_bad^@); do + for last_good in $last_good $(git rev-parse $first_bad^@); do # It seems that git-bisect assumes parent commit as "good" on # the basis of one of its children being "good". Therefore we # can have a situation when we have parent P with children C1 and C2, @@ -218,37 +309,43 @@ if [ x"$res" = x"0" ]; then break fi done - if [ x"$res" != x"0" ]; then + # Add both $last_good and $first_bad to interesting commits. + push_interesting_commits $last_good $first_bad + if [ x"$res" = x"0" ]; then + # Success! Touch $artifacts/first-bad as a marker of successful bisect. + echo $first_bad > $artifacts/first-bad + else # It seems $last_good was on a path that tested good, even though - # it itself is bad. We re-trigger the bisection job with updated - # parameters. + # it itself is bad. + # # We need to be careful to avoid re-trigger loops, so verify that # last_good is an ancestor of bad_rev. assert git merge-base --is-ancestor $last_good $bad_rev - cat > $artifacts/trigger-bisect <<EOF + if git merge-base --is-ancestor $baseline_rev $last_good; then + # $last_good is a child of $baseline_rev, so we can re-trigger + # bisection with reduced bisection range. + cat > $artifacts/trigger-bisect <<EOF current_project=$current_project baseline_branch=$baseline_rev bad_branch=$last_good EOF - # Don't send any emails. - echo > $artifacts/jenkins/mail-recipients.txt - touch $artifacts/jenkins/build-name - sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name - trap "" EXIT - exit 0 + touch $artifacts/jenkins/build-name + sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name + # Don't send any emails. + echo > $artifacts/jenkins/mail-recipients.txt + trap "" EXIT + exit 0 + fi + # This case will be handled similar to "git bisect run" failure below. + # We are going to reset baseline to $first_bad. fi - - echo $first_bad > $artifacts/first-bad else + # When "git bisect run" fails, e.g., due to merge-base of $baseline_rev and + # $bad_rev is worse than $baseline_rev, we want to reset baseline to HEAD, + # so that we catch most of the commits that introduced change in the result + # metric. first_bad=$(git rev-parse HEAD) - if ! [ -f .git/BISECT_LOG ]; then - # It seems this was a trivial bisect with $bad_rev^ == $baseline_rev. - first_bad=$bad_rev - last_good=$(git rev-parse $first_bad^) - assert [ x"$last_good" = x"$baseline_rev" ] - - echo $first_bad > $artifacts/first-bad - fi + push_interesting_commits $first_bad fi cd .. @@ -293,7 +390,7 @@ EOF fi # In log scan for errors below -# - ": error:" detects compiler errors from GCC and Clang +# - " error:" detects compiler errors from GCC and Clang (including GCC ICEs) # - "^ERROR:" detects linker errors # - "] Error " detects GNU make errors cat >> $artifacts/jenkins/mail-body.txt <<EOF @@ -307,7 +404,7 @@ to (for $bad_name == $bad_sha1) $(cat $artifacts/build-$bad_sha1/results) First few errors in logs of $bad_name: -$(grep ": error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head) +$(grep " error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head) Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/ Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/ @@ -364,9 +461,9 @@ fi CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org" case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in tcwg_kernel/gnu-*:linux) ;; - tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;; + tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, tcwg-gcc@linaro.org" ;; tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;; - tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;; + tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, llvm@linaro.org, ndesaulniers@google.com, trong@google.com" ;; esac cat > $artifacts/jenkins/mail-recipients.txt <<EOF $CI_MAIL_RECIPIENTS |