diff options
Diffstat (limited to 'tcwg_kernel-bisect.sh')
| -rwxr-xr-x | tcwg_kernel-bisect.sh | 120 |
1 files changed, 69 insertions, 51 deletions
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index 45a6aaf0..136dc24c 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -16,7 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*" convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" -obligatory_variables bad_rev baseline_rev build_script rr[ci_project] +obligatory_variables bad_branch baseline_branch build_script current_project BUILD_URL="${BUILD_URL:-$(pwd)}" reproduce_bisect="${reproduce_bisect:-false}" @@ -25,13 +25,7 @@ reproduce_bisect="${reproduce_bisect:-false}" convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@" $reproduce_bisect || manifest_pop -obligatory_variables rr[current_project] rr[ci_config] - -current_project=${rr[current_project]} - -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then - obligatory_variables toolchain -fi +obligatory_variables rr[ci_project] rr[ci_config] verbose="${verbose:-true}" @@ -43,38 +37,41 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/fai rebase_workaround=false -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ] \ - && [ x"$current_project" = x"linux" -a x"${rr[linux_version]}" = x"next" ]; then - # Workaround linux-next/master rebasing on top of linux-next/stable. - # Search for regressions against linux-mainline:master (aka linux-next:stable). - clone_or_update_repo $current_project stable ${rr[linux_url]} - # Just in case linux-next:stable has advanced between the build and bisect jobs, - # use merge base between linux-next:stable and $bad_rev. - linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}" - cat <<EOF | manifest_out +case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in + tcwg_kernel/*-next-*:linux) + # Workaround linux-next/master rebasing on top of linux-next/stable. + # Search for regressions against linux-mainline:master (aka linux-next:stable). + clone_or_update_repo $current_project stable ${rr[linux_url]} + # Just in case linux-next:stable has advanced between the build and bisect jobs, + # use merge base between linux-next:stable and $bad_branch. + bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}" + linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}" + cat <<EOF | manifest_out declare -g linux_next_stable=$linux_next_stable EOF - echo "Rebase workaround: forcing baseline_rev to $linux_next_stable" - baseline_rev=$linux_next_stable - rebase_workaround=true -fi + echo "Rebase workaround: forcing baseline_rev to $linux_next_stable" + baseline_rev=$linux_next_stable + rebase_workaround=true + ;; +esac # Build baseline that we are going to re-use to speed-up bisection. # (This also confirms that infrastructure is OK.) -echo "Testing baseline_rev $baseline_rev (should be success)" +echo "Testing baseline_branch $baseline_branch (should be success)" $build_script \ ^^ $reproduce_bisect \ %% $rel_artifacts/manifests/build-baseline.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ ==rr[mode] "baseline" \ - ==rr[current_branch] "$baseline_rev" \ + ==rr[${current_project}_branch] "$baseline_branch" \ ==rr[reset_baseline] true \ ==rr[top_artifacts] "$rel_artifacts/build-baseline" \ --verbose "$verbose" -assert ! [ -f $artifacts/failures ] - -cd $current_project +baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}" +cat <<EOF | manifest_out +declare -g baseline_rev=$baseline_rev +EOF ln -f -s "build-baseline" "$artifacts/build-$baseline_rev" ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh" @@ -124,6 +121,7 @@ $build_script \ %% $rel_artifacts/manifests/build-\$rev.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ ==rr[mode] bisect \ + ==rr[${current_project}_branch] \$rev \ ==rr[top_artifacts] $rel_artifacts/build-\$rev \ --verbose "$verbose" & res=0 && wait \$! || res=\$? @@ -138,6 +136,13 @@ fi EOF chmod +x $artifacts/test.sh +bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}" +cat <<EOF | manifest_out +declare -g bad_rev=$bad_rev +EOF + +cd $current_project + git checkout --detach $bad_rev $artifacts/test.sh & res=0 && wait $! || res=$? @@ -147,14 +152,30 @@ if [ x"$res" = x"0" ]; then if $rebase_workaround; then echo "Rebase workaround: no regression between $baseline_rev and $bad_rev" sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name + case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in + tcwg_kernel/llvm-*-next-*:linux) + cat > $artifacts/trigger-build-rebase <<EOF +llvm_branch=baseline +EOF + ;; + tcwg_kernel/gnu-*-next-*:linux) + cat > $artifacts/trigger-build-rebase <<EOF +binutils_branch=baseline +gcc_branch=baseline +EOF + ;; + *) assert false ;; + esac + cat >> $artifacts/trigger-build-rebase <<EOF +linux_branch=$baseline_rev +reset_baseline=true +EOF else echo "WARNING: build for bad_rev $bad_rev succeeded" - # Build job had a spurious failure. Re-try. - cat > $artifacts/trigger-2-build-master <<EOF -current_project=$current_project -current_branch=$bad_rev -EOF sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name + # Retry build with default parameters + cat > $artifacts/trigger-build-retry <<EOF +EOF fi echo > $artifacts/jenkins/mail-recipients.txt trap "" EXIT @@ -204,10 +225,10 @@ if [ x"$res" = x"0" ]; then # We need to be careful to avoid re-trigger loops, so verify that # last_good is an ancestor of bad_rev. assert git merge-base --is-ancestor $last_good $bad_rev - cat > $artifacts/trigger-0-bisect-again <<EOF + cat > $artifacts/trigger-bisect <<EOF current_project=$current_project -baseline_rev=$baseline_rev -bad_rev=$last_good +baseline_branch=$baseline_rev +bad_branch=$last_good EOF # Don't send any emails. echo > $artifacts/jenkins/mail-recipients.txt @@ -271,6 +292,10 @@ Could not identify regression in $current_project for CI configuration ${rr[ci_c EOF fi +# In log scan for errors below +# - ": error:" detects compiler errors from GCC and Clang +# - "^ERROR:" detects linker errors +# - "] Error " detects GNU make errors cat >> $artifacts/jenkins/mail-body.txt <<EOF Configuration details: $(cat $artifacts/manifests/build-baseline.sh | grep '_url]\|_branch]' | grep -v '="no_') @@ -282,7 +307,7 @@ to (for $bad_name == $bad_sha1) $(cat $artifacts/build-$bad_sha1/results) First few errors in logs of $bad_name: -$(grep ": error:" $artifacts/build-$bad_sha1/console.log | head) +$(grep ": error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head) Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/ Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/ @@ -337,29 +362,22 @@ fi # Set mail recipients last to preserve catch-error value from .yaml file. # Email developers. CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org" -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then - case "$toolchain:$current_project" in - gnu:linux) ;; - gnu:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;; - llvm:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;; - llvm:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;; - esac -fi +case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in + tcwg_kernel/gnu-*:linux) ;; + tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;; + tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;; + tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;; +esac cat > $artifacts/jenkins/mail-recipients.txt <<EOF $CI_MAIL_RECIPIENTS EOF # Reset baseline to the regressed commit so that we will catch subsequent # regressions (worse than $bad_rev). -cat > $artifacts/trigger-1-reset-baseline <<EOF -current_project=$current_project -current_branch=$first_bad -reset_baseline=true -EOF +cp $artifacts/build-$first_bad/trigger-build-$current_project $artifacts/trigger-build-1-reset +echo "reset_baseline=true" >> $artifacts/trigger-build-1-reset # Trigger master build now instead of waiting for next timed SCM trigger. -cat > $artifacts/trigger-2-build-master <<EOF -current_project=$current_project -EOF +cp $artifacts/build-$bad_rev/trigger-build-$current_project $artifacts/trigger-build-2-default trap "" EXIT |