diff options
author | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2021-09-10 12:44:38 +0000 |
---|---|---|
committer | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2021-09-10 12:44:38 +0000 |
commit | 645c96436cbef9ee42ba5c39a37d32345bc77198 (patch) | |
tree | e155328da91fc7d0261661dee65268e071afd51c | |
parent | 765e0768ba6dcc91114a35219a56dc7c8435217d (diff) |
round-robin-bisect.sh: Re-structure email body generation
Change-Id: I9993da061612d67aa375b5cde4a3b3c324e52e70
-rwxr-xr-x | round-robin-bisect.sh | 203 |
1 files changed, 75 insertions, 128 deletions
diff --git a/round-robin-bisect.sh b/round-robin-bisect.sh index 9d8a1c3c..3008b649 100755 --- a/round-robin-bisect.sh +++ b/round-robin-bisect.sh @@ -627,57 +627,37 @@ cp $artifacts/build-$bad_rev/trigger-build-$current_project \ # Save BISECT_* logs find "$current_project" -path "$current_project/.git/BISECT_*" -print0 | xargs -0 -I@ mv @ $artifacts/git-logs/ -# Remove any fail-safe email body -rm -f $artifacts/jenkins/mail-body.txt - if [ x"$first_bad" != x"" ]; then - mkdir -p $artifacts/jenkins sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name ln -f -s "build-$first_bad" "$artifacts/build-first_bad" ln -f -s "build-$first_bad.sh" "$artifacts/manifests/build-first_bad.sh" - good_name="last_good" - good_sha1="$last_good" - bad_name="first_bad" - bad_sha1="$first_bad" - ln -f -s "build-$last_good" "$artifacts/build-last_good" ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh" - - occurences="$(cat $current_project/$interesting_commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")" - if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then - notify_author=$(git -C $current_project log --pretty="%an <%ae>" -n 1 \ - "$first_bad") - fi - - cat >> $artifacts/jenkins/mail-body.txt <<EOF -Identified regression in *$current_project:$first_bad* -$(git -C $current_project log --pretty=short -n 1 $first_bad) - -EOF else - good_name="baseline_rev" - good_sha1="$baseline_rev" - bad_name="bad" - bad_sha1="$bad_rev" - cat >> $artifacts/jenkins/mail-body.txt <<EOF -Could not identify regression in *$current_project* in CI configuration ${rr[ci_project]}/${rr[ci_config]}. See 'Bisect log' in the links below for bisection details. - -EOF + notify_devs=false fi case "${rr[ci_project]}" in tcwg_gcc_check*|tcwg_gnu_*_check_*) # We are building up lists of flaky tests and we need to stabilize - # GCC and GDB testsuites. Currently dejagnu testsuites are too flaky - # for automated regression detection. + # GCC and GDB testsuites. Currently dejagnu testsuites are too + # flaky for automated regression detection. notify_devs=false - notify_author="" ;; esac -if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$first_bad" != x"" ]; then +# Remove any fail-safe email body +rm -f $artifacts/jenkins/mail-body.txt + +if $notify_devs; then + occurences="$(cat $current_project/$interesting_commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")" + if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then + notify_author=$(git -C $current_project log --pretty="%an <%ae>" -n 1 \ + "$first_bad") + fi + case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;; tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;; @@ -695,10 +675,14 @@ if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$first_bad" != x"" ]; then *) jira_card="GNU-692" ;; esac - if $notify_devs && [ x"$jira_card" != x"" ]; then + if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$jira_card" != x"" ]; then cat > $artifacts/jenkins/jira-body.txt <<EOF [$jira_card] -$(cat $artifacts/jenkins/mail-body.txt) +Identified regression caused by *$current_project:$first_bad*: +$(git -C $current_project log --pretty=short -n 1 $first_bad) + +This commit has regressed these CI configurations: +$occurences Details: ${BUILD_URL}artifact/$rel_artifacts/jenkins/mail-body.txt/*view*/ Even more details: ${BUILD_URL}artifact/$rel_artifacts/ @@ -720,62 +704,33 @@ EOF >> $artifacts/jenkins/mail-body.txt fi fi -fi -cat >> $artifacts/jenkins/mail-body.txt <<EOF -Results regressed to (for $bad_name == $bad_sha1) -$(cat $artifacts/build-$bad_sha1/results) + cat >> $artifacts/jenkins/mail-body.txt <<EOF +Identified regression caused by *$current_project:$first_bad*: +$(git -C $current_project log --pretty=short -n 1 $first_bad) -from (for $good_name == $good_sha1) -$(cat $artifacts/build-$good_sha1/results) +Results regressed to (for first_bad == $first_bad) +$(cat $artifacts/build-$first_bad/results) -EOF +from (for last_good == $last_good) +$(cat $artifacts/build-$last_good/results) -good_build_artifacts="Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/" -bad_build_artifacts="Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/" - -# Benchmark builds have a results ID that is used to download the results -if test -f "$artifacts/build-$good_sha1/results_id"; then - cat >> $artifacts/jenkins/mail-body.txt <<EOF -$good_build_artifacts -Results ID of $good_name: $(cat $artifacts/build-$good_sha1/results_id) -$bad_build_artifacts -Results ID of $bad_name: $(cat $artifacts/build-$bad_sha1/results_id) -EOF -# Os vs Os LTO builds have two result IDs per job -elif test -f "$artifacts/build-$good_sha1/results_id-1"; then - cat >> $artifacts/jenkins/mail-body.txt <<EOF -$good_build_artifacts -Results IDs of $good_name: -$(cat $artifacts/build-$good_sha1/results_id-1) -$(cat $artifacts/build-$good_sha1/results_id-2) -$bad_build_artifacts -Results ID of $bad_name: -$(cat $artifacts/build-$bad_sha1/results_id-1) -$(cat $artifacts/build-$bad_sha1/results_id-2) -EOF -else - cat >> $artifacts/jenkins/mail-body.txt <<EOF -$good_build_artifacts -$bad_build_artifacts -EOF -fi - -cat >> $artifacts/jenkins/mail-body.txt <<EOF -Build top page/logs: ${BUILD_URL} - -Configuration details: -CI project/config: ${rr[ci_project]}/${rr[ci_config]} -This commit also regressed: +This commit has regressed these CI configurations: $occurences +Artifacts of last_good build: ${BUILD_URL}artifact/$rel_artifacts/build-$last_good/ +Artifacts of first_bad build: ${BUILD_URL}artifact/$rel_artifacts/build-$first_bad/ +Even more details: ${BUILD_URL}artifact/$rel_artifacts/ + Reproduce builds: <cut> -mkdir investigate-$current_project-$bad_sha1 -cd investigate-$current_project-$bad_sha1 +mkdir investigate-$current_project-$first_bad +cd investigate-$current_project-$first_bad +# Fetch scripts git clone https://git.linaro.org/toolchain/jenkins-scripts +# Fetch manifests and test.sh script mkdir -p $rel_artifacts/manifests curl -o $rel_artifacts/manifests/build-baseline.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-baseline.sh --fail curl -o $rel_artifacts/manifests/build-parameters.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-parameters.sh --fail @@ -791,78 +746,70 @@ rsync -a --del --delete-excluded ${baseline_exclude[@]} ./ ./bisect/baseline/ cd $current_project -# Reproduce $bad_name build -git checkout --detach $bad_sha1 +# Reproduce first_bad build +git checkout --detach $first_bad ../$rel_artifacts/test.sh -# Reproduce $good_name build -git checkout --detach $good_sha1 +# Reproduce last_good build +git checkout --detach $last_good ../$rel_artifacts/test.sh cd .. </cut> -History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/${rr[ci_project]}/${rr[ci_config]} - -Artifacts: ${BUILD_URL}artifact/$rel_artifacts/ -Build log: ${BUILD_URL}consoleText -EOF - -if [ x"$first_bad" != x"" ]; then - cat >> $artifacts/jenkins/mail-body.txt <<EOF - Full commit (up to 1000 lines): <cut> $(git -C $current_project show --stat --patch $first_bad | head -n 1000) </cut> EOF -fi -# Set mail recipients last to preserve catch-error value from .yaml file. -# Email developers. -CI_MAIL_RECIPIENTS=("bcc:tcwg-validation@linaro.org") -case "$notify_author@${rr[ci_project]}/${rr[ci_config]}:$current_project" in - ""@*/*:*) ;; - *@tcwg_gcc*/*:*) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") - ;; - *@tcwg_gnu*/*:*) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") - ;; - *@tcwg_kernel/llvm-*:linux) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") - CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev") - CI_MAIL_RECIPIENTS+=("arnd@linaro.org") - ;; - *@tcwg_kernel/llvm-*:llvm) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") - CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev") - ;; - *@tcwg_kernel/*:*) - CI_MAIL_RECIPIENTS+=("$notify_author") - CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") - ;; - *@tcwg_bmk*/*:*) + # Set mail recipients last to preserve catch-error value from .yaml file. + # Email developers. + CI_MAIL_RECIPIENTS=("bcc:tcwg-validation@linaro.org") + case "$notify_author@${rr[ci_project]}/${rr[ci_config]}:$current_project" in + ""@*/*:*) ;; + *@tcwg_gcc*/*:*) + CI_MAIL_RECIPIENTS+=("$notify_author") + CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") + ;; + *@tcwg_gnu*/*:*) + CI_MAIL_RECIPIENTS+=("$notify_author") + CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") + ;; + *@tcwg_kernel/llvm-*:linux) + CI_MAIL_RECIPIENTS+=("$notify_author") + CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") + CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev") + CI_MAIL_RECIPIENTS+=("arnd@linaro.org") + ;; + *@tcwg_kernel/llvm-*:llvm) + CI_MAIL_RECIPIENTS+=("$notify_author") + CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") + CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev") + ;; + *@tcwg_kernel/*:*) + CI_MAIL_RECIPIENTS+=("$notify_author") + CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org") + ;; + *@tcwg_bmk*/*:*) # Don't notify patch authors until we improve benchmarking email # reporting. #CI_MAIL_RECIPIENTS+=("$notify_author") CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") ;; - *@*/*:*) - CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") - ;; -esac -if $notify_devs; then + *@*/*:*) + CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org") + ;; + esac + ( IFS="," cat > $artifacts/jenkins/mail-recipients.txt <<EOF ${CI_MAIL_RECIPIENTS[*]} EOF ) +else + echo > $artifacts/jenkins/mail-recipients.txt fi trap "" EXIT |