summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-09-10 12:44:38 +0000
committerMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-09-10 12:44:38 +0000
commit645c96436cbef9ee42ba5c39a37d32345bc77198 (patch)
treee155328da91fc7d0261661dee65268e071afd51c
parent765e0768ba6dcc91114a35219a56dc7c8435217d (diff)
round-robin-bisect.sh: Re-structure email body generation
Change-Id: I9993da061612d67aa375b5cde4a3b3c324e52e70
-rwxr-xr-xround-robin-bisect.sh203
1 files changed, 75 insertions, 128 deletions
diff --git a/round-robin-bisect.sh b/round-robin-bisect.sh
index 9d8a1c3c..3008b649 100755
--- a/round-robin-bisect.sh
+++ b/round-robin-bisect.sh
@@ -627,57 +627,37 @@ cp $artifacts/build-$bad_rev/trigger-build-$current_project \
# Save BISECT_* logs
find "$current_project" -path "$current_project/.git/BISECT_*" -print0 | xargs -0 -I@ mv @ $artifacts/git-logs/
-# Remove any fail-safe email body
-rm -f $artifacts/jenkins/mail-body.txt
-
if [ x"$first_bad" != x"" ]; then
- mkdir -p $artifacts/jenkins
sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name
ln -f -s "build-$first_bad" "$artifacts/build-first_bad"
ln -f -s "build-$first_bad.sh" "$artifacts/manifests/build-first_bad.sh"
- good_name="last_good"
- good_sha1="$last_good"
- bad_name="first_bad"
- bad_sha1="$first_bad"
-
ln -f -s "build-$last_good" "$artifacts/build-last_good"
ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh"
-
- occurences="$(cat $current_project/$interesting_commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")"
- if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then
- notify_author=$(git -C $current_project log --pretty="%an <%ae>" -n 1 \
- "$first_bad")
- fi
-
- cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Identified regression in *$current_project:$first_bad*
-$(git -C $current_project log --pretty=short -n 1 $first_bad)
-
-EOF
else
- good_name="baseline_rev"
- good_sha1="$baseline_rev"
- bad_name="bad"
- bad_sha1="$bad_rev"
- cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Could not identify regression in *$current_project* in CI configuration ${rr[ci_project]}/${rr[ci_config]}. See 'Bisect log' in the links below for bisection details.
-
-EOF
+ notify_devs=false
fi
case "${rr[ci_project]}" in
tcwg_gcc_check*|tcwg_gnu_*_check_*)
# We are building up lists of flaky tests and we need to stabilize
- # GCC and GDB testsuites. Currently dejagnu testsuites are too flaky
- # for automated regression detection.
+ # GCC and GDB testsuites. Currently dejagnu testsuites are too
+ # flaky for automated regression detection.
notify_devs=false
- notify_author=""
;;
esac
-if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$first_bad" != x"" ]; then
+# Remove any fail-safe email body
+rm -f $artifacts/jenkins/mail-body.txt
+
+if $notify_devs; then
+ occurences="$(cat $current_project/$interesting_commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")"
+ if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then
+ notify_author=$(git -C $current_project log --pretty="%an <%ae>" -n 1 \
+ "$first_bad")
+ fi
+
case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;;
tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;;
@@ -695,10 +675,14 @@ if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$first_bad" != x"" ]; then
*) jira_card="GNU-692" ;;
esac
- if $notify_devs && [ x"$jira_card" != x"" ]; then
+ if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$jira_card" != x"" ]; then
cat > $artifacts/jenkins/jira-body.txt <<EOF
[$jira_card]
-$(cat $artifacts/jenkins/mail-body.txt)
+Identified regression caused by *$current_project:$first_bad*:
+$(git -C $current_project log --pretty=short -n 1 $first_bad)
+
+This commit has regressed these CI configurations:
+$occurences
Details: ${BUILD_URL}artifact/$rel_artifacts/jenkins/mail-body.txt/*view*/
Even more details: ${BUILD_URL}artifact/$rel_artifacts/
@@ -720,62 +704,33 @@ EOF
>> $artifacts/jenkins/mail-body.txt
fi
fi
-fi
-cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Results regressed to (for $bad_name == $bad_sha1)
-$(cat $artifacts/build-$bad_sha1/results)
+ cat >> $artifacts/jenkins/mail-body.txt <<EOF
+Identified regression caused by *$current_project:$first_bad*:
+$(git -C $current_project log --pretty=short -n 1 $first_bad)
-from (for $good_name == $good_sha1)
-$(cat $artifacts/build-$good_sha1/results)
+Results regressed to (for first_bad == $first_bad)
+$(cat $artifacts/build-$first_bad/results)
-EOF
+from (for last_good == $last_good)
+$(cat $artifacts/build-$last_good/results)
-good_build_artifacts="Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/"
-bad_build_artifacts="Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/"
-
-# Benchmark builds have a results ID that is used to download the results
-if test -f "$artifacts/build-$good_sha1/results_id"; then
- cat >> $artifacts/jenkins/mail-body.txt <<EOF
-$good_build_artifacts
-Results ID of $good_name: $(cat $artifacts/build-$good_sha1/results_id)
-$bad_build_artifacts
-Results ID of $bad_name: $(cat $artifacts/build-$bad_sha1/results_id)
-EOF
-# Os vs Os LTO builds have two result IDs per job
-elif test -f "$artifacts/build-$good_sha1/results_id-1"; then
- cat >> $artifacts/jenkins/mail-body.txt <<EOF
-$good_build_artifacts
-Results IDs of $good_name:
-$(cat $artifacts/build-$good_sha1/results_id-1)
-$(cat $artifacts/build-$good_sha1/results_id-2)
-$bad_build_artifacts
-Results ID of $bad_name:
-$(cat $artifacts/build-$bad_sha1/results_id-1)
-$(cat $artifacts/build-$bad_sha1/results_id-2)
-EOF
-else
- cat >> $artifacts/jenkins/mail-body.txt <<EOF
-$good_build_artifacts
-$bad_build_artifacts
-EOF
-fi
-
-cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Build top page/logs: ${BUILD_URL}
-
-Configuration details:
-CI project/config: ${rr[ci_project]}/${rr[ci_config]}
-This commit also regressed:
+This commit has regressed these CI configurations:
$occurences
+Artifacts of last_good build: ${BUILD_URL}artifact/$rel_artifacts/build-$last_good/
+Artifacts of first_bad build: ${BUILD_URL}artifact/$rel_artifacts/build-$first_bad/
+Even more details: ${BUILD_URL}artifact/$rel_artifacts/
+
Reproduce builds:
<cut>
-mkdir investigate-$current_project-$bad_sha1
-cd investigate-$current_project-$bad_sha1
+mkdir investigate-$current_project-$first_bad
+cd investigate-$current_project-$first_bad
+# Fetch scripts
git clone https://git.linaro.org/toolchain/jenkins-scripts
+# Fetch manifests and test.sh script
mkdir -p $rel_artifacts/manifests
curl -o $rel_artifacts/manifests/build-baseline.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-baseline.sh --fail
curl -o $rel_artifacts/manifests/build-parameters.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-parameters.sh --fail
@@ -791,78 +746,70 @@ rsync -a --del --delete-excluded ${baseline_exclude[@]} ./ ./bisect/baseline/
cd $current_project
-# Reproduce $bad_name build
-git checkout --detach $bad_sha1
+# Reproduce first_bad build
+git checkout --detach $first_bad
../$rel_artifacts/test.sh
-# Reproduce $good_name build
-git checkout --detach $good_sha1
+# Reproduce last_good build
+git checkout --detach $last_good
../$rel_artifacts/test.sh
cd ..
</cut>
-History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}
-
-Artifacts: ${BUILD_URL}artifact/$rel_artifacts/
-Build log: ${BUILD_URL}consoleText
-EOF
-
-if [ x"$first_bad" != x"" ]; then
- cat >> $artifacts/jenkins/mail-body.txt <<EOF
-
Full commit (up to 1000 lines):
<cut>
$(git -C $current_project show --stat --patch $first_bad | head -n 1000)
</cut>
EOF
-fi
-# Set mail recipients last to preserve catch-error value from .yaml file.
-# Email developers.
-CI_MAIL_RECIPIENTS=("bcc:tcwg-validation@linaro.org")
-case "$notify_author@${rr[ci_project]}/${rr[ci_config]}:$current_project" in
- ""@*/*:*) ;;
- *@tcwg_gcc*/*:*)
- CI_MAIL_RECIPIENTS+=("$notify_author")
- CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
- ;;
- *@tcwg_gnu*/*:*)
- CI_MAIL_RECIPIENTS+=("$notify_author")
- CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
- ;;
- *@tcwg_kernel/llvm-*:linux)
- CI_MAIL_RECIPIENTS+=("$notify_author")
- CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
- CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev")
- CI_MAIL_RECIPIENTS+=("arnd@linaro.org")
- ;;
- *@tcwg_kernel/llvm-*:llvm)
- CI_MAIL_RECIPIENTS+=("$notify_author")
- CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
- CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev")
- ;;
- *@tcwg_kernel/*:*)
- CI_MAIL_RECIPIENTS+=("$notify_author")
- CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
- ;;
- *@tcwg_bmk*/*:*)
+ # Set mail recipients last to preserve catch-error value from .yaml file.
+ # Email developers.
+ CI_MAIL_RECIPIENTS=("bcc:tcwg-validation@linaro.org")
+ case "$notify_author@${rr[ci_project]}/${rr[ci_config]}:$current_project" in
+ ""@*/*:*) ;;
+ *@tcwg_gcc*/*:*)
+ CI_MAIL_RECIPIENTS+=("$notify_author")
+ CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
+ ;;
+ *@tcwg_gnu*/*:*)
+ CI_MAIL_RECIPIENTS+=("$notify_author")
+ CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
+ ;;
+ *@tcwg_kernel/llvm-*:linux)
+ CI_MAIL_RECIPIENTS+=("$notify_author")
+ CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
+ CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev")
+ CI_MAIL_RECIPIENTS+=("arnd@linaro.org")
+ ;;
+ *@tcwg_kernel/llvm-*:llvm)
+ CI_MAIL_RECIPIENTS+=("$notify_author")
+ CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
+ CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev")
+ ;;
+ *@tcwg_kernel/*:*)
+ CI_MAIL_RECIPIENTS+=("$notify_author")
+ CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
+ ;;
+ *@tcwg_bmk*/*:*)
# Don't notify patch authors until we improve benchmarking email
# reporting.
#CI_MAIL_RECIPIENTS+=("$notify_author")
CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org")
;;
- *@*/*:*)
- CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org")
- ;;
-esac
-if $notify_devs; then
+ *@*/*:*)
+ CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org")
+ ;;
+ esac
+
(
IFS=","
cat > $artifacts/jenkins/mail-recipients.txt <<EOF
${CI_MAIL_RECIPIENTS[*]}
EOF
)
+else
+ echo > $artifacts/jenkins/mail-recipients.txt
fi
trap "" EXIT