summaryrefslogtreecommitdiff
path: root/round-robin-notify.sh
diff options
context:
space:
mode:
Diffstat (limited to 'round-robin-notify.sh')
-rwxr-xr-xround-robin-notify.sh2414
1 files changed, 2414 insertions, 0 deletions
diff --git a/round-robin-notify.sh b/round-robin-notify.sh
new file mode 100755
index 00000000..f95c4969
--- /dev/null
+++ b/round-robin-notify.sh
@@ -0,0 +1,2414 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+scripts=$(dirname $0)
+
+# shellcheck source=jenkins-helpers.sh
+. $scripts/jenkins-helpers.sh
+
+# DEBUG
+echo -e "\n$0 $*\n"
+
+convert_args_to_variables "$@"
+shift "$SHIFT_CONVERTED_ARGS"
+
+obligatory_variables rr[top_artifacts] notify build_script
+declare -A rr
+declare notify build_script
+
+generate_all="${generate_all-true}"
+generate_jira="${generate_jira-$generate_all}"
+generate_mail="${generate_mail-$generate_all}"
+generate_jenkins_html=${generate_jenkins_html-$generate_all}
+generate_dashboard="${generate_dashboard-$generate_all}"
+generate_lnt="${generate_lnt-$generate_all}"
+post_all="${post_all-true}"
+post_jira_comment="${post_jira_comment-$post_all}"
+post_jira_card="${post_jira_card-$post_all}"
+post_mail="${post_mail-$post_all}"
+post_gcc_testresults="${post_gcc_testresults-$post_mail}"
+post_icommits="${post_icommits-$post_all}"
+post_dashboard="${post_dashboard-$post_all}"
+
+dryrun="${dryrun-false}"
+icommits="interesting-commits"
+stage="${stage-full}"
+verbose="${verbose-false}"
+
+# Generate notification files based on current and previous artifacts, which
+# are assumed to be from two consecutive builds.
+#
+# Notes:
+# 1. Current artifacts are in $top_artifacts. All files from $top_artifacts
+# are accessible read-only, and round-robin-notify.sh writes its files
+# into $top_artifacts/notify/.
+# 2. Files in "numbered" directories NN-* should not be used, since they
+# will be eventually removed.
+# 3. Previous artifacts are in base-artifacts/, which is a git repository.
+# All files from base-artifacts/ are accessible read-only.
+# 4. Round-robin-notify.sh can assume that files in $top_artifacts/ and
+# in base-artifacts/ have been generated by the latest version of scripts.
+# Developers will run round-robin-rewrite.sh when format of files changes
+# to bring "history" results up-to-date with what current scripts expect.
+# 5. Round-robin-notify.sh can assume that check_regression() step was run
+# just before invocation of round-robin-notify.sh.
+
+if $verbose; then
+ set -x
+fi
+
+if $dryrun; then
+ dryrun="echo DRYRUN: "
+else
+ dryrun=""
+fi
+
+
+#========================================================================================
+#
+# SETUP/FINALIZE PROCEDURES
+#
+#========================================================================================
+
+# affect the following global variables
+declare top_artifacts ci_project ci_config notify_email
+setup_notify_environment ()
+{
+ echo "# ${FUNCNAME[0]}"
+
+ # setup global variables
+ top_artifacts="${rr[top_artifacts]}"
+ ci_project=$(get_current_manifest "{rr[ci_project]}")
+ ci_config=$(get_current_manifest "{rr[ci_config]}")
+
+ # Debug dump
+ echo "# Debug traces :"
+ echo "# Baseline : $(get_baseline_manifest BUILD_URL)"
+ echo "# Using dir : base-artifacts"
+ echo "# Artifacts : $(get_current_manifest BUILD_URL)"
+ echo "# Using dir : $top_artifacts"
+ echo ""
+
+ mkdir -p "$top_artifacts/notify"
+
+ case "$notify" in
+ *"@"*)
+ # Normally pw[PROJECT_patch_submitter] will have been set to
+ # "$notify" by precommit-ssh-apply.sh or pw-apply.sh. Manually
+ # started jobs where ${component}_git doesn't start with "ssh://" or
+ # "pw://" are an exception though, because then those scripts aren't
+ # called and thus the $pw array will be empty. Let's save the value
+ # here in case it's needed later.
+ notify_email="$notify"
+ notify="precommit"
+ ;;
+ esac
+
+ declare -Ag pw
+ if [ "$notify" = "precommit" ]; then
+ obligatory_variables pw_dir
+ declare -g pw_dir
+
+ local pw_file
+ while IFS= read -r -d '' pw_file; do
+ # shellcheck disable=SC1090
+ source "$pw_file"
+ done < <(find "$pw_dir" -type f -print0)
+ fi
+}
+
+# Exit with 0 status if given component has a single commit compared to baseline.
+# $1: component
+single_commit_p ()
+{
+ (
+ set -euf -o pipefail
+
+ local c="$1"
+
+ local base_rev cur_rev sha1
+ base_rev=$(get_baseline_git "${c}_rev")
+ cur_rev=$(get_current_git "${c}_rev")
+
+ for sha1 in $(git_component_cmd "$c" rev-parse "$cur_rev^@"); do
+ if [ "$sha1" = "$base_rev" ]; then
+ # We have a single-commit build
+ return 0
+ fi
+ done
+
+ return 1
+ )
+}
+
+# check_source_changes : Looks at base-artifacts and artifacts to compute the
+# changes for the toolchain source files.
+# This procedure updates the following global variables for the rest of notify pass.
+declare change_kind changed_single_component last_good first_bad
+declare -a changed_components
+
+check_source_changes ()
+{
+ echo "# ${FUNCNAME[0]}"
+
+ # Set ${changed_components[@]} unless this is "init" build. For "init"
+ # builds we have no baseline_git data, so leave changed_components empty.
+ if [ "$(get_current_manifest "{rr[update_baseline]}")" != "init" ]; then
+ IFS=" " read -r -a changed_components <<< "$(print_changed_components)"
+ else
+ changed_components=()
+ fi
+
+ local c base_rev cur_rev c_commits
+ if [ ${#changed_components[@]} = 0 ]; then
+ change_kind="no_change"
+ changed_single_component=""
+ elif [ ${#changed_components[@]} = 1 ]; then
+ changed_single_component="${changed_components[0]}"
+
+ first_bad="$(get_current_git ${changed_single_component}_rev)"
+ last_good="$(get_baseline_git ${changed_single_component}_rev)"
+
+ # single_commit_p() depend on $c to exist
+ local res
+ git_component_cmd "$changed_single_component" rev-parse --verify "HEAD" \
+ &>/dev/null &
+ res=0 && wait $! || res=$?
+ assert_with_msg "Cannot parse HEAD in repo $changed_single_component" \
+ [ $res = 0 ]
+
+ if single_commit_p "$changed_single_component"; then
+ change_kind="single_commit"
+ else
+ change_kind="single_component"
+ fi
+ else
+ change_kind="multiple_components"
+ changed_single_component=""
+ fi
+
+ # Debug dump
+ echo "# Debug traces :"
+ echo "# change_kind=$change_kind : ${changed_components[*]}"
+ for c in "${changed_components[@]}"; do
+ base_rev=$(get_baseline_git ${c}_rev)
+ cur_rev=$(get_current_git ${c}_rev)
+ c_commits=$(git_component_cmd "$c" rev-list --count $base_rev..$cur_rev || echo "??")
+ echo "# rev for $c : $base_rev..$cur_rev ($c_commits commits)"
+ done
+ echo ""
+}
+
+# print routines pointers
+declare print_commits_f print_result_f print_config_f print_last_icommit_f
+
+setup_stages_to_run ()
+{
+ # if everything is fine. No reason to report to jira & icommits
+ if [ "$notify" = "onregression" ] \
+ && { [ "${rr[no_regression_result]}" = "0" ] \
+ || [ "$change_kind" != "single_commit" ]; }; then
+ notify=ignore
+ elif [ "$notify" = "precommit" ] \
+ && [ "${rr[no_regression_result]}" = "0" ]; then
+ notify=ignore
+ fi
+
+ if [ "$notify" = "ignore" ] || [ "$notify" = "precommit" ]; then
+ post_jira_comment=false
+ post_jira_card=false
+ post_icommits=false
+ # Even in "ignore" mode (successful build with no regression),
+ # we want to post gcc_testresults if relevant.
+ if [ "$notify" = "ignore" ]; then
+ post_mail=false
+ fi
+ if [ "$notify" = "precommit" ]; then
+ post_gcc_testresults=false
+ fi
+ fi
+
+ # Disable dashboard generation as results-vs-first is now disabled
+ generate_dashboard=false
+ post_dashboard=false
+
+ # print routines pointers
+ print_commits_f=print_commits
+ print_result_f=print_result
+ print_config_f=print_config
+ print_last_icommit_f=print_last_icommit
+ generate_extra_details_f=generate_extra_details
+ case "$ci_project" in
+ tcwg_binutils*|tcwg_bootstrap*|tcwg_gcc*|tcwg_gdb*|tcwg_glibc*|tcwg_gnu*)
+ print_result_f=gnu_print_result
+ generate_extra_details_f=gnu_generate_extra_details
+ print_config_f=gnu_print_config
+ ;;
+ tcwg_bmk*)
+ print_result_f=bmk_print_result
+ print_config_f=bmk_print_config
+ generate_extra_details_f=bmk_generate_extra_details
+ ;;
+ *)
+ # By default keep generic ones set above
+ ;;
+ esac
+}
+
+release_notification_files ()
+{
+ echo "# ${FUNCNAME[0]}"
+ if ! [ -d $top_artifacts/jenkins ]; then
+ return
+ fi
+
+ local f
+ for f in mail-body.txt mail-subject.txt mail-recipients.txt; do
+ # copy the file if exists, and not emtpy.
+ # this important for mail-recipient.txt that may be empty.
+ if [ -s $top_artifacts/notify/$f ]; then
+ cp $top_artifacts/notify/$f $top_artifacts/jenkins/$f
+ fi
+ done
+
+ echo "... Done"
+}
+
+release_gcc_testresults_files ()
+{
+ echo "# ${FUNCNAME[0]}"
+ if ! [ -d $top_artifacts/jenkins ]; then
+ return
+ fi
+
+ # Send GCC test results only if the baseline was updated more than
+ # 1 day ago, so that we send at most one such email per day.
+ if [ -f $top_artifacts/testresults/testresults-mail-body.txt ]; then
+ base_d=$(get_baseline_manifest "{rr[gcc_testresults_date]}")
+ cur_d=$(get_current_component_date gcc || true)
+ if [ x"$cur_d" = x"" ]; then
+ return
+ fi
+ if [ x"$base_d" = x"" ]; then
+ base_d=0
+ fi
+
+ if [ $(( $cur_d - $base_d )) -ge 86400 ]; then
+ # The current date will become the baseline for the next build
+ # if the current baseline occurred more than one day ago.
+ cp $top_artifacts/testresults/testresults-mail-subject.txt \
+ $top_artifacts/testresults/testresults-mail-body.txt $top_artifacts/jenkins/
+ echo "gcc-testresults@gcc.gnu.org" > $top_artifacts/jenkins/testresults-mail-recipients.txt
+ cat <<EOF | manifest_out
+rr[gcc_testresults_date]=$cur_d
+EOF
+ else
+ # Otherwise, record the baseline date as the current
+ # gcc_testresults_date. Failing that, the next build will
+ # compare its date with 0 and may send its results more
+ # often than every day.
+ cat <<EOF | manifest_out
+rr[gcc_testresults_date]=$base_d
+EOF
+ fi
+ fi
+
+ echo "... Done"
+}
+
+#========================================================================================
+#
+# GENERATE EXTRA DETAILS
+#
+#========================================================================================
+gnu_generate_extra_details()
+{
+ (
+ set -euf -o pipefail
+
+ # Extract 'configure' and 'make' lines for steps where it makes sense.
+ local tmpfile
+ tmpfile=$(mktemp)
+
+ echo > $top_artifacts/notify/configure-make.txt
+
+ while read step_console
+ do
+ artifact_dir="$(dirname ${step_console})"
+ (
+ # We want to accept that 'xzgrep' can have zero match below
+ set +o pipefail
+ xzgrep RUN: ${step_console} | sed 's/.* RUN: //' > $tmpfile
+ )
+
+ grep /configure $tmpfile > $tmpfile-configure || true
+ grep "^make " $tmpfile > $tmpfile-make || true
+ if [ -s $tmpfile-configure ] || [ -s $tmpfile-make ]; then
+ echo "# $(basename $artifact_dir)" >> $top_artifacts/notify/configure-make.txt
+ cat $tmpfile-configure $tmpfile-make >> $top_artifacts/notify/configure-make.txt
+ echo >> $top_artifacts/notify/configure-make.txt
+ fi
+ done < <(set +f ; find $top_artifacts/[0-9][0-9]-* -name console.log.xz)
+
+ rm -f $tmpfile $tmpfile-configure $tmpfile-make
+
+ [ -s $top_artifacts/notify/configure-make.txt ] || rm -f $top_artifacts/notify/configure-make.txt
+
+ if ! [ -d $top_artifacts/sumfiles ]; then
+ return 0
+ fi
+
+ # Compare results using compare_tests for information purposes.
+ # It works well only when we compare complete testsuites.
+ # shellcheck disable=SC2154
+ gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
+ base-artifacts/sumfiles \
+ "$top_artifacts/sumfiles" \
+ > $top_artifacts/notify/results.compare.txt &
+ wait $! || true
+ )
+}
+
+bmk_generate_extra_details()
+{
+ (
+ set -euf -o pipefail
+ local artifacts_mail_dir
+
+ artifacts_mail_dir=$top_artifacts/notify
+
+ $scripts/../bmk-scripts/output-bmk-results.py \
+ --compare_results $top_artifacts/results-vs-prev/compare-results-internal.csv \
+ --variability_file $top_artifacts/results-vs-prev/bmk-specific-variability-avg.csv \
+ --variability_file_data "avg" \
+ --run_step_dir "$artifacts_mail_dir/" \
+ --metric "${rr[metric_id]}" --mode "${rr[mode]}" \
+ --details verbose > "$artifacts_mail_dir/output-bmk-results.log" &
+
+ local res=0 && wait $! || res=$?
+ assert_with_msg "ERROR while trying to regenerate bmk-data results. Aborting.." [ $res = 0 ]
+ )
+}
+
+generate_extra_details()
+{
+ (
+ # nothing to do
+ true
+ )
+}
+
+#========================================================================================
+#
+# PRINT ROUTINES
+#
+#========================================================================================
+
+dump_model_only=${dump_model_only-false}
+
+#==========================================
+# *GENERIC* PRINT ROUTINES
+#==========================================
+
+###### PRINT COMMITS
+# --link : link to the commit (only if single_commit)
+# --oneline : commit title
+# --short : commit log
+print_commits()
+{
+ (
+ set -euf -o pipefail
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+ local print_arg=$1
+
+ if [ "$change_kind" = "no_change" ]; then
+ echo "baseline build"
+ return 0
+ fi
+
+ local more_lines
+ if [ "$change_kind" = "single_commit" ]; then
+ local c="${changed_single_component}"
+
+ if [ "$print_arg" = "--link" ]; then
+ local url
+ url=$(get_baseline_git ${c}_url)
+ if [[ "$url" =~ git://sourceware.org/git/ ]]; then
+ url="${url#git://sourceware.org/git/}"
+ url="https://sourceware.org/git/?p=$url"
+ echo "$url;a=commitdiff;h=$first_bad"
+ elif [[ "$url" =~ https://github.com/ ]] \
+ || [[ "$url" =~ https://gitlab.com/ ]]; then
+ echo "${url%.git}/commit/$first_bad"
+ elif [[ "$url" =~ https://git.linaro.org/ ]]; then
+ echo "${url}/commit/?id=$first_bad"
+ else
+ echo "See in $url"
+ fi
+
+ return 0
+ fi
+
+ local describe
+ if [ "${pw[$c]-}" = "" ]; then
+ describe=$(describe_sha1 "$c" "$first_bad" true)
+ # Remove the leading "basepoints/"
+ describe=$(echo "$describe" | sed 's,^basepoints/,,')
+ else
+ describe="$c patch #${pw[${c}_patch_id]}"
+ fi
+
+ if [ "$print_arg" = "--oneline" ]; then
+ echo "$describe"
+ return 0
+ fi
+
+ if [ "${pw[$c]-}" = "" ]; then
+ echo "commit $describe"
+ else
+ echo "$c patch ${pw[${c}_patch_url]}"
+ fi
+ local tmpfile
+ tmpfile=$(mktemp)
+ git -C "$c" log -n1 "$first_bad" | tail -n +2 > "$tmpfile"
+ head -n 10 $tmpfile
+ more_lines=$(($(cat "$tmpfile" | wc -l) - 10))
+ if [ $more_lines -gt 0 ]; then
+ echo "... $more_lines lines of the commit log omitted."
+ fi
+ rm $tmpfile
+
+ if [ "${pw[$c]-}" != "" ]; then
+ echo "... applied on top of baseline commit:"
+ git -C $c log -n1 --oneline $last_good || true
+ fi
+ return 0
+ fi
+
+ if [ "$change_kind" = "single_component" ] \
+ || [ "$change_kind" = "multiple_components" ]; then
+ local new_commits c base_rev cur_rev c_commits components
+
+ local commits_or_patches
+ if [ "${pw[project]-}" != "" ]; then
+ commits_or_patches="patches"
+ else
+ commits_or_patches="commits"
+ fi
+
+ new_commits=0
+ for c in "${changed_components[@]}"; do
+ base_rev=$(get_baseline_git ${c}_rev)
+ cur_rev=$(get_current_git ${c}_rev)
+ c_commits=$(git_component_cmd "$c" rev-list --count $base_rev..$cur_rev \
+ || echo 0)
+ new_commits=$(($new_commits + $c_commits))
+ done
+ components=$(echo "${changed_components[@]}" | tr ' ' ',')
+
+ echo "$new_commits $commits_or_patches in $components"
+
+ if [ "$print_arg" = "--oneline" ]; then
+ return 0
+ fi
+
+ for c in "${changed_components[@]}"; do
+ base_rev=$(get_baseline_git ${c}_rev)
+ cur_rev=$(get_current_git ${c}_rev)
+ c_commits=$(git_component_cmd "$c" rev-list --count $base_rev..$cur_rev \
+ || echo 0)
+
+ if [ "${pw[$c]-}" != "" ]; then
+ echo "Patchwork URL: ${pw[${c}_patch_url]}"
+ fi
+
+ git -C $c log -n 5 --oneline $base_rev..$cur_rev || true
+ if [ $c_commits -gt 5 ]; then
+ echo "... and $(($c_commits-5)) more $commits_or_patches in $c"
+ fi
+
+ if [ "${pw[$c]-}" != "" ]; then
+ echo "... applied on top of baseline commit:"
+ git -C $c log -n1 --oneline $base_rev || true
+ fi
+ done
+
+ return 0
+ fi
+ )
+}
+
+###### PRINT the RESULTS of this build
+# --oneline : either success ot failure
+# --short : change of results.txt file between baseline and artifact
+# --long : idem
+print_result()
+{
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+
+ local print_arg=$1
+ case "$print_arg" in
+ --oneline)
+ if [ "${rr[no_regression_result]}" = "0" ]; then
+ echo "Success"
+ else
+ echo "Failure"
+ fi
+ ;;
+ --short|--long)
+ echo "Results changed to"
+ echo "$(cat $top_artifacts/results)"
+ echo ""
+ echo "From"
+ echo "$(cat base-artifacts/results)"
+ ;;
+ esac
+}
+
+###### PRINT LAST ICOMMIT
+# Extract the last status from icommit
+# --entry : output directory of icommit entry
+# --status : will output the icommit status
+# --reproduction_instructions_link : will output the link to the reproduction_instructions
+print_last_icommit ()
+{
+ (
+ $dump_model_only && echo "<<${FUNCNAME[0]}>>" && return
+ set -euf -o pipefail
+ local print_arg="$1"
+ shift 1
+
+ if [ x"$change_kind" != x"single_commit" ]; then
+ return 0
+ fi
+
+ local isubdir
+ isubdir=$(interesting_subdir "$changed_single_component" "$first_bad" "$@")
+
+ case "$print_arg" in
+ --entry)
+ echo "$icommits/$isubdir"
+ ;;
+ --status)
+ cat "$icommits/$isubdir/status.txt"
+ ;;
+ --reproduction_instructions_link)
+ print_icommits_link "$isubdir/reproduction_instructions.txt"
+ ;;
+ esac
+ )
+}
+
+# Print link url to interesting-commits.git repo for "$path".
+print_icommits_link ()
+{
+ (
+ set -euf -o pipefail
+ local path="$1"
+
+ local url="https://git-us.linaro.org/toolchain/ci/interesting-commits.git"
+ echo "$url/plain/$path"
+ )
+}
+
+# CHECK_IF_FIRST_REPORT : Check if this is the first report in icommits
+# Result stored in first_icommit_to_report global variable
+check_if_first_report()
+{
+ declare -g first_icommit_to_report
+
+ first_icommit_to_report=false
+ if [ x"$change_kind" != x"single_commit" ]; then
+ return
+ fi
+
+ local isubdir
+ isubdir=$(interesting_subdir "$changed_single_component" "$first_bad")
+ if ! [ -f "$icommits/$isubdir/first_url" ]; then
+ return
+ fi
+
+ local first_url
+ first_url=$(cat "$icommits/$isubdir/first_url")
+ if [ "$first_url" = "$(get_current_manifest BUILD_URL)" ]; then
+ first_icommit_to_report=true
+ elif [ "$notify" = "onregression" ]; then
+ # Send out emails for post-commit "onregression" reports only for
+ # the first detection.
+ post_mail=false
+ fi
+}
+
+###### PRINT the configuration of this build
+# --oneline : target short name (eg: arm/aarch64)
+# --short : short "pretty" version suitable for summary
+# --long : full details
+print_config()
+{
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+
+ local print_arg=$1
+ case "$print_arg" in
+ --oneline)
+ case "$ci_config" in
+ *arm*) echo "arm" ;;
+ *aarch64*) echo "aarch64" ;;
+ *) echo "$ci_config" ;;
+ esac
+ ;;
+ --short|--long)
+ echo "CI config $ci_project/$ci_config"
+ ;;
+ esac
+}
+
+print_artifacts_url ()
+{
+ (
+ set -euf -o pipefail
+
+ local url
+ url="$(get_current_manifest BUILD_URL)artifact/artifacts"
+ if [ "${pw[project]-}" != "" ]; then
+ url="$url/artifacts.precommit"
+ fi
+ echo "$url/$*"
+ )
+}
+
+#==========================================
+# *GNU* PRINT ROUTINES
+#==========================================
+
+###### PRINT the configuration of this build
+# --oneline : target short name (eg: arm/aarch64)
+# --short : short "pretty" version suitable for summary
+# --long : full details
+gnu_print_config()
+{
+ (
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+
+ # shellcheck source=tcwg_gnu-config.sh
+ . $scripts/tcwg_gnu-config.sh
+
+ settings_for_ci_project_and_config "$ci_project" "$ci_config"
+
+ local print_arg=$1
+ case "$print_arg" in
+ --oneline)
+ print_config "$print_arg"
+ ;;
+ --short)
+ echo "${gnu_data[pretty_project]} ${gnu_data[pretty_config]}"
+ ;;
+ --long)
+ echo "CI config $ci_project ${gnu_data[long_config]}"
+ ;;
+ esac
+ )
+}
+
+# most of them mapped to generic implementation
+gnu_print_result()
+{
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+ local print_arg="$1"
+
+ if ! [ -d $top_artifacts/sumfiles ]; then
+ print_result "$@"
+ return 0
+ fi
+
+ # From now on, we should have called already gnu_generate_extra_details
+
+ local validate_failures="gcc-compare-results/contrib/testsuite-management/validate_failures.py"
+ local xfails="$top_artifacts/sumfiles/xfails.xfail"
+
+ if ! [ -f "$xfails" ]; then
+ return 0
+ fi
+
+ "$validate_failures" --manifest="$xfails" \
+ --expiry_date="${rr[result_expiry_date]}" \
+ --build_dir="$top_artifacts/sumfiles" --verbosity=1 \
+ > $top_artifacts/notify/regressions.sum &
+ wait $! || true
+ "$validate_failures" --inverse_match --manifest="$xfails" \
+ --expiry_date="${rr[result_expiry_date]}" \
+ --build_dir="$top_artifacts/sumfiles" --verbosity=1 \
+ > $top_artifacts/notify/progressions.sum &
+ wait $! || true
+
+ grep -A10 "=== Results Summary ===" $top_artifacts/notify/regressions.sum \
+ > $top_artifacts/notify/results-summary.txt
+
+ local n_regressions n_progressions pass_fail=PASS
+ if [ "${rr[no_regression_result]}" != "0" ]; then
+ pass_fail=FAIL
+ fi
+ n_regressions=$(grep -c "^[A-Z]\+:" \
+ $top_artifacts/notify/regressions.sum || true)
+ n_progressions=$(grep -c "^[A-Z]\+:" \
+ $top_artifacts/notify/progressions.sum || true)
+
+ printf "$pass_fail"
+ if [ "$n_regressions" != "0" ]; then
+ printf ": $n_regressions regressions"
+ else
+ rm $top_artifacts/notify/regressions.sum
+ fi
+ if [ "$n_progressions" != "0" ]; then
+ printf ": $n_progressions progressions"
+ else
+ rm $top_artifacts/notify/progressions.sum
+ fi
+ printf "\n"
+
+ if [ "$print_arg" = "--oneline" ]; then
+ return 0
+ fi
+
+ local length=10 outfile n_lines
+ if [ "$print_arg" = "--long" ]; then
+ # Ask "head" to print out all the lines except for the last 0 lines.
+ length=-0
+ fi
+
+ for outfile in regressions.sum progressions.sum; do
+ if ! [ -f $top_artifacts/notify/$outfile ]; then
+ continue
+ fi
+
+ echo
+ echo "$outfile:"
+ # Copy contents until "Results Summary" line (excluded)
+ n_lines=$(grep -n "Results Summary" $top_artifacts/notify/$outfile \
+ | cut -d: -f1)
+ n_lines=$(($n_lines - 1))
+
+ if [ $n_lines -lt $length ]; then
+ length=$n_lines
+ fi
+
+ head -n$length $top_artifacts/notify/$outfile
+
+ n_lines=$(($n_lines - $length))
+ if [ $n_lines -gt 0 ] && [ $length != -0 ]; then
+ echo "... and $n_lines more entries"
+ fi
+ done
+
+ cat <<EOF
+
+You can find the failure logs in *.log.1.xz files in
+ - $(print_artifacts_url 00-sumfiles/)
+The full lists of regressions and progressions as well as configure and make commands are in
+ - $(print_artifacts_url notify/)
+The list of [ignored] baseline and flaky failures are in
+ - $(print_artifacts_url sumfiles/xfails.xfail)
+EOF
+}
+
+
+#==========================================
+# *BMK* PRINT ROUTINES
+#==========================================
+
+# print_result. either oneline version or short/long.
+# Here is an ex :
+# --oneline :
+# 644.nab_s slowed down of 4%
+# --short/long :
+# the following benchmarks slowed down by more than 3%:
+# - 644.nab_s slowed down by 4% from 112963 to 117189 perf samples
+bmk_print_result()
+{
+ (
+ set -euf -o pipefail
+
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+
+ local print_arg=$1
+
+ artifacts_mail_dir=$top_artifacts/notify
+
+ if [ "$stage" != "full" ]; then
+ return
+ fi
+
+ ## Prepare data
+
+ # From now on, we should have called already bmk_generate_extra_details
+
+ # If there's one regression. Don't bother about improvements.
+ local improved_or_regressed
+ if [ -f $artifacts_mail_dir/exe.regression ] || [ -f $artifacts_mail_dir/symbol.regression ]; then
+ improved_or_regressed=regression
+ else
+ improved_or_regressed=improvement
+ fi
+
+ declare -A changed_by_msg
+ changed_by_msg[size-regression]="grew in size by"
+ changed_by_msg[size-improvement]="reduced in size by"
+ changed_by_msg[sample-regression]="slowed down by"
+ changed_by_msg[sample-improvement]="speeds up by"
+ changed_by_msg[num_vect_loops-regression]="reduced the number of vect loops by"
+ changed_by_msg[num_vect_loops-improvement]="increased the number of vect loops by"
+ changed_by_msg[num_sve_loops-regression]="reduced the number of sve instructions by"
+ changed_by_msg[num_sve_loops-improvement]="increased the number of sve instructions by"
+ changed_by=${changed_by_msg[${rr[metric_id]}-$improved_or_regressed]}
+
+ # FIXME: Remove hard-coded thresholds
+ # thresholds
+ case ${rr[metric_id]} in
+ size)
+ exe_threshold=1 # We use 1% tolerance for binary size
+ symbol_threshold=10 # and 10% tolerance for symbol size.
+ ;;
+ sample)
+ # Reduce thresholds when bisecting to avoid considering borderline
+ # regressions as spurious. This should break cycles of build and
+ # bisect jobs triggering each other on borderline regressions.
+ exe_threshold=3
+ symbol_threshold=15
+ ;;
+ num_vect_loops|num_sve_loops)
+ exe_threshold=0
+ symbol_threshold=0
+ ;;
+ *) assert false ;;
+ esac
+
+ # Now print result
+ case "$print_arg" in
+ --oneline)
+ assert_with_msg "Builds with infra problems should never get here" \
+ [ "${rr[no_regression_result]}" != "$EXTERNAL_FAIL" ]
+
+ # Generate readable oneline diag
+ local metric bmk symbol short_diag long_diag
+ if [ -f $artifacts_mail_dir/exe.$improved_or_regressed ]; then
+ # shellcheck disable=SC2034
+ IFS=, read metric bmk symbol short_diag long_diag < <(head -n1 $artifacts_mail_dir/exe.$improved_or_regressed)
+ elif [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
+ # shellcheck disable=SC2034
+ IFS=, read metric bmk symbol short_diag long_diag < <(head -n1 $artifacts_mail_dir/symbol.$improved_or_regressed)
+ else
+ short_diag="No change"
+ fi
+ echo "$short_diag"
+ ;;
+
+ --short|--long)
+ # The following exe regressed/improved:
+ if [ -f $artifacts_mail_dir/exe.$improved_or_regressed ]; then
+ sort -gr -o $artifacts_mail_dir/exe.$improved_or_regressed \
+ $artifacts_mail_dir/exe.$improved_or_regressed
+
+ echo "the following benchmarks $changed_by more than ${exe_threshold}%:"
+ local metric exe symbol short_diag long_diag
+ while IFS=, read metric exe symbol short_diag long_diag; do
+ echo "- $long_diag"
+ if [ -f $artifacts_mail_dir/$exe.symbols-$improved_or_regressed ]; then
+ while IFS=, read metric bmk symbol short_diag long_diag; do
+ echo " - $long_diag"
+ done < $artifacts_mail_dir/$exe.symbols-$improved_or_regressed
+ # Delete $bmk.regressions so that it doesn't show up
+ # in symbol-regression loop below.
+ rm $artifacts_mail_dir/$exe.symbols-$improved_or_regressed
+ fi
+ done < $artifacts_mail_dir/exe.$improved_or_regressed
+ fi
+
+ # The following functions regressed/improved:
+ if [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
+ echo "the following hot functions $changed_by more than ${symbol_threshold}% (but their benchmarks $changed_by less than ${exe_threshold}%):"
+ local metric bmk symbol short_diag long_diag
+ # shellcheck disable=SC2034
+ while IFS=, read metric bmk symbol short_diag long_diag; do
+ echo "- $long_diag"
+ done < $artifacts_mail_dir/symbol.$improved_or_regressed
+ fi
+
+ if ! [ -f $artifacts_mail_dir/exe.$improved_or_regressed ] && ! [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
+ echo "No change"
+ fi
+ ;;
+ esac
+ )
+}
+
+###### PRINT the configuration of this build
+# --oneline : target short name (eg: arm/aarch64)
+# --short : short "pretty" version suitable for summary
+# --long : full details
+bmk_print_config()
+{
+ # shellcheck source=tcwg_bmk-config.sh
+ . $scripts/tcwg_bmk-config.sh
+
+ $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
+
+ # ${ci_project}--${ci_config} format is :
+ # 'tcwg_bmk-#{PROFILE_NAME}-#{BMK}--#{TOOLCHAIN}-#{TARGET}-{toolchain_ver}-{cflags}'
+ IFS=- read -a ci_pjt_cfg <<EOF
+$ci_project--$ci_config
+EOF
+ local toolchain target cflags
+ toolchain="${ci_pjt_cfg[4]}"
+ target="${ci_pjt_cfg[5]}"
+ cflags="${ci_pjt_cfg[7]}"
+
+ local compiler="" libc="" linker="" version="" bmk_flags="" hw=""
+ bmk_flags=$(echo "$cflags" | sed -e "s/_/ -/g")
+
+ local print_arg=$1
+ case "$print_arg" in
+ --oneline)
+ case "$ci_config" in
+ *arm*) echo "arm $bmk_flags" ;;
+ *aarch64*) echo "aarch64 $bmk_flags" ;;
+ *) echo "$ci_config" ;;
+ esac
+ return 0
+ ;;
+ --short)
+ print_config "$print_arg"
+ return 0
+ ;;
+ esac
+
+ # --long as default
+
+ local bmk_suite publish_save_temps
+ bmk_suite=""
+ publish_save_temps=false
+ case "$(tcwg_bmk_benchs)" in
+ coremark)
+ bmk_suite="EEMBC CoreMark"
+ ;;
+ spec2k6|4*)
+ bmk_suite="SPEC CPU2006"
+ publish_save_temps=true
+ ;;
+ spec2017*|5*|6*)
+ bmk_suite="SPEC CPU2017"
+ publish_save_temps=true
+ ;;
+ esac
+
+ cat <<EOF
+Below reproducer instructions can be used to re-build both "first_bad" and "last_good" cross-toolchains used in this bisection. Naturally, the scripts will fail when triggerring benchmarking jobs if you don\'t have access to Linaro TCWG CI.
+EOF
+
+ # Copy save-temps tarballs to artifacts, so that they are accessible.
+ # We can publish pre-processed source only for benchmarks derived from
+ # open-source projects.
+ # Note that we include save-temps artifacts for successful builds so that
+ # "last_good" build has the artifacts.
+ if $publish_save_temps; then
+ mkdir -p ${top_artifacts}/top-artifacts
+ local s_t
+ while read s_t; do
+ case "$s_t" in
+ 400.perlbench*|500.perlbench*|600.perlbench*) ;;
+ 401.bzip2*) ;;
+ 403.gcc*|502.gcc*|602.gcc*) ;;
+ 435.gromacs*) ;;
+ 436.cactusADM*|507.cactuBSSN*|607.cactuBSSN*) ;;
+ 445.gobmk*) ;;
+ 454.calculix*) ;;
+ 456.hmmer*) ;;
+ 462.libquantum*) ;;
+ 465.tonto*) ;;
+ 481.wrf*|521.wrf*|621.wrf*) ;;
+ 482.sphinx3*) ;;
+ 483.xalanc*) ;;
+ 505.mcf*|605.mcf*)
+ # 429.mcf is not present in redistributable_sources/ :-|
+ ;;
+ 511.povray*) ;;
+ 525.x264*|625.x264*)
+ # 464.h264ref is not present in redistributable_sources/ :-|
+ ;;
+ 526.blender*) ;;
+ 527.cam4*|627.cam4*) ;;
+ 538.imagick*|638.imagick*) ;;
+ 544.nab*|644.nab*) ;;
+ 554.roms*|654.roms*) ;;
+ 557.xz*|657.xz*) ;;
+ 628.pop2) ;;
+ *)
+ # Can't redistribute benchmark sources.
+ continue
+ ;;
+ esac
+ cp "$s_t" ${top_artifacts}/top-artifacts/save-temps/
+ done < <(find results-1 -path "save.*.temps/*.tar.xz")
+ fi
+
+ if [ -d ${top_artifacts}/top-artifacts/save-temps/ ]; then
+ cat <<EOF
+
+For your convenience, we have uploaded tarballs with pre-processed source and assembly files at:
+- First_bad save-temps: \$FIRST_BAD_ARTIFACTS/save-temps/
+- Last_good save-temps: \$LAST_GOOD_ARTIFACTS/save-temps/
+- Baseline save-temps: \$BASELINE_ARTIFACTS/save-temps/
+EOF
+ fi
+
+ case "$toolchain" in
+ gnu)
+ compiler="GCC"
+ libc="Glibc"
+ linker="GNU Linker"
+ ;;
+ gnu_eabi)
+ compiler="GCC"
+ libc="Newlib"
+ linker="GNU LD"
+ ;;
+ llvm)
+ compiler="Clang"
+ libc="Glibc"
+ linker="LLVM Linker"
+ ;;
+ esac
+ case "$ci_config" in
+ *-master-*) version="tip of trunk" ;;
+ *-release-*) version="latest release branch" ;;
+ esac
+ case "$(tcwg_bmk_hw)" in
+ *apm*) hw="APM Mustang 8x X-Gene1" ;;
+ *tk1*) hw="NVidia TK1 4x Cortex-A15" ;;
+ *tx1*) hw="NVidia TX1 4x Cortex-A57" ;;
+ *stm32*) hw="STMicroelectronics STM32L476RGTx 1x Cortex-M4" ;;
+ *fx*) hw="Fujitsu FX700 48x AA64" ;;
+ *qc*) hw="Qualcomm 8x AA64" ;;
+ *) hw="<unknown>"
+ esac
+
+ cat <<EOF
+
+Configuration:
+- Benchmark: $bmk_suite
+- Toolchain: $compiler + $libc + $linker
+- Version: all components were built from their $version
+- Target: $(print_gnu_target $target)
+- Compiler flags: $bmk_flags
+- Hardware: $hw
+
+This benchmarking CI is work-in-progress, and we welcome feedback and suggestions at linaro-toolchain@lists.linaro.org . In our improvement plans is to add support for SPEC CPU2017 benchmarks and provide "perf report/annotate" data behind these reports.
+EOF
+}
+
+#========================================================================================
+#
+# INTERESTING_COMMITS PROCEDURES
+#
+#========================================================================================
+
+# Print out merged version of status-summary.txt files at a given subdir level.
+# As an initial approximation, just pick a summary with the biggest number.
+merge_status_summary ()
+{
+ (
+ set -euf -o pipefail
+ local subdir="$1"
+
+ local cur_file cur best="" best_file
+
+ while read -r cur_file; do
+ # Extract a number " N " or " N%"
+ cur=$(sed -e "s/.* \([0-9]\+\)[ %].*/\1/" "$cur_file")
+ if ! [ "$cur" -le "$best" ]; then
+ best="$cur"
+ best_file="$cur_file"
+ fi
+ done < <(find "$subdir" -mindepth 2 -maxdepth 2 \
+ -name "status-summary.txt" | sort)
+
+ cat "$best_file"
+ )
+}
+
+# update_interesting_commits : will update the local icommit repository with
+# the new run.
+# If it is a regression, single_commit, may be updating first_report variable
+#
+# Interesting-commits store regression history in the following hierarchy:
+# 1. At the top level we have COMPONENT directories.
+# 2. At the 2nd level we have SHA1 directories.
+# 2a. We also have "git describe" symlinks to SHA1 directories for convenience.
+# 3. At the 3rd level we have CI_PROJECT directories.
+# 4. At the 4th level we have CI_CONFIG directories, and ...
+# 4a. ... status.txt, which contains status of changes from SHA1 across all
+# CI_CONFIGs in CI_PROJECT.
+# 5. At the 5th level we have per-build files last_good, details.txt, etc.
+update_interesting_commits ()
+{
+ echo "# ${FUNCNAME[0]}"
+
+ local stage="$1"
+ local jira_key="$2"
+
+ # We have successfully identified a bad commit with a good parent!
+ # Add $first_bad to interesting commits.
+
+ local subdir3 subdir4 subdir5
+ # Commit top-level dir
+ subdir3=$(interesting_subdir "$changed_single_component" "$first_bad")
+ # Commit project-level dir
+ subdir4=$(interesting_subdir "$changed_single_component" "$first_bad" \
+ "$ci_project")
+ # Commit config-level dir
+ subdir5=$(interesting_subdir "$changed_single_component" "$first_bad" \
+ "$ci_project" "$ci_config")
+
+ if ! [ -d "$icommits/$subdir3" ]; then
+ mkdir -p $icommits/$subdir3
+ get_current_manifest BUILD_URL > $icommits/$subdir3/first_url
+ git -C $icommits add $subdir3/first_url
+ fi
+
+ # Update interesting-commits/$component/$first_bad/$ci_project/$ci_config
+ mkdir -p "$icommits/$subdir5"
+ echo "$(get_current_manifest BUILD_URL)artifact/artifacts" > "$icommits/$subdir5/build_url"
+ echo "$last_good" > "$icommits/$subdir5/last_good"
+ git -C "$icommits" add "$subdir5/build_url" "$subdir5/last_good"
+
+ if [ "$stage" != "full" ]; then
+ return
+ fi
+
+ # $icommit/<comp>/sha1/<sha1>/.../status-summary.txt
+ $print_result_f --oneline > "$icommits/$subdir5/status-summary.txt"
+ merge_status_summary $icommits/$subdir4 \
+ > "$icommits/$subdir4/status-summary.txt"
+ merge_status_summary $icommits/$subdir3 \
+ > "$icommits/$subdir3/status-summary.txt"
+ git -C "$icommits" add \
+ "$subdir5/status-summary.txt" \
+ "$subdir4/status-summary.txt" \
+ "$subdir3/status-summary.txt"
+
+ $print_result_f --long > "$icommits/$subdir5/details.txt"
+ if [ -f $top_artifacts/notify/configure-make.txt ]; then
+ cat $top_artifacts/notify/configure-make.txt >> "$icommits/$subdir5/details.txt"
+ fi
+
+ # $icommit/<comp>/sha1/<sha1>/<ci_project>/<ci_config>/status.txt
+ (
+ cat "$icommits/$subdir5/status-summary.txt"
+ print_icommits_link "$subdir5/details.txt"
+ cat $icommits/$subdir5/build_url
+ ) | sed "s/^/* /" > "$icommits/$subdir5/status.txt"
+
+ git -C "$icommits" add "$subdir5/details.txt" "$subdir5/status.txt"
+
+ # FIXME: Remove transisional workaround for summary.txt -> status.txt
+ if [ -f "$icommits/$subdir5/summary.txt" ]; then
+ git -C "$icommits" rm "$subdir5/summary.txt"
+ fi
+
+ # $icommit/<comp>/sha1/<sha1>/<ci_project>/<ci_config>/reproduction_instructions.txt
+ local bad_artifacts_url good_artifacts_url
+ bad_artifacts_url="$(get_current_manifest BUILD_URL)artifact/artifacts"
+ good_artifacts_url="$(get_baseline_manifest BUILD_URL)artifact/artifacts"
+ cat > $icommits/$subdir5/reproduction_instructions.txt << EOF
+mkdir -p investigate-$changed_single_component-$first_bad
+cd investigate-$changed_single_component-$first_bad
+
+# Fetch scripts
+git clone https://git.linaro.org/toolchain/jenkins-scripts
+
+# Fetch manifests for bad and good builds
+mkdir -p bad/artifacts good/artifacts
+curl -o bad/artifacts/manifest.sh $bad_artifacts_url/manifest.sh --fail
+curl -o good/artifacts/manifest.sh $good_artifacts_url/manifest.sh --fail
+
+# Reproduce bad build
+(cd bad; ../jenkins-scripts/${build_script} @@rr[top_artifacts] artifacts)
+# Reproduce good build
+(cd good; ../jenkins-scripts/${build_script} @@rr[top_artifacts] artifacts)
+EOF
+ git -C "$icommits" add $subdir5/reproduction_instructions.txt
+
+ # $icommit/<comp>/sha1/<sha1>/<ci_project>/status.txt
+ local ci_config
+ while read ci_config; do
+ # FIXME: Remove transisional workaround for summary.txt -> status.txt
+ if [ -f "$icommits/$subdir4/$ci_config/summary.txt" ]; then
+ echo "* $ci_config"
+ cat $icommits/$subdir4/$ci_config/summary.txt | sed "s/^/** /"
+ continue
+ fi
+
+ if ! [ -f "$icommits/$subdir4/$ci_config/status.txt" ]; then
+ continue
+ fi
+
+ echo "* $ci_config"
+ cat $icommits/$subdir4/$ci_config/status.txt | sed "s/^/*/"
+ done < <(cd $icommits/$subdir4; ls) > "$icommits/$subdir4/status.txt"
+ git -C "$icommits" add "$subdir4/status.txt"
+
+ # $icommit/<comp>/sha1/<sha1>/status.txt
+ local ci_project
+ while read ci_project; do
+ if ! [ -f "$icommits/$subdir3/$ci_project/status.txt" ]; then
+ continue
+ fi
+
+ echo "* $ci_project"
+ cat $icommits/$subdir3/$ci_project/status.txt | sed "s/^/*/"
+ done < <(cd $icommits/$subdir3; ls) > "$icommits/$subdir3/status.txt"
+ git -C "$icommits" add "$subdir3/status.txt"
+
+ # $icommit/<comp>/sha1/<sha1>/commit-log.txt
+ $print_commits_f --short > "$icommits/$subdir3/commit-log.txt"
+ git -C "$icommits" add "$subdir3/commit-log.txt"
+
+ if $generate_jira; then
+ local jira_dir="$subdir3/jira"
+
+ if [ -f "$icommits/$jira_dir/key" ]; then
+ assert_with_msg "Should not have created multiple jira cards" \
+ [ "$jira_key" = "" ]
+ jira_key=$(cat "$icommits/$jira_dir/key")
+ fi
+
+ # Recreate jira/ dir with up-to-date info.
+ if [ -e "$icommits/$jira_dir" ]; then
+ git -C "$icommits" rm -rf "$jira_dir"
+ fi
+ mkdir "$icommits/$jira_dir"
+
+ if [ "$jira_key" != "" ]; then
+ echo "$jira_key" > "$icommits/$jira_dir/key"
+ git -C "$icommits" add "$jira_dir/key"
+ fi
+
+ # Keep jira/summary in sync with mail-subject.txt
+ echo "$($print_commits_f --oneline):" \
+ "$(cat "$icommits/$subdir3/status-summary.txt")" \
+ > "$icommits/$jira_dir/summary"
+ git -C "$icommits" add "$jira_dir/summary"
+
+ # FIXME: Unify format with email
+ # We stop posting updates to jira cards once they are closed to avoid
+ # spamming developers about resolved issues. Therefore, jira data in
+ # interesting-commits.git repo may be more up-to-date than in
+ # the actual jira cards. We add a link to jira/yaml in the card
+ # description to make it easy for developers to look at the latest
+ # info for closed cards.
+ cat > "$icommits/$jira_dir/description" <<EOF
+Commit: $($print_commits_f --link)
+$(cat "$icommits/$subdir3/commit-log.txt")
+
+$(cat "$icommits/$subdir3/status.txt")
+
+Latest data: $(print_icommits_link "$jira_dir/yaml")
+EOF
+ git -C "$icommits" add "$jira_dir/description"
+
+ update_jira_card
+ fi
+
+ # Generate a $describe_sha1 symlink
+ local describe
+ describe=$(describe_sha1 "$changed_single_component" "$first_bad" false)
+ if [ "$describe" != "" ]; then
+ local d
+ d=$(dirname "$describe")
+ mkdir -p $icommits/$changed_single_component/$d
+ local symlink=""
+ while [ "$d" != "." ]; do
+ symlink="../$symlink"
+ d=$(dirname "$d")
+ done
+ symlink="${symlink}sha1/$first_bad"
+ # ??? For some reason I don't understand overwriting symlink with
+ # "ln -sf" may create a second [broken] symlink.
+ rm -f $icommits/$changed_single_component/$describe
+ ln -s $symlink $icommits/$changed_single_component/$describe
+ git -C $icommits add $changed_single_component/$describe
+ fi
+}
+
+
+# Generate entry in interesting-commits.git and, if $post_icommits,
+# push it. This is applicable only for single-commit case.
+# This is called twice -- first time with $stage == init, and second time
+# with $stage == full.
+# During "init" stage we fetch existing entries and create a very basic entry
+# if there isn't one. This allows us to determine in check_if_first_report()
+# whether this is the first build to discover $first_bad as interesting commit.
+post_interesting_commits ()
+{
+ (
+ set -euf -o pipefail
+ echo "# ${FUNCNAME[0]}"
+
+ local stage="$1"
+
+ if [ "$change_kind" != "single_commit" ]; then
+ return
+ fi
+
+ # Clone interesting-commits.git repo, which contains a regression
+ # summaries for SHA1s. These are the "first_bad" commits.
+ clone_or_update_repo $icommits master \
+ https://git-us.linaro.org/toolchain/ci/interesting-commits.git \
+ auto master
+
+ if ! $post_icommits; then
+ dryrun="echo DRYRUN: "
+ fi
+
+ local jira_dir jira_key=""
+ jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
+ jira_dir="$jira_dir/jira"
+ if [ "$stage" = "full" ] && $post_jira_card && $first_icommit_to_report \
+ && [ "$dryrun" = "" ]; then
+ if ! [ -f "$icommits/$jira_dir/key" ]; then
+ # Create jira card for this interesting commit.
+ # We first create the card,
+ # then add a link to it in interesting-commits.git,
+ # then we update the card's content every time we change entry in
+ # interesting-commits.
+ jira_key=$(create_jira_card)
+ else
+ echo "WARNING: jira card already exists $icommits/$jira_dir/key:" \
+ "$(cat "$icommits/$jira_dir/key")"
+ fi
+ fi
+
+ # Regenerate interesting-commits entry and try to push to upstream.
+ # - If failed to push, update icommits again
+ # This can happen if another job pushed to interesting-commits just
+ # before this job.
+ # - If successful we consider round-robin-notify.sh to be successful.
+ while true; do
+ # Reset to master branch
+ git -C $icommits remote update -p
+ git_clean $icommits refs/remotes/origin/master
+
+ update_interesting_commits "$stage" "$jira_key"
+
+ # Commit changes (if any) to local clone of interesting-commits.git
+ git -C $icommits commit \
+ -m "Add entry $first_bad from $(get_current_manifest BUILD_URL)" \
+ || break
+
+ $dryrun git -C $icommits push \
+ ssh://git-us.linaro.org/toolchain/ci/interesting-commits.git \
+ HEAD:refs/heads/master &
+ if wait $!; then
+ # Push successful : stop here
+ break
+ fi
+ # Push failed. update icommit and retry pushing
+ done
+ )
+}
+
+#========================================================================================
+#
+# JIRA RELATED PROCEDURES
+#
+#========================================================================================
+
+print_jira_template_card ()
+{
+ # Catch-all case for when project/config IDs change, so that we
+ # won't miss notifications. Forward all that to GNU-692.
+ local jira_card="GNU-692"
+ case "$ci_project/$ci_config:$changed_single_component" in
+ tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;;
+ tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;;
+ tcwg_kernel/llvm-*:linux) jira_card="LLVM-647" ;;
+ tcwg_kernel/llvm-*:*) jira_card="LLVM-646" ;;
+ tcwg_bmk-*_speed*/gnu*) jira_card="GNU-689" ;;
+ tcwg_bmk-*_size*/gnu*) jira_card="GNU-686" ;;
+ tcwg_bmk-*_vect*/gnu*) jira_card="GNU-988" ;;
+ tcwg_bmk-*_sve*/gnu*) jira_card="GNU-988" ;;
+ tcwg_bmk-*_speed*/llvm*) jira_card="LLVM-651" ;;
+ tcwg_bmk-*_size*/llvm*) jira_card="LLVM-650" ;;
+ tcwg_bmk-*_vect*/llvm*) jira_card="LLVM-1013" ;;
+ tcwg_bmk-*_sve*/llvm*) jira_card="LLVM-1013" ;;
+ tcwg_aosp-*/*) jira_card="LLVM-1014" ;;
+ esac
+ echo "$jira_card"
+}
+
+# Create jira card for this interesting commit.
+# Link to this card is stored in $icommits/.../jira/key
+create_jira_card ()
+{
+ (
+ set -euf -o pipefail
+
+ local template project parent assignee yaml
+ template=$(print_jira_template_card)
+ project="${template%%-*}"
+ parent=$(jipsearch -j "key=$template" -s parent:key \
+ | sed -e "s/.* , //")
+ assignee=$(jipsearch -j "key=$template" -s assignee:emailAddress \
+ | sed -e "s/.* , //" || true)
+ if [ "$assignee" = "" ]; then
+ # The template card is unassigned, so use parent's assignee.
+ assignee=$(jipsearch -j "key=$parent" -s assignee:emailAddress \
+ | sed -e "s/.* , //")
+ fi
+
+ yaml=$(mktemp)
+ # shellcheck disable=SC2064
+ trap "rm $yaml" EXIT
+
+ cat > $yaml <<EOF
+- Project: $project
+ IssueType: Sub-task
+ Parent: $parent
+ Summary: $changed_single_component:$first_bad
+ AssigneeEmail: $assignee
+EOF
+ local key
+ key=$(jipcreate -f $yaml | sed -e "s#.*/##")
+ echo "$key"
+ )
+}
+
+# Print the existing jira card number for this interesting commit.
+print_jira_card_key ()
+{
+ (
+ set -euf -o pipefail
+
+ local jira_dir
+ jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
+ jira_dir="$jira_dir/jira"
+
+ if ! [ -f "$icommits/$jira_dir/key" ]; then
+ return 0
+ fi
+
+ cat "$icommits/$jira_dir/key"
+ )
+}
+
+# Update jira card for this interesting commit.
+# Link to this card is stored in $icommits/.../jira/key
+update_jira_card ()
+{
+ (
+ set -euf -o pipefail
+ echo "# ${FUNCNAME[0]}"
+
+ local jira_dir
+ jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
+ jira_dir="$jira_dir/jira"
+
+ local -a components=()
+ case "$changed_single_component" in
+ binutils) components+=(Binutils) ;;
+ gcc) components+=(GCC) ;;
+ gdb) components+=(GDB) ;;
+ glibc) components+=(Glibc) ;;
+ linux) components+=(Linux) ;;
+ llvm) components+=(LLVM) ;;
+ newlib) components+=(Newlib) ;;
+ qemu) components+=(QEMU) ;;
+ *) components+=(CI) ;;
+ esac
+ (IFS=","; echo "${components[*]}") > "$icommits/$jira_dir/components"
+ git -C "$icommits" add "$jira_dir/components"
+
+ local commit_date
+ commit_date=$(git -C "$changed_single_component" log -n1 \
+ --pretty="%cd" --date=iso "$first_bad")
+ date -d "$commit_date" +%Y-%m-%d > "$icommits/$jira_dir/startdate"
+ git -C "$icommits" add "$jira_dir/startdate"
+
+ local key project
+ key=$(print_jira_card_key)
+ if [ -z "$key" ]; then
+ echo "WARNING: no existing jira card $icommits/$jira_dir/key"
+ return 0
+ fi
+
+ project="${key%%-*}"
+
+ local yaml="$icommits/$jira_dir/yaml"
+
+ cat > "$yaml" <<EOF
+- Project: $project
+ IssueType: Sub-task
+ Key: $key
+ Summary: |
+EOF
+ # Summary can have spaces and other special-to-yaml symbols;
+ # quote using " |"
+ sed -e "s/^/ /" "$icommits/$jira_dir/summary" >> "$yaml"
+ cat >> "$yaml" <<EOF
+ Components: $(cat "$icommits/$jira_dir/components")
+ Start date: $(cat "$icommits/$jira_dir/startdate")
+ Description: |
+EOF
+ sed -e "s/^/ /" "$icommits/$jira_dir/description" >> "$yaml"
+ git -C "$icommits" add "$jira_dir/yaml"
+ )
+}
+
+# Generate notify/jira/* files
+generate_jira_dir()
+{
+ (
+ set -euf -o pipefail
+
+ local icommit_entry jira_key=""
+ icommit_entry=$($print_last_icommit_f --entry)
+
+ if [ "$icommit_entry" != "" ] && [ -d "$icommit_entry/jira" ]; then
+ rsync -a "$icommit_entry/jira/" "$top_artifacts/notify/jira/"
+ if [ -f "$top_artifacts/notify/jira/key" ]; then
+ jira_key=$(cat "$top_artifacts/notify/jira/key")
+ fi
+ else
+ mkdir -p "$top_artifacts/notify/jira"
+ fi
+
+ if [ "$jira_key" != "" ]; then
+ cat > $top_artifacts/notify/jira/comment-card.txt <<EOF
+[$jira_key]
+$($print_result_f --oneline)
+Details: $(print_artifacts_url notify/mail-body.txt/*view*/)
+EOF
+ cat > $top_artifacts/notify/jira/comment-template.txt <<EOF
+[$(print_jira_template_card)]
+https://linaro.atlassian.net/browse/$jira_key
+$($print_result_f --oneline)
+Details: $(print_artifacts_url notify/mail-body.txt/*view*/)
+EOF
+ else
+ cat > $top_artifacts/notify/jira/comment-template.txt << EOF
+[$(print_jira_template_card)]
+$($print_result_f --oneline)
+Details: $(print_artifacts_url notify/mail-body.txt/*view*/)
+EOF
+ fi
+ )
+}
+
+# Post update to Jira
+post_to_jira ()
+{
+ (
+ set -euf -o pipefail
+ echo "# ${FUNCNAME[0]}"
+
+ local post_card_comment=$post_jira_comment
+ local post_template_comment=$post_jira_comment
+
+ if $post_jira_card && [ -f $top_artifacts/notify/jira/yaml ]; then
+ local key status
+ key=$(print_jira_card_key)
+ status=$(jipsearch -j "key=$key" -s status:name \
+ | sed -e "s/.* , //")
+
+ # Do not update closed cards to avoid spamming developers about
+ # updating resolved issues. Note that this also skips posting
+ # comments below.
+ case "$status" in
+ "Closed")
+ post_card_comment=false
+ # In this case we may still post a comment to the template
+ # card if $post_jira_comment is true.
+ ;;
+ *)
+ $dryrun jipcreate -f $top_artifacts/notify/jira/yaml
+ post_template_comment=false
+ ;;
+ esac
+ fi
+
+ if $post_card_comment \
+ && [ -f $top_artifacts/notify/jira/comment-card.txt ]; then
+ echo y | $dryrun jipdate \
+ -f $top_artifacts/notify/jira/comment-card.txt
+ fi
+
+ if $post_template_comment \
+ && [ -f $top_artifacts/notify/jira/comment-template.txt ]; then
+ echo y | $dryrun jipdate \
+ -f $top_artifacts/notify/jira/comment-template.txt
+ fi
+ )
+}
+
+
+#========================================================================================
+#
+# MAIL RELATED PROCEDURES
+#
+#========================================================================================
+# Model for mail-recipient.txt (may contain some of the following):
+# <author>, <compiler-mailing-list>, <linaro-toolchain-mailing-list>, <tcwg-validation-mailing-list>
+# (generate_mail_recipients : Generic)
+#
+# Model for mail-subject.txt :
+# [Linaro-TCWG-CI] <diag> after <changes> (generate_mail_subject : Generic)
+#
+# Model for mail-body.txt :
+# <mail regression details> (generate_mail_body_regression : Specific to the kind of project)
+#
+# <reproduction instructions details> (generate_mail_body_reproduction_instructions : Generic)
+#
+
+print_mail_recipients ()
+{
+ (
+ set -euf -o pipefail
+
+ local c="$changed_single_component"
+ if [ "$c" = "" ]; then
+ echo "bcc:tcwg-validation@linaro.org"
+ return 0
+ fi
+
+ local -A emails
+ emails["tcwg-validation@linaro.org"]=bcc
+ emails["author"]=cc
+ emails["committer"]=to
+
+ case "$ci_project/$ci_config:$c" in
+ *_fast_*/*:*)
+ # FIXME: testing maintainer-mode
+ emails["author"]=no
+ emails["committer"]=no
+ emails["christophe.lyon@linaro.org"]=to
+ ;;
+ tcwg_aosp-*/*:*)
+ # FIXME: stabilize and enable notifications.
+ emails["author"]=no
+ emails["committer"]=no
+ emails["antoine.moynault@linaro.org"]=to
+ ;;
+ tcwg_bmk-*/*:*)
+ # FIXME: stabilize and enable notifications.
+ emails["author"]=no
+ emails["committer"]=no
+ emails["maxim.kuvyrkov@linaro.org"]=to
+ ;;
+ tcwg_kernel/llvm-*:linux|tcwg_kernel/*:llvm)
+ emails["author"]=no
+ emails["committer"]=no
+ emails["llvm@lists.linux.dev"]=to_postcommit
+ ;;
+ tcwg_kernel/*:linux)
+ emails["author"]=no
+ emails["committer"]=no
+ emails["linaro-kernel@lists.linaro.org"]=to_postcommit
+ ;;
+ tcwg_kernel/*:*)
+ emails["author"]=no
+ emails["committer"]=no
+ emails["linaro-toolchain@lists.linaro.org"]=to_postcommit
+ ;;
+ */*:gcc)
+ emails["gcc-regression@gcc.gnu.org"]=cc_postcommit
+ ;;
+ */*:gdb)
+ emails["gdb-testers@sourceware.org"]=cc_postcommit
+ ;;
+ */*:*)
+ emails["linaro-toolchain@lists.linaro.org"]=cc_postcommit
+ ;;
+ esac
+
+ local c email base_rev cur_rev
+
+ # CC: author
+ base_rev=$(get_baseline_git "${c}_rev")
+ cur_rev=$(get_current_git "${c}_rev")
+ while read -r email; do
+ emails["$email"]="${emails[author]}"
+ done < <(git_component_cmd "$c" log --pretty='%ae' "$base_rev..$cur_rev" || true)
+
+
+ local precommit_postcommit=postcommit
+ if [ "$notify" != precommit ]; then
+ # TO: committer
+ base_rev=$(get_baseline_git "${c}_rev")
+ cur_rev=$(get_current_git "${c}_rev")
+ while read -r email; do
+ # shellcheck disable=SC2034
+ emails["$email"]="${emails[committer]}"
+ done < <(git_component_cmd "$c" log --pretty='%ce' "$base_rev..$cur_rev" || true)
+ else
+ precommit_postcommit=precommit
+
+ if [ "${notify_email-}" = "" ]; then
+ # TO: precommit submitter
+ # Note that for precommit testing "git log" will specify
+ # tcwg-buildslave@linaro.org as committer, so use patchwork submitter
+ # instead.
+ notify_email="${pw[${c}_patch_submitter]-}"
+ fi
+
+ if [ "${notify_email-}" != "" ]; then
+ emails["$notify_email"]="${emails[committer]}"
+ fi
+ fi
+
+ unset "emails[author]" "emails[committer]"
+
+ local type
+ local -a recipients=()
+ for email in "${!emails[@]}"; do
+ type="${emails[$email]}"
+ case "$precommit_postcommit:$type" in
+ precommit:to_precommit) type=to ;;
+ precommit:to_postcommit) type=no ;;
+ precommit:cc_precommit) type=cc ;;
+ precommit:cc_postcommit) type=no ;;
+ postcommit:to_precommit) type=no ;;
+ postcommit:to_postcommit) type=to ;;
+ postcommit:cc_precommit) type=no ;;
+ postcommit:cc_postcommit) type=cc ;;
+ esac
+
+ case "$type" in
+ no) ;;
+ to) recipients+=("$email") ;;
+ *) recipients+=("$type:$email") ;;
+ esac
+ done
+
+ (IFS=","; echo "${recipients[*]}")
+ )
+}
+
+##### Model for GNU SPECIFIC mail-body-regression.txt regression details
+#
+# After commit <>
+#
+# The following .. slowed down/grew up ..
+#
+# The configuration is ...
+#
+
+##### Generates mail/mail-body.txt file
+print_mail_body()
+{
+ local bad_artifacts_url good_artifacts_url
+ bad_artifacts_url="$(get_current_manifest BUILD_URL)artifact/artifacts"
+ good_artifacts_url="$(get_baseline_manifest BUILD_URL)artifact/artifacts"
+
+ local key=""
+ # We create a Jira card only for single-commit regressions in
+ # post-commit CI
+ if [ "$change_kind" = "single_commit" ] \
+ && [ "${pw[project]-}" = "" ]; then
+ key=$(print_jira_card_key)
+ if [ -z "$key" ]; then
+ key=$(print_jira_template_card)
+ fi
+ fi
+
+ cat << EOF
+Dear contributor, our automatic CI has detected problems related to your \
+patch(es). Please find some details below. If you have any questions, \
+please follow up on linaro-toolchain@lists.linaro.org mailing list, Libera's \
+#linaro-tcwg channel, or ping your favourite Linaro toolchain developer \
+on the usual project channel.
+
+We appreciate that it might be difficult to find the necessary logs or \
+reproduce the issue locally. If you can't get what you need from our \
+CI within minutes, let us know and we will be happy to help.
+
+EOF
+
+ if [ "$key" != "" ]; then
+ cat <<EOF
+We track this report status in https://linaro.atlassian.net/browse/$key , \
+please let us know if you are looking at the problem and/or when you have a fix.
+
+EOF
+ fi
+
+ cat <<EOF
+In $($print_config_f --short) after:
+
+$($print_commits_f --short | sed -e 's/^/ | /')
+
+$($print_result_f --short)
+
+The configuration of this build is:
+$($print_config_f --long)
+
+-----------------8<--------------------------8<--------------------------8<--------------------------
+The information below can be used to reproduce a debug environment:
+
+Current build : $bad_artifacts_url
+Reference build : $good_artifacts_url
+
+EOF
+
+ # FIXME: Remove this warning when we enable maintainer-mode in
+ # production.
+ if [ "${pw[project]-}" != "" ]; then
+ cat <<EOF
+Warning: we do not enable maintainer-mode nor automatically update
+generated files, which may lead to failures if the patch modifies the
+master files.
+
+EOF
+ fi
+
+ if [ "$change_kind" != "single_commit" ] \
+ || [ "${pw[project]-}" != "" ]; then
+ return
+ fi
+
+ cat <<EOF
+Reproduce last good and first bad builds: $($print_last_icommit_f --reproduction_instructions_link "$ci_project" "$ci_config")
+
+Full commit : $($print_commits_f --link)
+
+List of configurations that regressed due to this commit :
+$($print_last_icommit_f --status)
+
+EOF
+}
+
+# Generate notify/mail-*.txt files
+generate_mail_files()
+{
+ (
+ set -euf -o pipefail
+
+ print_mail_recipients > $top_artifacts/notify/mail-recipients.txt
+
+ # Keep jira/summary in sync with mail-subject.txt
+ echo "[Linaro-TCWG-CI]" \
+ "$($print_commits_f --oneline): $($print_result_f --oneline) on $($print_config_f --oneline)" \
+ > $top_artifacts/notify/mail-subject.txt
+
+ print_mail_body > $top_artifacts/notify/mail-body.txt
+ )
+}
+
+print_readme_header()
+{
+ (
+ set -euf -o pipefail
+
+ local text_type="$1"
+
+ local msg="How to browse artifacts of this build"
+ case $text_type in
+ html)
+ cat <<EOF
+<!DOCTYPE html>
+<html>
+<body>
+<font color="black">
+<h2>$msg</h2>
+EOF
+ ;;
+ txt)
+ cat <<EOF
+$msg
+
+EOF
+ ;;
+ esac
+ )
+}
+
+# print MSG as a link, if appropriate
+# $1: type of output (html/txt)
+# $2: message (directory or file name)
+# #3: directory where $2 resides, used to detertime the file type
+print_readme_link()
+{
+ (
+ set -euf -o pipefail
+
+ local text_type="$1"
+ local msg="$2"
+ local home="$3"
+
+ case $text_type in
+ html)
+ # Jenkins webserver need a /*view*/ decoration so that
+ # text files are displayed in the browser.
+ view=""
+ if [ -f "$home/$msg" ]; then
+ if file "$home/$msg" | grep -qw text ; then
+ view="/*view*/"
+ fi
+ fi
+ echo -n "<a href=\"$msg$view\">$msg</a>"
+ ;;
+ txt)
+ echo -n "$msg"
+ ;;
+ esac
+ )
+}
+print_readme_footer()
+{
+ (
+ set -euf -o pipefail
+
+ local text_type="$1"
+ case $text_type in
+ html)
+ cat << EOF
+</body>
+</html>
+EOF
+ ;;
+ esac
+ )
+}
+
+# Provide some hints to users, to help them find their way in our
+# artifacts.
+# $1: type of text (txt, html)
+generate_readme()
+{
+ (
+ set -euf -o pipefail
+
+ local text_type="$1"
+
+ local gnu_text=false
+ case "$ci_project" in
+ *check*)
+ case "$ci_project" in
+ *gnu*|*gcc*|*binutils*|*gdb*|*bootstrap*)
+ gnu_text=true
+ ;;
+ esac
+ ;;
+ esac
+
+ local list_start=""
+ local list_end=""
+ local list_item="- "
+ local new_parag=""
+
+ if [ "$text_type" = "html" ]; then
+ list_start="<ul>"
+ list_end="</ul>"
+ list_item="<li>"
+ new_parag="<p>"
+ fi
+
+ print_readme_header $text_type
+
+ cat << EOF
+The artifact directories contain a lot of information related to the
+results of this build.
+$new_parag
+Directories starting with a number contain the logs of each step of
+the build. More synthetic information is available in other directories,
+as described below:
+$new_parag
+$list_start
+EOF
+
+ if [ -d $top_artifacts/00-sumfiles ]; then
+ cat <<EOF
+$list_item$(print_readme_link "$text_type" "00-sumfiles/" "") contains .log and possibly .sum files generated by the
+ build. Files with .0 suffix contain the results of the initial full
+ testsuite run, files with .1, .2 etc... contain logs restricted to
+ the parts (.exp) of the testsuite where we detected regressions.
+ .1, .2, .... represent the number of times this subset of the testsuite
+ was executed in order to also identify flaky tests. The last one
+ contains what is considered as the results of this build.
+
+EOF
+ fi
+
+ cat <<EOF
+$list_item$(print_readme_link "$text_type" "git/" "") contains the revision and repository of each toolchain
+ component built
+
+$list_item$(print_readme_link "$text_type" "jenkins/" "") contains information useful for the CI maintainers
+
+$list_item$(print_readme_link "$text_type" "notify/" "") contains the material used to build various
+ notifications/reports (emails, Jira, LNT, ...)
+EOF
+
+ if $gnu_text; then
+ cat <<EOF
+
+$list_item$(print_readme_link "$text_type" "sumfiles/" "") contains the .sum files produced by this build.
+EOF
+ fi
+
+ cat <<EOF
+$list_end
+$new_parag
+If you received a notification about one of your patches causing
+problems, the information you received is in $(print_readme_link "$text_type" "notify/" "") and has
+links to other artifacts from this directory.
+EOF
+
+ if $gnu_text; then
+ local regressions=""
+ if [ -f $top_artifacts/notify/regressions.sum ]; then
+ regressions="$(print_readme_link "$text_type" "notify/regressions.sum" "$top_artifacts") and "
+ fi
+ cat <<EOF
+$new_parag
+If you are investigating such a problem, you are probably primarily
+interested in:
+$new_parag
+$list_start
+$list_item$regressions$(print_readme_link "$text_type" "notify/results.compare.txt" "$top_artifacts") (regression report).
+
+EOF
+ if [ -d $top_artifacts/00-sumfiles ]; then
+ cat <<EOF
+$list_item$(print_readme_link "$text_type" "00-sumfiles/" "") .log files with detailed errors, to save
+ yourself reproducing the problem on your machine.
+EOF
+ fi
+
+ cat <<EOF
+$list_end
+EOF
+ fi
+
+ # Print the list of files below top_artifacts, some users find
+ # this useful.
+ cat <<EOF
+$new_parag
+List of files below:
+$new_parag
+$list_start
+EOF
+
+ while read -r cur_file; do
+ echo "$list_item$(print_readme_link "$text_type" "$cur_file" "$top_artifacts")"
+ done < <(cd $top_artifacts ; find . -type f | sort)
+
+ cat <<EOF
+$list_end
+EOF
+
+ print_readme_footer $text_type
+ )
+}
+
+# Procedure to generate a nice html to publish in jenkins job
+generate_jenkins_html_files()
+{
+
+ (
+ set -euf -o pipefail
+
+ echo "# ${FUNCNAME[0]}"
+ if ! $generate_jenkins_html; then
+ echo "... Skipping"
+ return
+ fi
+
+ case "$ci_project" in
+ tcwg_bmk-*)
+ (
+ status_file="$top_artifacts/results-vs-prev/csv-results-1/status.csv"
+ if [ -f $status_file ]; then
+ # status is one of : success, failed-to-build or failed-to-run
+ nb_succeed=$(sort -u $status_file | grep -c ",success$")
+ nb_failed=$(sort -u $status_file | grep -c ",failed-to-")
+ title="$nb_succeed benchmarks succeeded"
+ if [ "$nb_failed" != "0" ]; then
+ title+=", <FONT COLOR=\"orange\">$nb_failed failed<FONT COLOR=\"black\">"
+ fi
+ cat << EOF
+ <!DOCTYPE html>
+ <html>
+ <body>
+
+ <h2>Status of this run : $title</h2>
+
+ <FONT COLOR="orange">
+EOF
+
+ sort -u $status_file | grep ",failed-to-" | cut -d, -f1,3 | \
+ sed -e 's|\(.*\),\(.*\)|<h3> - \1 : \2</h3>|'
+
+ cat << EOF
+ <FONT COLOR="black">
+
+ </body>
+ </html>
+EOF
+ fi
+ ) > $top_artifacts/jenkins/status.html
+ ;;
+ *)
+ # no implementation
+ echo "... Skipping"
+ return
+ ;;
+ esac
+ ) &
+ wait $! || true
+
+ generate_readme html > $top_artifacts/README.html
+ generate_readme txt > $top_artifacts/README.txt
+}
+
+
+#========================================================================================
+#
+# DASHBOARD RELATED PROCEDURES
+#
+#========================================================================================
+# Calculate a reasonable date to associate with the current results / artifacts
+# and prints this date in manifest.sh.
+calculate_results_date ()
+{
+ (
+ set -euf -o pipefail
+
+ local c base_d cur_d results_date=0
+
+ # Firstly, set results_date to the max of commit dates of all components.
+ for c in $(get_current_manifest "{rr[components]}"); do
+ base_d=""
+ if [ -f base-artifacts/git/${c}_rev ]; then
+ base_d=$(get_baseline_component_date ${c} || true)
+ fi
+ cur_d=$(get_current_component_date ${c} || true)
+ if [ x"$base_d" != x"" ]; then
+ if [ x"$cur_d" = x"" ] || [ $cur_d -lt $base_d ]; then
+ cur_d="$base_d"
+ fi
+ fi
+ if [ x"$cur_d" = x"" ]; then
+ continue
+ fi
+
+ if [ $cur_d -gt $results_date ]; then
+ results_date="$cur_d"
+ fi
+ done
+
+ assert_with_msg "Failed to produce results_date" [ $results_date -gt 0 ]
+
+ base_d=$(get_baseline_manifest "{rr[results_date]}")
+
+ # Normally there's a rr[results_date] in baseline manifest.
+ # The rr[results_date] was useless at one point (no dashboard for a period
+ # of time) and disapeared from the manifest for that time. It is now useful
+ # to have it back because we are setting another dashboard backend.
+ # This assertion on the existing results blocks any new results.
+ # Disabling it.
+ #
+ # TODO:
+ # Once all base-artifacts history are rewritten with the results_date,
+ # we will re-enable this assertion.
+ #assert_with_msg "Missing rr[results_date] from baseline manifest" \
+ # [ "$base_d" != "" ]
+
+ if [ "$base_d" != "" ]; then
+ if [ $results_date -gt $base_d ]; then
+ # Average between our current results_date and baseline date.
+ # The reason behind average is to spread out dates between bursts
+ # of builds, which can occur when reducing a regression.
+ results_date=$((($results_date + $base_d) / 2))
+ elif [ $results_date -eq $base_d ]; then
+ # If the dates are equal, then no point in taking average.
+ # Instead just add some arbitrary, but reasonable, amount.
+ results_date=$(($results_date + 600))
+ else
+ # If the baseline date is in the future (e.g., because git commit
+ # dates are weird in one of the components), then add some
+ # arbitrary, but reasonable, amount.
+ results_date=$(($base_d + 600))
+ fi
+ fi
+
+ # Save results_date in the manifest so that we can fetch it as $base_d
+ # above for the next build.
+ rr[results_date]="$results_date"
+ cat <<EOF | manifest_out
+rr[results_date]="$results_date"
+EOF
+ )
+}
+
+generate_dashboard_squad ()
+{
+ local results_date
+
+ echo "# ${FUNCNAME[0]}"
+ if ! $generate_dashboard; then
+ echo "... Skipping"
+ return
+ fi
+
+ results_date="${rr[results_date]}"
+ results_date=$(date --utc --iso-8601=seconds --date="@$results_date")
+
+ $scripts/dashboard-generate-squad.sh \
+ --top_artifacts "$top_artifacts" \
+ --baseline_branch "$(get_current_manifest "{rr[baseline_branch]}")" \
+ --components "$(get_current_manifest "{rr[components]}")" \
+ --run_date "$results_date" \
+ --relative_results true \
+ --squad_mode "vs-first"
+ echo "... Done"
+}
+
+post_dashboard_squad ()
+{
+ echo "# ${FUNCNAME[0]}"
+ if ! $post_dashboard; then
+ echo "... Skipping"
+ return
+ fi
+
+ if ! [ -d $top_artifacts/notify/dashboard/squad-vs-first ]; then
+ return
+ fi
+
+ $dryrun $top_artifacts/notify/dashboard/squad-vs-first/dashboard-push-squad.sh
+ echo "... Done"
+}
+
+generate_lnt_report()
+{
+ (
+ set -euf -o pipefail
+ local results_date
+
+ echo "# ${FUNCNAME[0]}"
+ if ! $generate_lnt; then
+ echo "... Skipping"
+ return
+ fi
+
+ # shellcheck source=lnt-utils
+ . $scripts/lnt-utils.sh
+
+ results_date="$(get_current_manifest "{rr[results_date]}")"
+ results_date=$(date +"%Y-%m-%d %H:%M:%S" --date "@$results_date")
+
+ local jira_key="-"
+ if [ -f "$top_artifacts/notify/jira/key" ]; then
+ jira_key=$(cat "$top_artifacts/notify/jira/key")
+ fi
+
+ case "$ci_project" in
+ tcwg_binutils*|tcwg_bootstrap*|tcwg_gcc*|tcwg_gdb*|tcwg_glibc*|tcwg_gnu*)
+ generate_lnt_gnu_check_report \
+ "$(get_current_manifest BUILD_URL)" "$ci_project" "$ci_config" \
+ "$results_date" "$jira_key" \
+ $top_artifacts/notify/results-summary.txt \
+ $top_artifacts/sumfiles \
+ $top_artifacts/notify/lnt_report.json
+ ;;
+ tcwg_bmk-*)
+ local cc cur_rev describe
+ case "${rr[toolchain]}" in
+ llvm) cc=llvm ;;
+ gnu) cc=gcc ;;
+ *) false ;;
+ esac
+ cur_rev=$(get_current_git ${cc}_rev)
+ describe=$(describe_sha1 "${cc}" "$cur_rev" false)
+ generate_lnt_bmk_report \
+ "$(get_current_manifest BUILD_URL)" "$ci_project" "$ci_config" \
+ "$results_date" "$jira_key" \
+ $top_artifacts/results-vs-prev/csv-results-1/size.csv \
+ $top_artifacts/results-vs-prev/csv-results-1/perf.csv \
+ $top_artifacts/results-vs-prev/csv-results-1/status.csv \
+ $top_artifacts/results-vs-prev/bmk-specific-variability-avg.csv \
+ $top_artifacts/results-vs-prev/bmk-specific-variability-max.csv \
+ $top_artifacts/results-vs-prev/compare-results-internal.csv \
+ $top_artifacts/notify/lnt_report.json
+ ;;
+ *)
+ # no lnt support
+ echo "... Skipping"
+ return
+ ;;
+ esac
+
+
+
+ ) &
+ wait $! || true
+}
+
+#========================================================================================
+#
+# MAIN FLOW
+#
+#========================================================================================
+
+# setup the environment to run notify stage
+setup_notify_environment
+check_source_changes
+setup_stages_to_run
+
+# Initialize icommits
+post_interesting_commits init
+
+if [ "$stage" != "full" ]; then
+ echo "Init stage ran successfully."
+ exit 0
+fi
+
+$generate_extra_details_f
+calculate_results_date
+
+check_if_first_report
+
+# Update entry with full information
+post_interesting_commits full
+
+if $generate_jira; then
+ generate_jira_dir
+fi
+
+echo "# print all notification files"
+if $generate_mail; then
+ generate_mail_files
+fi
+
+if $generate_jenkins_html; then
+ generate_jenkins_html_files
+fi
+
+# generate and post Dashboard
+echo "# generate dashboard"
+generate_dashboard_squad
+post_dashboard_squad
+
+generate_lnt_report
+
+if $post_mail; then
+ release_notification_files
+fi
+
+if $post_gcc_testresults; then
+ release_gcc_testresults_files
+fi
+
+# Update jira card description
+post_to_jira
+
+echo "Full stage ran successfully."
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-27 23:26:33 +0000