path: root/round-robin-notify.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)

# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh

# DEBUG
echo -e "\n$0 $*\n"

convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"

obligatory_variables rr[top_artifacts] notify
declare -A           rr
declare                                notify

generate_all="${generate_all-true}"
generate_jira="${generate_jira-$generate_all}"
generate_mail="${generate_mail-$generate_all}"
generate_dashboard="${generate_dashboard-$generate_all}"
post_all="${post_all-true}"
post_jira_comment="${post_jira_comment-$post_all}"
post_jira_card="${post_jira_card-$post_all}"
post_mail="${post_mail-$post_all}"
post_icommits="${post_icommits-$post_all}"
post_dashboard="${post_dashboard-$post_all}"

dryrun="${dryrun-false}"
icommits="interesting-commits"
stage="${stage-full}"
verbose="${verbose-false}"

# Generate notification files based on current and previous artifacts, which
# are assumed to be from two consecutive builds.
#
# Notes:
# 1. Current artifacts are in $top_artifacts.  All files from $top_artifacts
#    are accessible read-only, and round-robin-notify.sh writes its files
#    into $top_artifacts/notify/.
# 2. Files in "numbered" directories NN-* should not be used, since they
#    will be eventually removed.
# 3. Previous artifacts are in base-artifacts/, which is a git repository.
#    All files from base-artifacts/ are accessible read-only.
# 4. Round-robin-notify.sh can assume that files in $top_artifacts/ and
#    in base-artifacts/ have been generated by the latest version of scripts.
#    Developers will run round-robin-rewrite.sh when format of files changes
#    to bring "history" results up-to-date with what current scripts expect.
# 5. Round-robin-notify.sh can assume that check_regression() step was run
#    just before invocation of round-robin-notify.sh.

if $verbose; then
    set -x
fi

if $dryrun; then
    dryrun="echo DRYRUN: "
else
    dryrun=""
fi


#========================================================================================
#
#                               SETUP/FINALIZE PROCEDURES
#
#========================================================================================

# affect the following glabal variables
declare top_artifacts ci_project ci_config
setup_notify_environment ()
{
    echo "# ${FUNCNAME[0]}"

    # setup global variables
    top_artifacts="${rr[top_artifacts]}"
    ci_project=$(get_current_manifest "{rr[ci_project]}")
    ci_config=$(get_current_manifest "{rr[ci_config]}")

    # Debug dump
    echo "# Debug traces :"
    echo "#   Baseline  : $(get_baseline_manifest BUILD_URL)"
    echo "#   Using dir : base-artifacts"
    echo "#   Artifacts : $(get_current_manifest BUILD_URL)"
    echo "#   Using dir : $top_artifacts"
    echo ""

    mkdir -p "$top_artifacts/notify"

    if [ -f $HOME/.jipdate.yml ]; then
        (
	    # BE CAREFUL WITH TCWG_JIRA_TOKEN
	    set +x
	    if [ "${TCWG_JIRA_TOKEN-}" != "" ]; then
		sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" \
		    "$HOME/.jipdate.yml"
	    fi
	)
    fi

    declare -Ag pw
    if [ "$notify" = "precommit" ]; then
	obligatory_variables pw_dir
	declare -g           pw_dir

	local pw_file
	while IFS= read -r -d '' pw_file; do
	    # shellcheck disable=SC1090
	    source "$pw_file"
	done < <(find "$pw_dir" -type f -print0)
    fi
}

# Exit with 0 status if given component has a single commit compared to baseline.
# $1: component
single_commit_p ()
{
    (
    set -euf -o pipefail

    local c="$1"

    local base_rev cur_rev sha1
    base_rev=$(get_baseline_git "${c}_rev")
    cur_rev=$(get_current_git "${c}_rev")

    for sha1 in $(git -C "$c" rev-parse "$cur_rev^@"); do
	if [ "$sha1" = "$base_rev" ]; then
	    # We have a single-commit build
	    return 0
	fi
    done

    return 1
    )
}

# check_source_changes : Looks at base-artifacts and artifacts to compute the
# changes for the toolchain source files.
# This procedure updates the following global variables for the rest of notify pass.
declare change_kind changed_single_component last_good first_bad
declare -a changed_components

check_source_changes ()
{
    echo "# ${FUNCNAME[0]}"

    # Set ${changed_components[@]} unless this is "init" build.  For "init"
    # builds we have no baseline_git data, so leave changed_components empty.
    if [ "$(get_current_manifest "{rr[update_baseline]}")" != "init" ]; then
	IFS=" " read -r -a changed_components <<< "$(print_changed_components)"
    else
	changed_components=()
    fi

    local c base_rev cur_rev c_commits
    if [ ${#changed_components[@]} = 0 ]; then
	change_kind="no_change"
	changed_single_component=""
    elif [ ${#changed_components[@]} = 1 ]; then
	changed_single_component="${changed_components[0]}"
	first_bad="$(get_current_git ${changed_single_component}_rev)"
	last_good="$(get_baseline_git ${changed_single_component}_rev)"

	# single_commit_p() depend on $c to exist
	local res
	git -C "$changed_single_component" rev-parse --verify "HEAD" \
	    &>/dev/null &
	res=0 && wait $! || res=$?
	assert_with_msg "Cannot parse HEAD in repo $changed_single_component" \
			[ $res = 0 ]

	if single_commit_p "$changed_single_component"; then
	    change_kind="single_commit"
	else
            change_kind="single_component"
	fi
    else
	change_kind="multiple_components"
	changed_single_component=""
    fi

    # Debug dump
    echo "# Debug traces :"
    echo "#   change_kind=$change_kind : ${changed_components[*]}"
    for c in "${changed_components[@]}"; do
       base_rev=$(get_baseline_git ${c}_rev)
       cur_rev=$(get_current_git ${c}_rev)
       c_commits=$(git -C $c rev-list --count $base_rev..$cur_rev || echo "??")
       echo "#   rev for $c : $base_rev..$cur_rev ($c_commits commits)"
    done
    echo ""
}

# print routines pointers
declare print_commits_f print_result_f print_config_f print_last_icommit_f

setup_stages_to_run ()
{
    # if everything is fine. No reason to report to jira & icommits
    if [ "$notify" = "onregression" ] \
	   && { [ "${rr[no_regression_result]}" = "0" ] \
		    || [ "$change_kind" != "single_commit" ]; }; then
	notify=ignore
    elif [ "$notify" = "precommit" ] \
	     && [ "${rr[no_regression_result]}" = "0" ]; then
	notify=ignore
    fi

    if [ "$notify" = "ignore" ] || [ "$notify" = "precommit" ]; then
	post_jira_comment=false
	post_jira_card=false
	post_icommits=false
	if [ "$notify" = "ignore" ]; then
	    post_mail=false
	fi
    fi

    # Disable dashboard generation as results-vs-first is now disabled
    generate_dashboard=false
    post_dashboard=false

    # print routines pointers
    print_commits_f=print_commits
    print_result_f=print_result
    print_config_f=print_config
    print_last_icommit_f=print_last_icommit
    case "$ci_project" in
	tcwg_binutils*|tcwg_bootstrap*|tcwg_gcc*|tcwg_gdb*|tcwg_glibc*|tcwg_gnu*)
	   print_result_f=gnu_print_result
	   ;;
	tcwg_bmk*)
	   print_result_f=bmk_print_result
	   print_config_f=bmk_print_config
	   ;;
	*)
	   # By default keep generic ones set above
	   ;;
    esac
}

release_notification_files ()
{
    echo "# ${FUNCNAME[0]}"
    if ! [ -d $top_artifacts/jenkins ]; then
      return
    fi

    local f
    for f in mail-body.txt mail-subject.txt mail-recipients.txt; do
      # copy the file if exists, and not emtpy.
      # this important for mail-recipient.txt that may be empty.
      if [ -s $top_artifacts/notify/$f ]; then
        cp $top_artifacts/notify/$f $top_artifacts/jenkins/$f
      fi
    done

    echo "... Done"
}

#========================================================================================
#
#                              GENERATE EXTRA DETAILS
#
#========================================================================================
generate_extra_details()
{
   (
   set -euf -o pipefail
   echo "# ${FUNCNAME[0]}"

   # TODO: Produce extra details about result comparison
   )
}

#========================================================================================
#
#                              PRINT ROUTINES
#
#========================================================================================

dump_model_only=${dump_model_only-false}

#==========================================
#   *GENERIC* PRINT ROUTINES
#==========================================

###### PRINT COMMITS
#  --oneline : commit title
#  --short   : commit log
#  --long    : commit log and changes (up to 100 lines)
#  --full    : commit log and changes
print_commits()
{
   $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
   local print_arg=$1

   local components new_commits more_lines
   case "$change_kind:$print_arg" in
     no_change:--oneline|no_change:--short)
        echo "baseline build"
        ;;
     single_commit:--link)
        url=$(get_baseline_git ${changed_single_component}_url)
	if [[ "$url" =~ git://sourceware.org/git/ ]]; then
	   echo "https://sourceware.org/git/?p=${url#git://sourceware.org/git/};a=commitdiff;h=$first_bad"
	elif [[ "$url" =~ https://github.com/ ]] || [[ "$url" =~ https://gitlab.com/ ]]; then
	   echo "${url%.git}/commit/$first_bad"
	elif [[ "$url" =~ https://git.linaro.org/ ]]; then
	   echo "${url}/commit/?id=$first_bad"
	else
	   echo "See in $url"
	fi
        ;;
     single_commit:--oneline)
	 echo "$(describe_sha1 ${changed_single_component} $first_bad true) $(git -C ${changed_single_component} show --no-patch --pretty=%s $first_bad)"
        ;;
     single_commit:--short)
	tmpfile=$(mktemp)
        git -C ${changed_single_component} show --no-patch $first_bad > $tmpfile
	head -n 10 $tmpfile
	more_lines=$(($(cat "$tmpfile"| wc -l) - 10))
	if [ $more_lines -gt 0 ]; then
	    echo "... $more_lines lines of the commit log omitted."
	fi
	rm $tmpfile
        ;;
     single_commit:--long)
	tmpfile=$(mktemp)
        git -C ${changed_single_component} show $first_bad > $tmpfile
	head -n 100 $tmpfile
	more_lines=$(($(cat "$tmpfile"| wc -l) - 100))
	if [ $more_lines -gt 0 ]; then
	    echo "... $more_lines lines of the commit omitted."
	fi
	rm $tmpfile
        ;;
     single_commit:--full)
        git -C ${changed_single_component} show $first_bad
        ;;
     single_component:--oneline|multiple_components:--oneline)
	new_commits=0
	for c in "${changed_components[@]}"; do
           base_rev=$(get_baseline_git ${c}_rev)
           cur_rev=$(get_current_git ${c}_rev)
           c_commits=$(git -C $c rev-list --count $base_rev..$cur_rev || echo 0)
           new_commits=$(($new_commits + $c_commits))
        done
	components=$(echo "${changed_components[@]}" | tr ' ' ',')
	echo "$new_commits commits in $components"
        ;;
     single_component:--short|multiple_components:--short)
        new_commits=0
	for c in "${changed_components[@]}"; do
           base_rev=$(get_baseline_git ${c}_rev)
           cur_rev=$(get_current_git ${c}_rev)
           c_commits=$(git -C $c rev-list --count $base_rev..$cur_rev || echo 0)
           new_commits=$(($new_commits + $c_commits))

	   echo "$c commits:"
	   echo "$(git -C $c log --pretty=oneline $base_rev..$cur_rev 2>&1 | head -n5 || true)"
	   if [ $c_commits -gt 5 ]; then
		echo "... and $(($c_commits - 5)) more"
	   fi
	   if [ "${pw[$c]-}" != "" ]; then
	       echo "Patchwork URL: ${pw[${c}_patch_url]}"
	   fi
        done
	;;
     single_component:--long|single_component:--full)
        # no dump code change if multiple commits
        ;;
     multiple_components:--long|multiple_components:--full)
        # no dump code change if multiple commits
        ;;
     *:*) ;;
   esac
}

###### PRINT the RESULTS of this build
#  --oneline : either success ot failure
#  --short   : change of results.txt file between baseline and artifact
#  --long    : idem
print_result()
{
   $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

   local print_arg=$1
   case "$print_arg" in
       --oneline)
	   if [ "${rr[no_regression_result]}" = "0" ]; then
	       echo "Success"
	   else
	       echo "Failure"
	   fi
	   ;;
    --short|--long)
	echo "Results changed to"
	echo "$(cat $top_artifacts/results)"
	echo ""
	echo "From"
	echo "$(cat base-artifacts/results)"
	;;
   esac
}

###### PRINT LAST ICOMMIT
#  Extract the last status from icommit
#  --entry : output directory of icommit entry
#  --status : will output the icommit status
#  --reproduction_instructions_link : will output the link to the reproduction_instructions
print_last_icommit ()
{
   $dump_model_only && echo "<<${FUNCNAME[0]}>>" && return

   local print_arg=$1
   if [ x"$change_kind" != x"single_commit" ]; then
       return
   fi

   local isubdir
   isubdir=$(interesting_subdir "$changed_single_component" "$first_bad")

   case "$print_arg" in
       --entry)
	   echo "$icommits/$isubdir"
	   ;;
       --status)
           cat "$icommits/$isubdir/status.txt"
           ;;
       --reproduction_instructions_link)
	   echo "https://git.linaro.org/toolchain/ci/interesting-commits.git/plain/$isubdir/reproduction_instructions.txt"
	   ;;
   esac
}

# CHECK_IF_FIRST_REPORT : Check if this is the first report in icommits
# Result stored in first_icommit_to_report global variable
check_if_first_report()
{
   declare -g first_icommit_to_report

   first_icommit_to_report=false
   if [ x"$change_kind" != x"single_commit" ]; then
       return
   fi

   local isubdir
   isubdir=$(interesting_subdir "$changed_single_component" "$first_bad")
   if ! [ -f "$icommits/$isubdir/first_url" ]; then
       return
   fi

   local first_url
   first_url=$(cat "$icommits/$isubdir/first_url")
   if [ "$first_url" = "$(get_current_manifest BUILD_URL)" ]; then
       first_icommit_to_report=true
   elif [ "$notify" = "onregression" ]; then
       # Send out emails for post-commit "onregression" reports only for
       # the first detection.
       post_mail=false
   fi
}

print_config()
{
   # no config printed
   :
}

#==========================================
#   *GNU* PRINT ROUTINES
#==========================================

# most of them mapped to generic implementation
gnu_print_result()
{
    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
    local print_arg="$1"

    if ! [ -d $top_artifacts/sumfiles ]; then
	print_result "$@"
	return 0
    fi

    # Compare results using compare_tests for information purposes.
    # It works well only when we compare complete testsuites.
    # shellcheck disable=SC2154
    gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
				      base-artifacts/sumfiles \
				      "$top_artifacts/sumfiles" \
				      > $top_artifacts/notify/results.compare &
    wait $! || true

    local validate_failures="gcc-compare-results/contrib/testsuite-management/validate_failures.py"
    local xfails="$top_artifacts/sumfiles/xfails.xfail"

    if ! [ -f "$xfails" ]; then
	return 0
    fi

    "$validate_failures" --manifest="$xfails" \
			 --expiry_date="${rr[result_expiry_date]}" \
			 --build_dir="$top_artifacts/sumfiles" --verbosity=1 \
			 > $top_artifacts/notify/regressions.sum &
    wait $! || true
    "$validate_failures" --inverse_match --manifest="$xfails" \
			 --expiry_date="${rr[result_expiry_date]}" \
			 --build_dir="$top_artifacts/sumfiles" --verbosity=1 \
			 > $top_artifacts/notify/progressions.sum &
    wait $! || true

    local n_regressions n_progressions pass_fail=PASS
    if [ "${rr[no_regression_result]}" != "0" ]; then
	pass_fail=FAIL
    fi
    n_regressions=$(grep -c "^[A-Z]\+:" \
			 $top_artifacts/notify/regressions.sum || true)
    n_progressions=$(grep -c "^[A-Z]\+:" \
			  $top_artifacts/notify/progressions.sum || true)

    printf "$pass_fail"
    if [ "$n_regressions" != "0" ]; then
	printf ": $n_regressions regressions"
    else
	rm $top_artifacts/notify/regressions.sum
    fi
    if [ "$n_progressions" != "0" ]; then
	printf ": $n_progressions progressions"
    else
	rm $top_artifacts/notify/progressions.sum
    fi
    printf "\n"

    if [ "$print_arg" = "--oneline" ]; then
	return 0
    fi

    local length=10 outfile n_lines
    if [ "$print_arg" = "--long" ]; then
	length=100
    fi

    for outfile in regressions.sum progressions.sum; do
	if ! [ -f $top_artifacts/notify/$outfile ]; then
	    continue
	fi

	echo
	echo "$outfile:"
	n_lines=$(cat $top_artifacts/notify/$outfile | wc -l)
	n_lines=$(($n_lines - $length))
	head -n$length $top_artifacts/notify/$outfile
	if [ $n_lines -gt 0 ]; then
	    echo "... and $n_lines more entries"
	fi
    done
}


#==========================================
#   *BMK* PRINT ROUTINES
#==========================================

# print_result. either oneline version or short/long.
# Here is an ex :
# --oneline    :
#     644.nab_s slowed down of 4%
# --short/long :
#     the following benchmarks slowed down by more than 3%:
#     - 644.nab_s slowed down by 4% from 112963 to 117189 perf samples
bmk_print_result()
{
   $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

   local print_arg=$1

   artifacts_mail_dir=$top_artifacts/notify

   ## Prepare data

   # metric
   case "$ci_project" in
       *-*_vect-*) metric_id="vect" ;;
       *-*_sve-*) metric_id="sve" ;;
       *-*_size-*) metric_id="size" ;;
       *-*_speed-*) metric_id="time" ;;
   esac

   # If there's one regression. Don't bother about improvements.
   local improved_or_regressed
   if [ -f $artifacts_mail_dir/exe.regression ] || [ -f $artifacts_mail_dir/symbol.regression ]; then
       improved_or_regressed=regression
   else
       improved_or_regressed=improvement
   fi

   declare -A changed_by_msg
   changed_by_msg[size-regression]="grew in size by"
   changed_by_msg[size-improvement]="reduced in size by"
   changed_by_msg[time-regression]="slowed down by"
   changed_by_msg[time-improvement]="speeds up by"
   changed_by_msg[vect-regression]="reduced by"
   changed_by_msg[vect-improvement]="increased up by"
   changed_by_msg[sve-regression]="reduced by"
   changed_by_msg[sve-improvement]="increased up by"
   changed_by=${changed_by_msg[$metric_id-$improved_or_regressed]}

    # thresholds
    case $metric_id in
	size)
	    exe_threshold=1	# We use 1% tolerance for binary size
	    symbol_threshold=10 # and 10% tolerance for symbol size.
	    ;;
	time)
	    # Reduce thresholds when bisecting to avoid considering borderline
	    # regressions as spurious.  This should break cycles of build and
	    # bisect jobs triggering each other on borderline regressions.
	    exe_threshold=3
	    symbol_threshold=15
	    ;;
	vect|sve)
	    exe_threshold=0
	    symbol_threshold=0
	    ;;
	*) assert false ;;
    esac

   # Now print result
   case "$print_arg" in
     --oneline)
	 assert_with_msg "Builds with infra problems should never get here" \
			 [ "${rr[no_regression_result]}" != "$EXTERNAL_FAIL" ]

	 # Generate readable oneline diag
	 local metric bmk symbol short_diag long_diag
	 if [ -f $artifacts_mail_dir/exe.$improved_or_regressed ]; then
	     # shellcheck disable=SC2034
	     IFS=, read metric bmk symbol short_diag long_diag < <(head -n1 $artifacts_mail_dir/exe.$improved_or_regressed)
	 elif [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
	     # shellcheck disable=SC2034
	     IFS=, read metric bmk symbol short_diag long_diag < <(head -n1 $artifacts_mail_dir/symbol.$improved_or_regressed)
	 else
	     short_diag="No change"
	 fi
	 echo "$short_diag"
	 ;;

    --short|--long)
	# The following exe regressed/improved:
	if [ -f $artifacts_mail_dir/exe.$improved_or_regressed ]; then
	   sort -gr -o $artifacts_mail_dir/exe.$improved_or_regressed \
                    $artifacts_mail_dir/exe.$improved_or_regressed

	   echo "the following benchmarks $changed_by more than ${exe_threshold}%:"
	   local metric exe symbol short_diag long_diag
	   while IFS=, read metric exe symbol short_diag long_diag; do
              echo "- $long_diag"
              if [ -f $artifacts_mail_dir/$exe.symbols-$improved_or_regressed ]; then
                 while IFS=, read metric bmk symbol short_diag long_diag; do
                   echo "  - $long_diag"
                 done < $artifacts_mail_dir/$exe.symbols-$improved_or_regressed
                 # Delete $bmk.regressions so that it doesn't show up
                 # in symbol-regression loop below.
                 rm $artifacts_mail_dir/$exe.symbols-$improved_or_regressed
              fi
	   done < $artifacts_mail_dir/exe.$improved_or_regressed
	fi

	# The following functions regressed/improved:
	if [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
	   echo "the following hot functions $changed_by more than ${symbol_threshold}% (but their benchmarks $changed_by less than ${exe_threshold}%):"
	   local metric bmk symbol short_diag long_diag
	   # shellcheck disable=SC2034
	   while IFS=, read metric bmk symbol short_diag long_diag; do
	     echo "- $long_diag"
	   done < $artifacts_mail_dir/symbol.$improved_or_regressed
	fi

	if ! [ -f $artifacts_mail_dir/exe.$improved_or_regressed ] && ! [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
	   echo "No change"
	fi
        ;;
   esac
}

bmk_print_config()
{
    # shellcheck source=tcwg_bmk-config.sh
    . $scripts/tcwg_bmk-config.sh

    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

    # metric
    case "$ci_project" in
	*-*_vect-*) metric_id="vect" ;;
	*-*_sve-*) metric_id="sve" ;;
	*-*_size-*) metric_id="size" ;;
	*-*_speed-*) metric_id="time" ;;
    esac

    # ${ci_project}--${ci_config} format is :
    #  'tcwg_bmk-#{PROFILE_NAME}-#{BMK}--#{TOOLCHAIN}-#{TARGET}-{toolchain_ver}-{cflags}'
    IFS=- read -a ci_pjt_cfg <<EOF
$ci_project--$ci_config
EOF
    local toolchain target cflags
    toolchain="${ci_pjt_cfg[4]}"
    target="${ci_pjt_cfg[5]}"
    cflags="${ci_pjt_cfg[7]}"

    local bmk_suite publish_save_temps
    bmk_suite=""
    publish_save_temps=false
    case "$(tcwg_bmk_benchs)" in
	coremark)
	    bmk_suite="EEMBC CoreMark"
	    ;;
	spec2k6|4*)
	    bmk_suite="SPEC CPU2006"
	    publish_save_temps=true
	    ;;
	spec2017*|5*|6*)
	    bmk_suite="SPEC CPU2017"
	    publish_save_temps=true
	    ;;
    esac

    cat <<EOF
Below reproducer instructions can be used to re-build both "first_bad" and "last_good" cross-toolchains used in this bisection.  Naturally, the scripts will fail when triggerring benchmarking jobs if you don\'t have access to Linaro TCWG CI.
EOF

    # Copy save-temps tarballs to artifacts, so that they are accessible.
    # We can publish pre-processed source only for benchmarks derived from
    # open-source projects.
    # Note that we include save-temps artifacts for successful builds so that
    # "last_good" build has the artifacts.
    if $publish_save_temps; then
	mkdir -p ${top_artifacts}/top-artifacts
	local s_t
	while read s_t; do
	    case "$s_t" in
		400.perlbench*|500.perlbench*|600.perlbench*) ;;
		401.bzip2*) ;;
		403.gcc*|502.gcc*|602.gcc*) ;;
		435.gromacs*) ;;
		436.cactusADM*|507.cactuBSSN*|607.cactuBSSN*) ;;
		445.gobmk*) ;;
		454.calculix*) ;;
		456.hmmer*) ;;
		462.libquantum*) ;;
		465.tonto*) ;;
		481.wrf*|521.wrf*|621.wrf*) ;;
		482.sphinx3*) ;;
		483.xalanc*) ;;
		505.mcf*|605.mcf*)
		# 429.mcf is not present in redistributable_sources/ :-|
		;;
		511.povray*) ;;
		525.x264*|625.x264*)
		# 464.h264ref is not present in redistributable_sources/ :-|
		;;
		526.blender*) ;;
		527.cam4*|627.cam4*) ;;
		538.imagick*|638.imagick*) ;;
		544.nab*|644.nab*) ;;
		554.roms*|654.roms*) ;;
		557.xz*|657.xz*) ;;
		628.pop2) ;;
		*)
		    # Can't redistribute benchmark sources.
		    continue
		    ;;
	    esac
	    cp "$s_t" ${top_artifacts}/top-artifacts/save-temps/
	done < <(find results-1 -path "save.*.temps/*.tar.xz")
    fi

    if [ -d ${top_artifacts}/top-artifacts/save-temps/ ]; then
	cat <<EOF

For your convenience, we have uploaded tarballs with pre-processed source and assembly files at:
- First_bad save-temps: \$FIRST_BAD_ARTIFACTS/save-temps/
- Last_good save-temps: \$LAST_GOOD_ARTIFACTS/save-temps/
- Baseline save-temps: \$BASELINE_ARTIFACTS/save-temps/
EOF
    fi

    local compiler="" libc="" linker="" version="" bmk_flags="" hw=""
    case "$toolchain" in
	gnu)
	    compiler="GCC"
	    libc="Glibc"
	    linker="GNU Linker"
	    ;;
	gnu_eabi)
	    compiler="GCC"
	    libc="Newlib"
	    linker="GNU LD"
	    ;;
	llvm)
	    compiler="Clang"
	    libc="Glibc"
	    linker="LLVM Linker"
	    ;;
    esac
    case "$ci_config" in
	*-master-*) version="tip of trunk" ;;
	*-release-*) version="latest release branch" ;;
    esac
    bmk_flags=$(echo "$cflags" | sed -e "s/_/ -/g" -e "s/LTO/flto/g" \
				     -e "s/VECT/fdump-tree-vect-details/g")
    case "$(tcwg_bmk_hw)" in
	*apm*) hw="APM Mustang 8x X-Gene1" ;;
	*tk1*) hw="NVidia TK1 4x Cortex-A15" ;;
	*tx1*) hw="NVidia TX1 4x Cortex-A57" ;;
	*stm32*) hw="STMicroelectronics STM32L476RGTx 1x Cortex-M4" ;;
	*fx*) hw="Fujitsu FX700 48x AA64" ;;
	*qc*) hw="Qualcomm 8x AA64" ;;
	*) hw="<unknown>"
    esac

    cat <<EOF

Configuration:
- Benchmark: $bmk_suite
- Toolchain: $compiler + $libc + $linker
- Version: all components were built from their $version
- Target: $(print_gnu_target $target)
- Compiler flags: $bmk_flags
- Hardware: $hw

This benchmarking CI is work-in-progress, and we welcome feedback and suggestions at linaro-toolchain@lists.linaro.org .  In our improvement plans is to add support for SPEC CPU2017 benchmarks and provide "perf report/annotate" data behind these reports.
EOF
}

#========================================================================================
#
#                            INTERESTING_COMMITS PROCEDURES
#
#========================================================================================

# update_interesting_commits : will update the local icommit repository with
# the new run.
# If it is a regression, single_commit, may be updating first_report variable
#
# Interesting-commits store regression history in the following hierarchy:
# 1. At the top level we have COMPONENT directories.
# 2. At the 2nd level we have SHA1 directories.
# 2a. We also have "git describe" symlinks to SHA1 directories for convenience.
# 3. At the 3rd level we have CI_PROJECT directories.
# 4. At the 4th level we have CI_CONFIG directories, and ...
# 4a. ... status.txt, which contains status of changes from SHA1 across all
#     CI_CONFIGs in CI_PROJECT.
# 5. At the 5th level we have per-build files last_good, summary.txt, etc.
update_interesting_commits ()
{
    echo "# ${FUNCNAME[0]}"

    local stage="$1"
    local jira_key="$2"

    # We have successfully identified a bad commit with a good parent!
    # Add $first_bad to interesting commits.
    local subdir
    subdir=$(interesting_subdir "$changed_single_component" "$first_bad")
    if ! [ -d "$icommits/$subdir" ]; then
	mkdir $icommits/$subdir
	get_current_manifest BUILD_URL > $icommits/$subdir/first_url
	git -C $icommits add $subdir/first_url
    fi

    # Update interesting-commits/$component/$first_bad/$ci_project/$ci_config
    subdir=$(interesting_subdir "$changed_single_component" "$first_bad" \
				"$ci_project" "$ci_config")
    mkdir -p "$icommits/$subdir"
    get_current_manifest BUILD_URL > "$icommits/$subdir/build_url"
    echo "$last_good" > "$icommits/$subdir/last_good"
    git -C "$icommits" add "$subdir/build_url" "$subdir/last_good"

    if [ "$stage" != "full" ]; then
	return
    fi

    # $icommit/<comp>/sha1/<sha1>/<ci_project>/<ci_config>/summary.txt
    $print_result_f --oneline > "$icommits/$subdir/summary.txt"
    git -C "$icommits" add "$subdir/summary.txt"

    # $icommit/<comp>/sha1/<sha1>/<ci_project>/<ci_config>/reproduction_instructions.txt
    local bad_artifacts_url good_artifacts_url
    bad_artifacts_url="$(get_current_manifest BUILD_URL)artifact/artifacts"
    good_artifacts_url="$(get_baseline_manifest BUILD_URL)artifact/artifacts"
    cat > $icommits/$subdir/reproduction_instructions.txt << EOF
mkdir -p investigate-$changed_single_component-$first_bad
cd investigate-$changed_single_component-$first_bad

# Fetch scripts
git clone https://git.linaro.org/toolchain/jenkins-scripts

# Fetch manifests for bad and good builds
mkdir -p bad/artifacts good/artifacts
curl -o bad/artifacts/manifest.sh $bad_artifacts_url/manifest.sh --fail
curl -o good/artifacts/manifest.sh $good_artifacts_url/manifest.sh --fail

# Reproduce bad build
(cd bad; ../jenkins-scripts/${ci_project/-*}-build.sh ^^ true %%rr[top_artifacts] artifacts)
# Reproduce good build
(cd good; ../jenkins-scripts/${ci_project/-*}-build.sh ^^ true %%rr[top_artifacts] artifacts)
EOF
    git -C "$icommits" add $subdir/reproduction_instructions.txt

    # $icommit/<comp>/sha1/<sha1>/commit-log.txt
    local commit_file="$subdir/../../commit-log.txt"
    local describe
    describe=$(describe_sha1 "$changed_single_component" "$first_bad" true)

    cat > $icommits/$commit_file <<EOF
Status of $describe commit for $ci_project:
$(git -C "$changed_single_component" log -n1 --pretty=short "$first_bad")
EOF
    git -C "$icommits" add $commit_file

    # $icommit/<comp>/sha1/<sha1>/<ci_project>/status.txt
    local ci_config
    local status="$subdir/../status.txt"
    while read ci_config; do
	if ! [ -f "$icommits/$subdir/../$ci_config/summary.txt" ]; then
	    continue
	fi

	echo "* $ci_config"
	(
	    cat $icommits/$subdir/../$ci_config/summary.txt
	    cat $icommits/$subdir/../$ci_config/build_url
	) | sed "s/^/** /"
    done < <(cd $icommits/$subdir/..; ls) > $icommits/$status
    git -C "$icommits" add "$status"

    # $icommit/<comp>/sha1/<sha1>/status.txt
    local ci_project
    status="$subdir/../../status.txt"
    while read ci_project; do
	if ! [ -f "$icommits/$subdir/../../$ci_project/status.txt" ]; then
	    continue
	fi

	echo "* $ci_project"
	cat $icommits/$subdir/../../$ci_project/status.txt | sed "s/^/*/"
    done < <(cd $icommits/$subdir/../..; ls) > $icommits/$status
    git -C "$icommits" add "$status"

    if $generate_jira; then
	subdir=$(interesting_subdir "$changed_single_component" "$first_bad")

	local jira_dir="$subdir/jira"

	if [ -f "$icommits/$jira_dir/key" ]; then
	    assert_with_msg "Should not have created multiple jira cards" \
			    [ "$jira_key" = "" ]
	    jira_key=$(cat "$icommits/$jira_dir/key")
	fi

	# Recreate jira/ dir with up-to-date info.
	if [ -e "$icommits/$jira_dir" ]; then
	    git -C "$icommits" rm -rf "$jira_dir"
	fi
	mkdir "$icommits/$jira_dir"

	if [ "$jira_key" != "" ]; then
	    echo "$jira_key" > "$icommits/$jira_dir/key"
	    git -C "$icommits" add "$jira_dir/key"
	fi

	echo "$describe" > "$icommits/$jira_dir/summary"
	git -C "$icommits" add "$jira_dir/summary"

	# FIXME: Unify format with email
	cat > "$icommits/$jira_dir/description" <<EOF
$(cat "$icommits/$subdir/commit-log.txt")

$(cat "$icommits/$subdir/status.txt")
EOF
	git -C "$icommits" add "$jira_dir/description"

	update_jira_card
    fi

    # Generate a $describe_sha1 symlink
    describe=$(describe_sha1 "$changed_single_component" "$first_bad" false)
    if [ "$describe" != "" ]; then
	local d
	d=$(dirname "$describe")
	mkdir -p $icommits/$changed_single_component/$d
	local symlink=""
	while [ "$d" != "." ]; do
	    symlink="../$symlink"
	    d=$(dirname "$d")
	done
	symlink="${symlink}sha1/$first_bad"
	# ??? For some reason I don't understand overwriting symlink with
	#     "ln -sf" may create a second [broken] symlink.
	rm -f $icommits/$changed_single_component/$describe
	ln -s $symlink $icommits/$changed_single_component/$describe
	git -C $icommits add $changed_single_component/$describe
    fi
}


# Generate entry in interesting-commits.git and, if $post_icommits,
# push it.  This is applicable only for single-commit case.
# This is called twice -- first time with $stage == init, and second time
# with $stage == full.
# During "init" stage we fetch existing entries and create a very basic entry
# if there isn't one.  This allows us to determine in check_if_first_report()
# whether this is the first build to discover $first_bad as interesting commit.
post_interesting_commits ()
{
    (
    set -euf -o pipefail
    echo "# ${FUNCNAME[0]}"

    local stage="$1"

    if [ "$change_kind" != "single_commit" ]; then
	return
    fi

    # Clone interesting-commits.git repo, which contains a regression
    # summaries for SHA1s.  These are the "first_bad" commits.
    clone_or_update_repo $icommits master \
			 https://git-us.linaro.org/toolchain/ci/interesting-commits.git \
			 auto master

    if ! $post_icommits; then
	dryrun="echo DRYRUN: "
    fi

    local jira_dir jira_key=""
    jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
    jira_dir="$jira_dir/jira"
    if [ "$stage" = "full" ] && $post_jira_card && $first_icommit_to_report \
	   && [ "$dryrun" = "" ]; then
	if ! [ -f "$icommits/$jira_dir/key" ]; then
	    # Create jira card for this interesting commit.
	    # We first create the card,
	    # then add a link to it in interesting-commits.git,
	    # then we update the card's content every time we change entry in
	    # interesting-commits.
	    jira_key=$(create_jira_card)
	else
	    echo "WARNING: jira card already exists $icommits/$jira_dir/key:" \
		 "$(cat "$icommits/$jira_dir/key")"
	fi
    fi

    # Regenerate interesting-commits entry and try to push to upstream.
    # - If failed to push, update icommits again
    #   This can happen if another job pushed to interesting-commits just
    #   before this job.
    # - If successful we consider round-robin-notify.sh to be successful.
    while true; do
	# Reset to master branch
	git -C $icommits remote update -p
	git_clean $icommits refs/remotes/origin/master

	update_interesting_commits "$stage" "$jira_key"

	# Commit changes (if any) to local clone of interesting-commits.git
	git -C $icommits commit \
	    -m "Add entry $first_bad from $(get_current_manifest BUILD_URL)" \
	    || break

	$dryrun git -C $icommits push \
		ssh://git-us.linaro.org/toolchain/ci/interesting-commits.git \
		HEAD:refs/heads/master &
	if wait $!; then
            # Push successful : stop here
	    break
	fi
	# Push failed. update icommit and retry pushing
    done
    )
}

#========================================================================================
#
#                            JIRA RELATED PROCEDURES
#
#========================================================================================

print_jira_template_card ()
{
    # Catch-all case for when project/config IDs change, so that we
    # won't miss notifications.  Forward all that to GNU-692.
    local jira_card="GNU-692"
    case "$ci_project/$ci_config:$changed_single_component" in
	tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;;
	tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;;
	tcwg_kernel/llvm-*:linux) jira_card="LLVM-647" ;;
	tcwg_kernel/llvm-*:*) jira_card="LLVM-646" ;;
	tcwg_bmk-*/gnu*-O[23]*) jira_card="GNU-689" ;;
	tcwg_bmk-*/gnu*) jira_card="GNU-686" ;;
	tcwg_bmk-*/llvm-*O[23]*) jira_card="LLVM-651" ;;
	tcwg_bmk-*/llvm-*) jira_card="LLVM-650" ;;
	tcwg_binutils/*) jira_card="GNU-692" ;;
	tcwg_cross/*) jira_card="GNU-692" ;;
	tcwg_gnu/*) jira_card="GNU-692" ;;
    esac
    echo "$jira_card"
}

# Create jira card for this interesting commit.
# Link to this card is stored in $icommits/.../jira/key
create_jira_card ()
{
    (
    set -euf -o pipefail

    local template project parent assignee yaml
    template=$(print_jira_template_card)
    project="${template%%-*}"
    parent=$(jipsearch -j "key=$template" -s parent:key \
		 | sed -e "s/.* , //")
    assignee=$(jipsearch -j "key=$parent" -s assignee:emailAddress \
		   | sed -e "s/.* , //")

    yaml=$(mktemp)
    # shellcheck disable=SC2064
    trap "rm $yaml" EXIT

    cat > $yaml <<EOF
- Project: $project
  IssueType: Sub-task
  Parent: $parent
  Summary: $changed_single_component:$first_bad
  AssigneeEmail: $assignee
EOF
    local key
    key=$(jipcreate -f $yaml | sed -e "s#.*/##")
    echo "$key"
    )
}

# Print the existing jira card number for this interesting commit.
print_jira_card_key ()
{
    (
    set -euf -o pipefail

    local jira_dir
    jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
    jira_dir="$jira_dir/jira"

    if ! [ -f "$icommits/$jira_dir/key" ]; then
	return 0
    fi

    cat "$icommits/$jira_dir/key"
    )
}

# Update jira card for this interesting commit.
# Link to this card is stored in $icommits/.../jira/key
update_jira_card ()
{
    (
    set -euf -o pipefail
    echo "# ${FUNCNAME[0]}"

    local jira_dir
    jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
    jira_dir="$jira_dir/jira"

    local -a components=(CI)
    case "$changed_single_component" in
	binutils) components+=(Binutils) ;;
	gcc) components+=(GCC) ;;
	gdb) components+=(GDB) ;;
	glibc) components+=(Glibc) ;;
	llvm) components+=(LLVM) ;;
	newlib) components+=(Newlib) ;;
    esac
    (IFS=","; echo "${components[*]}") > "$icommits/$jira_dir/components"
    git -C "$icommits" add "$jira_dir/components"

    local commit_date
    commit_date=$(git -C "$changed_single_component" log -n1 \
		      --pretty="%cd" --date=iso "$first_bad")
    date -d "$commit_date" +%Y-%m-%d > "$icommits/$jira_dir/startdate"
    git -C "$icommits" add "$jira_dir/startdate"

    local key project
    key=$(print_jira_card_key)
    if [ -z "$key" ]; then
	echo "WARNING: no existing jira card $icommits/$jira_dir/key"
	return 0
    fi

    project="${key%%-*}"

    local yaml="$icommits/$jira_dir/yaml"

    cat > "$yaml" <<EOF
- Project: $project
  IssueType: Sub-task
  Key: $key
  Summary: $(cat "$icommits/$jira_dir/summary")
  Components: $(cat "$icommits/$jira_dir/components")
  Start date: $(cat "$icommits/$jira_dir/startdate")
  Description: |
EOF
    sed -e "s/^/    /" "$icommits/$jira_dir/description" >> "$yaml"
    git -C "$icommits" add "$jira_dir/yaml"
    )
}

# Generate notify/jira/* files
generate_jira_dir()
{
    (
    set -euf -o pipefail

    local icommit_entry jira_key=""
    icommit_entry=$($print_last_icommit_f --entry)

    if [ "$icommit_entry" != "" ] && [ -d "$icommit_entry/jira" ]; then
	rsync -a "$icommit_entry/jira/" "$top_artifacts/notify/jira/"
	if [ -f "$top_artifacts/notify/jira/key" ]; then
	    jira_key=$(cat "$top_artifacts/notify/jira/key")
	fi
    else
	mkdir -p "$top_artifacts/notify/jira"
    fi

    if [ "$jira_key" != "" ]; then
	cat > $top_artifacts/notify/jira/comments.txt <<EOF
[$jira_key]
$($print_result_f --oneline)
Details: $(get_current_manifest BUILD_URL)artifact/artifacts/notify/mail-body.txt/*view*/

[$(print_jira_template_card)]
https://linaro.atlassian.net/browse/$jira_key
$($print_result_f --oneline)
Details: $(get_current_manifest BUILD_URL)artifact/artifacts/notify/mail-body.txt/*view*/
EOF
    else
	cat > $top_artifacts/notify/jira/comments.txt << EOF
[$(print_jira_template_card)]
$($print_result_f --oneline)
Details: $(get_current_manifest BUILD_URL)artifact/artifacts/notify/mail-body.txt/*view*/
EOF
    fi
    )
}

# Post update to Jira
post_to_jira ()
{
    echo "# ${FUNCNAME[0]}"

    if $post_jira_card && [ -f $top_artifacts/notify/jira/yaml ]; then
	$dryrun jipcreate -f $top_artifacts/notify/jira/yaml
    fi

    if $post_jira_comment \
	    && [ -f $top_artifacts/notify/jira/comments.txt ]; then
	echo y | $dryrun jipdate -f $top_artifacts/notify/jira/comments.txt
    fi
}


#========================================================================================
#
#                            MAIL RELATED PROCEDURES
#
#========================================================================================
# Model for mail-recipient.txt (may contain some of the following):
#    <author>, <compiler-mailing-list>, <linaro-toolchain-mailing-list>, <tcwg-validation-mailing-list>
#                                          (generate_mail_recipients : Generic)
#
# Model for mail-subject.txt :
#    [Linaro-TCWG-CI] <diag> after <changes> (generate_mail_subject : Generic)
#
# Model for mail-body.txt :
#    <mail regression details>                (generate_mail_body_regression : Specific to the kind of project)
#
#    <reproduction instructions details>      (generate_mail_body_reproduction_instructions : Generic)
#

print_mail_recipients ()
{
    (
    set -euf -o pipefail

    if ! $generate_mail; then
       return
    fi

    local -A emails
    emails["tcwg-validation@linaro.org"]=bcc

    # FIXME: Re-enable email to mailing lists and patch authors
    local -A no_emails

    # CC: mailing list (disabled for now)
    case "$ci_project/$ci_config:$changed_single_component" in
	tcwg_gcc*/*:*)
	    no_emails["gcc-regression@gcc.gnu.org"]=cc
	    ;;
	tcwg_gnu*/*:*)
	    no_emails["gcc-regression@gcc.gnu.org"]=cc
	    ;;
	tcwg_kernel/llvm-*:linux)
	    no_emails["llvm@lists.linux.dev"]=cc
	    no_emails["arnd@linaro.org"]=cc
	    ;;
	tcwg_kernel/*:linux)
	    no_emails["linaro-kernel@lists.linaro.org"]=cc
	    no_emails["arnd@linaro.org"]=cc
	    ;;
	tcwg_kernel/*:llvm)
	    no_emails["llvm@lists.linux.dev"]=cc
	    ;;
	tcwg_kernel/*:*)
	    no_emails["linaro-toolchain@lists.linaro.org"]=cc
	    ;;
	tcwg_bmk-vect_*/*:*|tcwg_bmk-sve_*/*:*)
	    # Don't report regressions for vectorization, while we are
	    # debugging it.
	    ;;
	tcwg_bmk-*/gnu*:*)
	    no_emails["gcc-regression@gcc.gnu.org"]=cc
	    ;;
	tcwg_bmk-*/llvm*:*)
	    no_emails["linaro-toolchain@lists.linaro.org"]=cc
	    ;;
	*/*:*)
	    no_emails["linaro-toolchain@lists.linaro.org"]=to
	    ;;
    esac

    local c email

    # CC: author (disabled for now)
    for c in "${changed_components[@]}"; do
	base_rev=$(get_baseline_git "${c}_rev")
	cur_rev=$(get_current_git "${c}_rev")
	while read -r email; do
	    no_emails["$email"]=cc
	done < <(git -C "$c" log --pretty='%ae' "$base_rev..$cur_rev")
    done

    # TO: committer (disabled for now)
    for c in "${changed_components[@]}"; do
	base_rev=$(get_baseline_git "${c}_rev")
	cur_rev=$(get_current_git "${c}_rev")
	while read -r email; do
	    # shellcheck disable=SC2034
	    no_emails["$email"]=to
	done < <(git -C "$c" log --pretty='%ce' "$base_rev..$cur_rev")
    done

    # TO: precommit submitter
    for c in "${changed_components[@]}"; do
	if [ "${pw[$c]-}" != "" ]; then
	    emails["${pw[${c}_patch_submitter]}"]=to
	fi
    done

    local -a recipients=()
    for email in "${!emails[@]}"; do
	recipients+=("${emails[$email]}:$email")
    done

    (IFS=","; echo "${recipients[*]}")
    )
}

##### Model for GNU SPECIFIC mail-body-regression.txt regression details
#
# After commit <>
#
# The following .. slowed down/grew up ..
#
# The configuration is ...
#

##### Generates mail/mail-body.txt file
print_mail_body()
{
    if ! $generate_mail; then
       return
    fi

    local bad_artifacts_url good_artifacts_url
    bad_artifacts_url="$(get_current_manifest BUILD_URL)artifact/artifacts"
    good_artifacts_url="$(get_baseline_manifest BUILD_URL)artifact/artifacts"

    cat << EOF
Dear contributor, our automatic CI has detected problems related to your patch.
Please find below some details about it.  If you have any questions, please
follow up on linaro-toolchain@lists.linaro.org mailing list.

In CI config $ci_project/$ci_config after:

$($print_commits_f --short | sed -e 's/^/  | /')

$($print_result_f --short)

$($print_config_f)

-----------------8<--------------------------8<--------------------------8<--------------------------
The information below can be used to reproduce a debug environment:

Current build   : $bad_artifacts_url
Reference build : $good_artifacts_url

EOF

    if [ x"$change_kind" != x"single_commit" ]; then
       return
    fi

    local key
    key=$(print_jira_card_key)
    if [ -z "$key" ]; then
	key=$(print_jira_template_card)
    fi

    cat <<EOF
Reproduce last good and first bad builds: $($print_last_icommit_f --reproduction_instructions_link)

Full commit : $($print_commits_f --link)

Latest bug report status : https://linaro.atlassian.net/browse/$key

List of configurations that regressed due to this commit :
$($print_last_icommit_f --status)

EOF
}

##### Generates mail/mail-subject.txt file (Generic)
print_mail_subject()
{
    if ! $generate_mail; then
       return
    fi

    # mail-subject.txt
    echo "[Linaro-TCWG-CI] $($print_result_f --oneline) after $($print_commits_f --oneline)"
}


#========================================================================================
#
#                               DASHBOARD RELATED PROCEDURES
#
#========================================================================================
# Calculate a reasonable date to associate with the current results / artifacts.
calculate_results_date ()
{
    (
    set -euf -o pipefail

    local c base_d cur_d results_date=0

    # Firstly, set results_date to the max of commit dates of all components.
    for c in $(get_current_manifest "{rr[components]}"); do
	base_d=$(get_baseline_manifest "{rr[debug_${c}_date]}" \
		     | cut -d" " -f 1 || true)
	cur_d=$(get_current_manifest "{rr[debug_${c}_date]}" \
		    | cut -d" " -f 1 || true)
	if [ x"$base_d" != x"" ]; then
	    if [ x"$cur_d" = x"" ] || [ $cur_d -lt $base_d ]; then
		cur_d="$base_d"
	    fi
	fi
	if [ x"$cur_d" = x"" ]; then
	    continue
	fi

	if [ $cur_d -gt $results_date ]; then
	    results_date="$cur_d"
	fi
    done

    assert_with_msg "Failed to produce results_date" [ $results_date -gt 0 ]

    # Secondly, if we have a baseline date, return average between our
    # current results_date and baseline date.  The reason behind average
    # is to spread out dates between bursts of builds, which can occur
    # when reducing a regression.
    base_d=$(get_baseline_manifest "{rr[results_date]}")
    assert_with_msg "Missing rr[results_date] from baseline manifest" \
		    [ "$base_d" != "" ]

    assert [ $results_date -ge $base_d ]

    if [ $results_date -gt $base_d ]; then
	results_date=$((($results_date + $base_d) / 2))
    else
	# If the dates are equal, then no point in taking average.
	# Instead just add some arbitrary, but reasonable, amount.
	results_date=$(($results_date + 600))
    fi

    # Save results_date in the manifest so that we can fetch it as $base_d
    # above for the next build.
    rr[results_date]="$results_date"
    cat <<EOF | manifest_out
rr[results_date]="$results_date"
EOF
    )
}

generate_dashboard_squad ()
{
    local results_date

    echo "# ${FUNCNAME[0]}"
    if ! $generate_dashboard; then
       echo "... Skipping"
       return
    fi

    calculate_results_date
    results_date="${rr[results_date]}"
    results_date=$(date --utc --iso-8601=seconds --date="@$results_date")

    $scripts/dashboard-generate-squad.sh \
	    --top_artifacts "$top_artifacts" \
	    --baseline_branch "$(get_current_manifest "{rr[baseline_branch]}")" \
	    --components "$(get_current_manifest "{rr[components]}")" \
	    --run_date "$results_date" \
	    --relative_results true \
	    --squad_mode "vs-first"
    echo "... Done"
}

post_dashboard_squad ()
{
    echo "# ${FUNCNAME[0]}"
    if ! $post_dashboard; then
       echo "... Skipping"
       return
    fi

    if ! [ -d $top_artifacts/notify/dashboard/squad-vs-first ]; then
       return
    fi

    $dryrun $top_artifacts/notify/dashboard/squad-vs-first/dashboard-push-squad.sh
    echo "... Done"
}

#========================================================================================
#
#                                    MAIN FLOW
#
#========================================================================================

# setup the environment to run notify stage
setup_notify_environment
check_source_changes
setup_stages_to_run

generate_extra_details

# Initialize icommits
post_interesting_commits init

if [ "$stage" != "full" ]; then
    echo "Init stage ran successfully."
    exit 0
fi

check_if_first_report

# Update entry with full information
post_interesting_commits full

# generate all notifications
echo "# print all notification files"
print_mail_recipients > $top_artifacts/notify/mail-recipients.txt
if $generate_jira; then
    generate_jira_dir
fi
$print_result_f --oneline > $top_artifacts/notify/summary.txt
print_mail_body > $top_artifacts/notify/mail-body.txt
print_mail_subject > $top_artifacts/notify/mail-subject.txt

# generate and post Dashboard
echo "# generate dashboard"
generate_dashboard_squad
post_dashboard_squad

if $post_mail; then
    release_notification_files
fi

# Update jira card description
post_to_jira

echo "Full stage ran successfully."