path: root/round-robin-notify.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname $0)

# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh

# DEBUG
echo -e "\n$0 $*\n"

convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"

obligatory_variables rr[top_artifacts] notify build_script
declare -A           rr
declare                                notify build_script

generate_all="${generate_all-true}"
generate_jira="${generate_jira-$generate_all}"
generate_mail="${generate_mail-$generate_all}"
generate_jenkins_html=${generate_jenkins_html-$generate_all}
generate_dashboard="${generate_dashboard-$generate_all}"
generate_lnt="${generate_lnt-$generate_all}"
post_all="${post_all-true}"
post_jira_comment="${post_jira_comment-$post_all}"
post_jira_card="${post_jira_card-$post_all}"
post_mail="${post_mail-$post_all}"
post_gcc_testresults="${post_gcc_testresults-$post_mail}"
post_icommits="${post_icommits-$post_all}"
post_dashboard="${post_dashboard-$post_all}"

dryrun="${dryrun-false}"
icommits="interesting-commits"
stage="${stage-full}"
verbose="${verbose-false}"

# Generate notification files based on current and previous artifacts, which
# are assumed to be from two consecutive builds.
#
# Notes:
# 1. Current artifacts are in $top_artifacts.  All files from $top_artifacts
#    are accessible read-only, and round-robin-notify.sh writes its files
#    into $top_artifacts/notify/.
# 2. Files in "numbered" directories NN-* should not be used, since they
#    will be eventually removed.
# 3. Previous artifacts are in base-artifacts/, which is a git repository.
#    All files from base-artifacts/ are accessible read-only.
# 4. Round-robin-notify.sh can assume that files in $top_artifacts/ and
#    in base-artifacts/ have been generated by the latest version of scripts.
#    Developers will run round-robin-rewrite.sh when format of files changes
#    to bring "history" results up-to-date with what current scripts expect.
# 5. Round-robin-notify.sh can assume that check_regression() step was run
#    just before invocation of round-robin-notify.sh.

if $verbose; then
    set -x
fi

if $dryrun; then
    dryrun="echo DRYRUN: "
else
    dryrun=""
fi


#========================================================================================
#
#                               SETUP/FINALIZE PROCEDURES
#
#========================================================================================

# affect the following global variables
declare top_artifacts ci_project ci_config notify_email
setup_notify_environment ()
{
    echo "# ${FUNCNAME[0]}"

    # setup global variables
    top_artifacts="${rr[top_artifacts]}"
    ci_project=$(get_current_manifest "{rr[ci_project]}")
    ci_config=$(get_current_manifest "{rr[ci_config]}")

    # Debug dump
    echo "# Debug traces :"
    echo "#   Baseline  : $(get_baseline_manifest BUILD_URL)"
    echo "#   Using dir : base-artifacts"
    echo "#   Artifacts : $(get_current_manifest BUILD_URL)"
    echo "#   Using dir : $top_artifacts"
    echo ""

    mkdir -p "$top_artifacts/notify"

    case "$notify" in
	*"@"*)
	    # Normally pw[PROJECT_patch_submitter] will have been set to
	    # "$notify" by precommit-ssh-apply.sh or pw-apply.sh.  Manually
	    # started jobs where ${component}_git doesn't start with "ssh://" or
	    # "pw://" are an exception though, because then those scripts aren't
	    # called and thus the $pw array will be empty.  Let's save the value
	    # here in case it's needed later.
	    notify_email="$notify"
	    notify="precommit"
	    ;;
    esac

    declare -Ag pw
    if [ "$notify" = "precommit" ]; then
	obligatory_variables pw_dir
	declare -g           pw_dir

	local pw_file
	while IFS= read -r -d '' pw_file; do
	    # shellcheck disable=SC1090
	    source "$pw_file"
	done < <(find "$pw_dir" -type f -print0)
    fi
}

# Exit with 0 status if given component has a single commit compared to baseline.
# $1: component
single_commit_p ()
{
    (
    set -euf -o pipefail

    local c="$1"

    local base_rev cur_rev sha1
    base_rev=$(get_baseline_git "${c}_rev")
    cur_rev=$(get_current_git "${c}_rev")

    for sha1 in $(git -C "$c" rev-parse "$cur_rev^@"); do
	if [ "$sha1" = "$base_rev" ]; then
	    # We have a single-commit build
	    return 0
	fi
    done

    return 1
    )
}

# check_source_changes : Looks at base-artifacts and artifacts to compute the
# changes for the toolchain source files.
# This procedure updates the following global variables for the rest of notify pass.
declare change_kind changed_single_component last_good first_bad
declare -a changed_components

check_source_changes ()
{
    echo "# ${FUNCNAME[0]}"

    # Set ${changed_components[@]} unless this is "init" build.  For "init"
    # builds we have no baseline_git data, so leave changed_components empty.
    if [ "$(get_current_manifest "{rr[update_baseline]}")" != "init" ]; then
	IFS=" " read -r -a changed_components <<< "$(print_changed_components)"
    else
	changed_components=()
    fi

    local c base_rev cur_rev c_commits
    if [ ${#changed_components[@]} = 0 ]; then
	change_kind="no_change"
	changed_single_component=""
    elif [ ${#changed_components[@]} = 1 ]; then
	changed_single_component="${changed_components[0]}"
	first_bad="$(get_current_git ${changed_single_component}_rev)"
	last_good="$(get_baseline_git ${changed_single_component}_rev)"

	# single_commit_p() depend on $c to exist
	local res
	git -C "$changed_single_component" rev-parse --verify "HEAD" \
	    &>/dev/null &
	res=0 && wait $! || res=$?
	assert_with_msg "Cannot parse HEAD in repo $changed_single_component" \
			[ $res = 0 ]

	if single_commit_p "$changed_single_component"; then
	    change_kind="single_commit"
	else
            change_kind="single_component"
	fi
    else
	change_kind="multiple_components"
	changed_single_component=""
    fi

    # Debug dump
    echo "# Debug traces :"
    echo "#   change_kind=$change_kind : ${changed_components[*]}"
    for c in "${changed_components[@]}"; do
       base_rev=$(get_baseline_git ${c}_rev)
       cur_rev=$(get_current_git ${c}_rev)
       c_commits=$(git -C $c rev-list --count $base_rev..$cur_rev || echo "??")
       echo "#   rev for $c : $base_rev..$cur_rev ($c_commits commits)"
    done
    echo ""
}

# print routines pointers
declare print_commits_f print_result_f print_config_f print_last_icommit_f

setup_stages_to_run ()
{
    # if everything is fine. No reason to report to jira & icommits
    if [ "$notify" = "onregression" ] \
	   && { [ "${rr[no_regression_result]}" = "0" ] \
		    || [ "$change_kind" != "single_commit" ]; }; then
	notify=ignore
    elif [ "$notify" = "precommit" ] \
	     && [ "${rr[no_regression_result]}" = "0" ]; then
	notify=ignore
    fi

    if [ "$notify" = "ignore" ] || [ "$notify" = "precommit" ]; then
	post_jira_comment=false
	post_jira_card=false
	post_icommits=false
	# Even in "ignore" mode (successful build with no regression),
	# we want to post gcc_testresults if relevant.
	if [ "$notify" = "ignore" ]; then
	    post_mail=false
	fi
	if [ "$notify" = "precommit" ]; then
	    post_gcc_testresults=false
	fi
    fi

    # Disable dashboard generation as results-vs-first is now disabled
    generate_dashboard=false
    post_dashboard=false

    # print routines pointers
    print_commits_f=print_commits
    print_result_f=print_result
    print_config_f=print_config
    print_last_icommit_f=print_last_icommit
    generate_extra_details_f=generate_extra_details
    case "$ci_project" in
	tcwg_binutils*|tcwg_bootstrap*|tcwg_gcc*|tcwg_gdb*|tcwg_glibc*|tcwg_gnu*)
	   print_result_f=gnu_print_result
	   generate_extra_details_f=gnu_generate_extra_details
	   print_config_f=gnu_print_config
	   ;;
	tcwg_bmk*)
	   print_result_f=bmk_print_result
	   print_config_f=bmk_print_config
	   generate_extra_details_f=bmk_generate_extra_details
	   ;;
	*)
	   # By default keep generic ones set above
	   ;;
    esac
}

release_notification_files ()
{
    echo "# ${FUNCNAME[0]}"
    if ! [ -d $top_artifacts/jenkins ]; then
      return
    fi

    local f
    for f in mail-body.txt mail-subject.txt mail-recipients.txt; do
      # copy the file if exists, and not emtpy.
      # this important for mail-recipient.txt that may be empty.
      if [ -s $top_artifacts/notify/$f ]; then
        cp $top_artifacts/notify/$f $top_artifacts/jenkins/$f
      fi
    done

    echo "... Done"
}

release_gcc_testresults_files ()
{
    echo "# ${FUNCNAME[0]}"
    if ! [ -d $top_artifacts/jenkins ]; then
      return
    fi

    # Send GCC test results only if the baseline was updated more than
    # 1 day ago, so that we send at most one such email per day.
    if [ -f $top_artifacts/testresults/testresults-mail-body.txt ]; then
	base_d=$(get_baseline_manifest "{rr[gcc_testresults_date]}")
	cur_d=$(get_current_component_date gcc || true)
	if [ x"$cur_d" = x"" ]; then
	    return
	fi
	if [ x"$base_d" = x"" ]; then
	    base_d=0
	fi

	if [ $(( $cur_d - $base_d )) -ge 86400 ]; then
	    # The current date will become the baseline for the next build
	    # if the current baseline occurred more than one day ago.
	    cp $top_artifacts/testresults/testresults-mail-subject.txt \
	       $top_artifacts/testresults/testresults-mail-body.txt $top_artifacts/jenkins/
	    echo "gcc-testresults@gcc.gnu.org" > $top_artifacts/jenkins/testresults-mail-recipients.txt
	    cat <<EOF | manifest_out
rr[gcc_testresults_date]=$cur_d
EOF
	else
	    # Otherwise, record the baseline date as the current
	    # gcc_testresults_date.  Failing that, the next build will
	    # compare its date with 0 and may send its results more
	    # often than every day.
	    cat <<EOF | manifest_out
rr[gcc_testresults_date]=$base_d
EOF
	fi
    fi

    echo "... Done"
}

#========================================================================================
#
#                              GENERATE EXTRA DETAILS
#
#========================================================================================
gnu_generate_extra_details()
{
    (
    set -euf -o pipefail

    # Extract 'configure' and 'make' lines for steps where it makes sense.
    local tmpfile
    tmpfile=$(mktemp)

    echo > $top_artifacts/notify/configure-make.txt

    while read step_console
    do
	artifact_dir="$(dirname ${step_console})"
	(
	    # We want to accept that 'xzgrep' can have zero match below
	    set +o pipefail
	    xzgrep RUN: ${step_console} | sed 's/.* RUN: //' > $tmpfile
	)

	grep /configure $tmpfile > $tmpfile-configure || true
	grep "^make " $tmpfile > $tmpfile-make || true
	if [ -s $tmpfile-configure ] || [ -s $tmpfile-make ]; then
	    echo "# $(basename $artifact_dir)" >> $top_artifacts/notify/configure-make.txt
	    cat $tmpfile-configure $tmpfile-make >> $top_artifacts/notify/configure-make.txt
	    echo >> $top_artifacts/notify/configure-make.txt
	fi
    done < <(set +f ; find $top_artifacts/[0-9][0-9]-* -name console.log.xz)

    rm -f $tmpfile $tmpfile-configure $tmpfile-make

    [ -s $top_artifacts/notify/configure-make.txt ] || rm -f $top_artifacts/notify/configure-make.txt

    if ! [ -d $top_artifacts/sumfiles ]; then
	return 0
    fi

    # Compare results using compare_tests for information purposes.
    # It works well only when we compare complete testsuites.
    # shellcheck disable=SC2154
    gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
				      base-artifacts/sumfiles \
				      "$top_artifacts/sumfiles" \
				      > $top_artifacts/notify/results.compare.txt &
    wait $! || true
    )
}

bmk_generate_extra_details()
{
    (
    set -euf -o pipefail
    local artifacts_mail_dir

    artifacts_mail_dir=$top_artifacts/notify

    $scripts/../bmk-scripts/output-bmk-results.py \
	--compare_results $top_artifacts/results-vs-prev/compare-results-internal.csv \
	--variability_file $top_artifacts/results-vs-prev/bmk-specific-variability-max.csv \
	--variability_file_data "max" \
	--run_step_dir "$artifacts_mail_dir/" \
	--metric "${rr[metric_id]}" --mode "${rr[mode]}" \
	--details verbose > "$artifacts_mail_dir/output-bmk-results.log" &

    local res=0 && wait $! || res=$?
    assert_with_msg "ERROR while trying to regenerate bmk-data results. Aborting.." [ $res = 0 ]
    )
}

generate_extra_details()
{
    (
    # nothing to do
    true
    )
}

#========================================================================================
#
#                              PRINT ROUTINES
#
#========================================================================================

dump_model_only=${dump_model_only-false}

#==========================================
#   *GENERIC* PRINT ROUTINES
#==========================================

###### PRINT COMMITS
#  --link    : link to the commit (only if single_commit)
#  --oneline : commit title
#  --short   : commit log
print_commits()
{
    (
    set -euf -o pipefail
    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
    local print_arg=$1

    if [ "$change_kind" = "no_change" ]; then
        echo "baseline build"
	return 0
    fi

    local more_lines
    if [ "$change_kind" = "single_commit" ]; then
	local c="${changed_single_component}"

	if [ "$print_arg" = "--link" ]; then
	    local url
            url=$(get_baseline_git ${c}_url)
	    if [[ "$url" =~ git://sourceware.org/git/ ]]; then
		url="${url#git://sourceware.org/git/}"
		url="https://sourceware.org/git/?p=$url"
		echo "$url;a=commitdiff;h=$first_bad"
	    elif [[ "$url" =~ https://github.com/ ]] \
		     || [[ "$url" =~ https://gitlab.com/ ]]; then
		echo "${url%.git}/commit/$first_bad"
	    elif [[ "$url" =~ https://git.linaro.org/ ]]; then
		echo "${url}/commit/?id=$first_bad"
	    else
		echo "See in $url"
	    fi

	    return 0
	fi

	local describe
	if [ "${pw[$c]-}" = "" ]; then
	    describe=$(describe_sha1 "$c" "$first_bad" true)
	    # Remove the leading "basepoints/"
	    describe=$(echo "$describe" | sed 's,^basepoints/,,')
	else
	    describe="$c patch #${pw[${c}_patch_id]}"
	fi

	if [ "$print_arg" = "--oneline" ]; then
	    echo "$describe"
	    return 0
	fi

	if [ "${pw[$c]-}" = "" ]; then
	    echo "commit $describe"
	else
	    echo "$c patch ${pw[${c}_patch_url]}"
	fi
	local tmpfile
	tmpfile=$(mktemp)
        git -C "$c" log -n1 "$first_bad" | tail -n +2 > "$tmpfile"
	head -n 10 $tmpfile
	more_lines=$(($(cat "$tmpfile" | wc -l) - 10))
	if [ $more_lines -gt 0 ]; then
	    echo "... $more_lines lines of the commit log omitted."
	fi
	rm $tmpfile

	if [ "${pw[$c]-}" != "" ]; then
	    echo "... applied on top of baseline commit:"
	    git -C $c log -n1 --oneline $last_good || true
	fi
	return 0
    fi

    if [ "$change_kind" = "single_component" ] \
	   || [ "$change_kind" = "multiple_components" ]; then
	local new_commits c base_rev cur_rev c_commits components

	local commits_or_patches
	if [ "${pw[project]-}" != "" ]; then
	    commits_or_patches="patches"
	else
	    commits_or_patches="commits"
	fi

	new_commits=0
	for c in "${changed_components[@]}"; do
	    base_rev=$(get_baseline_git ${c}_rev)
	    cur_rev=$(get_current_git ${c}_rev)
	    c_commits=$(git -C $c rev-list --count $base_rev..$cur_rev \
			    || echo 0)
	    new_commits=$(($new_commits + $c_commits))
        done
	components=$(echo "${changed_components[@]}" | tr ' ' ',')

	echo "$new_commits $commits_or_patches in $components"

	if [ "$print_arg" = "--oneline" ]; then
	    return 0
	fi

	for c in "${changed_components[@]}"; do
	    base_rev=$(get_baseline_git ${c}_rev)
	    cur_rev=$(get_current_git ${c}_rev)
	    c_commits=$(git -C $c rev-list --count $base_rev..$cur_rev \
			    || echo 0)

	    if [ "${pw[$c]-}" != "" ]; then
		echo "Patchwork URL: ${pw[${c}_patch_url]}"
	    fi

	    git -C $c log -n 5 --oneline $base_rev..$cur_rev || true
	    if [ $c_commits -gt 5 ]; then
		echo "... and $(($c_commits-5)) more $commits_or_patches in $c"
	    fi

	    if [ "${pw[$c]-}" != "" ]; then
		echo "... applied on top of baseline commit:"
		git -C $c log -n1 --oneline $base_rev || true
	    fi
        done

	return 0
    fi
    )
}

###### PRINT the RESULTS of this build
#  --oneline : either success ot failure
#  --short   : change of results.txt file between baseline and artifact
#  --long    : idem
print_result()
{
   $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

   local print_arg=$1
   case "$print_arg" in
       --oneline)
	   if [ "${rr[no_regression_result]}" = "0" ]; then
	       echo "Success"
	   else
	       echo "Failure"
	   fi
	   ;;
    --short|--long)
	echo "Results changed to"
	echo "$(cat $top_artifacts/results)"
	echo ""
	echo "From"
	echo "$(cat base-artifacts/results)"
	;;
   esac
}

###### PRINT LAST ICOMMIT
#  Extract the last status from icommit
#  --entry : output directory of icommit entry
#  --status : will output the icommit status
#  --reproduction_instructions_link : will output the link to the reproduction_instructions
print_last_icommit ()
{
    (
    $dump_model_only && echo "<<${FUNCNAME[0]}>>" && return
    set -euf -o pipefail
    local print_arg="$1"
    shift 1

    if [ x"$change_kind" != x"single_commit" ]; then
	return 0
    fi

    local isubdir
    isubdir=$(interesting_subdir "$changed_single_component" "$first_bad" "$@")

    case "$print_arg" in
	--entry)
	    echo "$icommits/$isubdir"
	    ;;
	--status)
            cat "$icommits/$isubdir/status.txt"
            ;;
	--reproduction_instructions_link)
	    print_icommits_link "$isubdir/reproduction_instructions.txt"
	    ;;
    esac
    )
}

# Print link url to interesting-commits.git repo for "$path".
print_icommits_link ()
{
    (
    set -euf -o pipefail
    local path="$1"

    local url="https://git-us.linaro.org/toolchain/ci/interesting-commits.git"
    echo "$url/plain/$path"
    )
}

# CHECK_IF_FIRST_REPORT : Check if this is the first report in icommits
# Result stored in first_icommit_to_report global variable
check_if_first_report()
{
   declare -g first_icommit_to_report

   first_icommit_to_report=false
   if [ x"$change_kind" != x"single_commit" ]; then
       return
   fi

   local isubdir
   isubdir=$(interesting_subdir "$changed_single_component" "$first_bad")
   if ! [ -f "$icommits/$isubdir/first_url" ]; then
       return
   fi

   local first_url
   first_url=$(cat "$icommits/$isubdir/first_url")
   if [ "$first_url" = "$(get_current_manifest BUILD_URL)" ]; then
       first_icommit_to_report=true
   elif [ "$notify" = "onregression" ]; then
       # Send out emails for post-commit "onregression" reports only for
       # the first detection.
       post_mail=false
   fi
}

###### PRINT the configuration of this build
#  --oneline : target short name (eg: arm/aarch64)
#  --short   : short "pretty" version suitable for summary
#  --long    : full details
print_config()
{
    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

    local print_arg=$1
    case "$print_arg" in
	--oneline)
	    case "$ci_config" in
		*arm*) echo "arm" ;;
		*aarch64*) echo "aarch64" ;;
		*) echo "$ci_config" ;;
	    esac
	    ;;
	--short|--long)
	    echo "CI config $ci_project/$ci_config"
	    ;;
    esac
}

print_artifacts_url ()
{
    (
    set -euf -o pipefail

    local url
    url="$(get_current_manifest BUILD_URL)artifact/artifacts"
    if [ "${pw[project]-}" != "" ]; then
	url="$url/artifacts.precommit"
    fi
    echo "$url/$*"
    )
}

#==========================================
#   *GNU* PRINT ROUTINES
#==========================================

###### PRINT the configuration of this build
#  --oneline : target short name (eg: arm/aarch64)
#  --short   : short "pretty" version suitable for summary
#  --long    : full details
gnu_print_config()
{
    (
    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

    # shellcheck source=tcwg_gnu-config.sh
    . $scripts/tcwg_gnu-config.sh

    settings_for_ci_project_and_config "$ci_project" "$ci_config"

    local print_arg=$1
    case "$print_arg" in
	--oneline)
	    print_config "$print_arg"
	    ;;
	--short)
	    echo "${gnu_data[pretty_project]} ${gnu_data[pretty_config]}"
	    ;;
	--long)
	    echo "CI config $ci_project ${gnu_data[long_config]}"
	    ;;
    esac
    )
}

# most of them mapped to generic implementation
gnu_print_result()
{
    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return
    local print_arg="$1"

    if ! [ -d $top_artifacts/sumfiles ]; then
	print_result "$@"
	return 0
    fi

    # From now on, we should have called already gnu_generate_extra_details

    local validate_failures="gcc-compare-results/contrib/testsuite-management/validate_failures.py"
    local xfails="$top_artifacts/sumfiles/xfails.xfail"

    if ! [ -f "$xfails" ]; then
	return 0
    fi

    "$validate_failures" --manifest="$xfails" \
			 --expiry_date="${rr[result_expiry_date]}" \
			 --build_dir="$top_artifacts/sumfiles" --verbosity=1 \
			 > $top_artifacts/notify/regressions.sum &
    wait $! || true
    "$validate_failures" --inverse_match --manifest="$xfails" \
			 --expiry_date="${rr[result_expiry_date]}" \
			 --build_dir="$top_artifacts/sumfiles" --verbosity=1 \
			 > $top_artifacts/notify/progressions.sum &
    wait $! || true

    local n_regressions n_progressions pass_fail=PASS
    if [ "${rr[no_regression_result]}" != "0" ]; then
	pass_fail=FAIL
    fi
    n_regressions=$(grep -c "^[A-Z]\+:" \
			 $top_artifacts/notify/regressions.sum || true)
    n_progressions=$(grep -c "^[A-Z]\+:" \
			  $top_artifacts/notify/progressions.sum || true)

    printf "$pass_fail"
    if [ "$n_regressions" != "0" ]; then
	printf ": $n_regressions regressions"
    else
	rm $top_artifacts/notify/regressions.sum
    fi
    if [ "$n_progressions" != "0" ]; then
	printf ": $n_progressions progressions"
    else
	rm $top_artifacts/notify/progressions.sum
    fi
    printf "\n"

    if [ "$print_arg" = "--oneline" ]; then
	return 0
    fi

    local length=10 outfile n_lines
    if [ "$print_arg" = "--long" ]; then
	# Ask "head" to print out all the lines except for the last 0 lines.
	length=-0
    fi

    for outfile in regressions.sum progressions.sum; do
	if ! [ -f $top_artifacts/notify/$outfile ]; then
	    continue
	fi

	echo
	echo "$outfile:"
	n_lines=$(cat $top_artifacts/notify/$outfile \
		      | grep -v "Results Summary" | wc -l)
	n_lines=$(($n_lines - $length))
	head -n$length $top_artifacts/notify/$outfile \
	     | grep -v "Results Summary"
	if [ $n_lines -gt 0 ] && [ $length != -0 ]; then
	    echo "... and $n_lines more entries"
	fi
    done

    cat <<EOF

You can find the failure logs in *.log.1.xz files in
 - $(print_artifacts_url 00-sumfiles/)
The full lists of regressions and progressions as well as configure and make commands are in
 - $(print_artifacts_url notify/)
The list of [ignored] baseline and flaky failures are in
 - $(print_artifacts_url sumfiles/xfails.xfail)
EOF
}


#==========================================
#   *BMK* PRINT ROUTINES
#==========================================

# print_result. either oneline version or short/long.
# Here is an ex :
# --oneline    :
#     644.nab_s slowed down of 4%
# --short/long :
#     the following benchmarks slowed down by more than 3%:
#     - 644.nab_s slowed down by 4% from 112963 to 117189 perf samples
bmk_print_result()
{
    (
    set -euf -o pipefail

    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

    local print_arg=$1

    artifacts_mail_dir=$top_artifacts/notify

    if [ "$stage" != "full" ]; then
      return
    fi

    ## Prepare data

    # From now on, we should have called already bmk_generate_extra_details

    # If there's one regression. Don't bother about improvements.
    local improved_or_regressed
    if [ -f $artifacts_mail_dir/exe.regression ] || [ -f $artifacts_mail_dir/symbol.regression ]; then
	improved_or_regressed=regression
    else
	improved_or_regressed=improvement
    fi

    declare -A changed_by_msg
    changed_by_msg[size-regression]="grew in size by"
    changed_by_msg[size-improvement]="reduced in size by"
    changed_by_msg[sample-regression]="slowed down by"
    changed_by_msg[sample-improvement]="speeds up by"
    changed_by_msg[num_vect_loops-regression]="reduced the number of vect loops by"
    changed_by_msg[num_vect_loops-improvement]="increased the number of vect loops by"
    changed_by_msg[num_sve_loops-regression]="reduced the number of sve instructions by"
    changed_by_msg[num_sve_loops-improvement]="increased the number of sve instructions by"
    changed_by=${changed_by_msg[${rr[metric_id]}-$improved_or_regressed]}

    # FIXME: Remove hard-coded thresholds
    # thresholds
    case ${rr[metric_id]} in
	size)
	    exe_threshold=1	# We use 1% tolerance for binary size
	    symbol_threshold=10 # and 10% tolerance for symbol size.
	    ;;
	sample)
	    # Reduce thresholds when bisecting to avoid considering borderline
	    # regressions as spurious.  This should break cycles of build and
	    # bisect jobs triggering each other on borderline regressions.
	    exe_threshold=3
	    symbol_threshold=15
	    ;;
	num_vect_loops|num_sve_loops)
	    exe_threshold=0
	    symbol_threshold=0
	    ;;
	*) assert false ;;
    esac

    # Now print result
    case "$print_arg" in
     --oneline)
	assert_with_msg "Builds with infra problems should never get here" \
			[ "${rr[no_regression_result]}" != "$EXTERNAL_FAIL" ]

	# Generate readable oneline diag
	local metric bmk symbol short_diag long_diag
	if [ -f $artifacts_mail_dir/exe.$improved_or_regressed ]; then
	    # shellcheck disable=SC2034
	    IFS=, read metric bmk symbol short_diag long_diag < <(head -n1 $artifacts_mail_dir/exe.$improved_or_regressed)
	elif [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
	    # shellcheck disable=SC2034
	    IFS=, read metric bmk symbol short_diag long_diag < <(head -n1 $artifacts_mail_dir/symbol.$improved_or_regressed)
	else
	    short_diag="No change"
	fi
	echo "$short_diag"
	;;

    --short|--long)
	# The following exe regressed/improved:
	if [ -f $artifacts_mail_dir/exe.$improved_or_regressed ]; then
	   sort -gr -o $artifacts_mail_dir/exe.$improved_or_regressed \
                    $artifacts_mail_dir/exe.$improved_or_regressed

	   echo "the following benchmarks $changed_by more than ${exe_threshold}%:"
	   local metric exe symbol short_diag long_diag
	   while IFS=, read metric exe symbol short_diag long_diag; do
              echo "- $long_diag"
              if [ -f $artifacts_mail_dir/$exe.symbols-$improved_or_regressed ]; then
                 while IFS=, read metric bmk symbol short_diag long_diag; do
                   echo "  - $long_diag"
                 done < $artifacts_mail_dir/$exe.symbols-$improved_or_regressed
                 # Delete $bmk.regressions so that it doesn't show up
                 # in symbol-regression loop below.
                 rm $artifacts_mail_dir/$exe.symbols-$improved_or_regressed
              fi
	   done < $artifacts_mail_dir/exe.$improved_or_regressed
	fi

	# The following functions regressed/improved:
	if [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
	   echo "the following hot functions $changed_by more than ${symbol_threshold}% (but their benchmarks $changed_by less than ${exe_threshold}%):"
	   local metric bmk symbol short_diag long_diag
	   # shellcheck disable=SC2034
	   while IFS=, read metric bmk symbol short_diag long_diag; do
	     echo "- $long_diag"
	   done < $artifacts_mail_dir/symbol.$improved_or_regressed
	fi

	if ! [ -f $artifacts_mail_dir/exe.$improved_or_regressed ] && ! [ -f $artifacts_mail_dir/symbol.$improved_or_regressed ]; then
	   echo "No change"
	fi
        ;;
    esac
    )
}

###### PRINT the configuration of this build
#  --oneline : target short name (eg: arm/aarch64)
#  --short   : short "pretty" version suitable for summary
#  --long    : full details
bmk_print_config()
{
    # shellcheck source=tcwg_bmk-config.sh
    . $scripts/tcwg_bmk-config.sh

    $dump_model_only && echo "<<${FUNCNAME[0]} $*>>" && return

    # ${ci_project}--${ci_config} format is :
    #  'tcwg_bmk-#{PROFILE_NAME}-#{BMK}--#{TOOLCHAIN}-#{TARGET}-{toolchain_ver}-{cflags}'
    IFS=- read -a ci_pjt_cfg <<EOF
$ci_project--$ci_config
EOF
    local toolchain target cflags
    toolchain="${ci_pjt_cfg[4]}"
    target="${ci_pjt_cfg[5]}"
    cflags="${ci_pjt_cfg[7]}"

    local compiler="" libc="" linker="" version="" bmk_flags="" hw=""
    bmk_flags=$(echo "$cflags" | sed -e "s/_/ -/g")

    local print_arg=$1
    case "$print_arg" in
	--oneline)
	    case "$ci_config" in
		*arm*) echo "arm $bmk_flags" ;;
		*aarch64*) echo "aarch64 $bmk_flags" ;;
		*) echo "$ci_config" ;;
	    esac
	    return 0
	    ;;
	--short)
	    print_config "$print_arg"
	    return 0
	    ;;
    esac

    # --long as default

    local bmk_suite publish_save_temps
    bmk_suite=""
    publish_save_temps=false
    case "$(tcwg_bmk_benchs)" in
	coremark)
	    bmk_suite="EEMBC CoreMark"
	    ;;
	spec2k6|4*)
	    bmk_suite="SPEC CPU2006"
	    publish_save_temps=true
	    ;;
	spec2017*|5*|6*)
	    bmk_suite="SPEC CPU2017"
	    publish_save_temps=true
	    ;;
    esac

    cat <<EOF
Below reproducer instructions can be used to re-build both "first_bad" and "last_good" cross-toolchains used in this bisection.  Naturally, the scripts will fail when triggerring benchmarking jobs if you don\'t have access to Linaro TCWG CI.
EOF

    # Copy save-temps tarballs to artifacts, so that they are accessible.
    # We can publish pre-processed source only for benchmarks derived from
    # open-source projects.
    # Note that we include save-temps artifacts for successful builds so that
    # "last_good" build has the artifacts.
    if $publish_save_temps; then
	mkdir -p ${top_artifacts}/top-artifacts
	local s_t
	while read s_t; do
	    case "$s_t" in
		400.perlbench*|500.perlbench*|600.perlbench*) ;;
		401.bzip2*) ;;
		403.gcc*|502.gcc*|602.gcc*) ;;
		435.gromacs*) ;;
		436.cactusADM*|507.cactuBSSN*|607.cactuBSSN*) ;;
		445.gobmk*) ;;
		454.calculix*) ;;
		456.hmmer*) ;;
		462.libquantum*) ;;
		465.tonto*) ;;
		481.wrf*|521.wrf*|621.wrf*) ;;
		482.sphinx3*) ;;
		483.xalanc*) ;;
		505.mcf*|605.mcf*)
		# 429.mcf is not present in redistributable_sources/ :-|
		;;
		511.povray*) ;;
		525.x264*|625.x264*)
		# 464.h264ref is not present in redistributable_sources/ :-|
		;;
		526.blender*) ;;
		527.cam4*|627.cam4*) ;;
		538.imagick*|638.imagick*) ;;
		544.nab*|644.nab*) ;;
		554.roms*|654.roms*) ;;
		557.xz*|657.xz*) ;;
		628.pop2) ;;
		*)
		    # Can't redistribute benchmark sources.
		    continue
		    ;;
	    esac
	    cp "$s_t" ${top_artifacts}/top-artifacts/save-temps/
	done < <(find results-1 -path "save.*.temps/*.tar.xz")
    fi

    if [ -d ${top_artifacts}/top-artifacts/save-temps/ ]; then
	cat <<EOF

For your convenience, we have uploaded tarballs with pre-processed source and assembly files at:
- First_bad save-temps: \$FIRST_BAD_ARTIFACTS/save-temps/
- Last_good save-temps: \$LAST_GOOD_ARTIFACTS/save-temps/
- Baseline save-temps: \$BASELINE_ARTIFACTS/save-temps/
EOF
    fi

    case "$toolchain" in
	gnu)
	    compiler="GCC"
	    libc="Glibc"
	    linker="GNU Linker"
	    ;;
	gnu_eabi)
	    compiler="GCC"
	    libc="Newlib"
	    linker="GNU LD"
	    ;;
	llvm)
	    compiler="Clang"
	    libc="Glibc"
	    linker="LLVM Linker"
	    ;;
    esac
    case "$ci_config" in
	*-master-*) version="tip of trunk" ;;
	*-release-*) version="latest release branch" ;;
    esac
    case "$(tcwg_bmk_hw)" in
	*apm*) hw="APM Mustang 8x X-Gene1" ;;
	*tk1*) hw="NVidia TK1 4x Cortex-A15" ;;
	*tx1*) hw="NVidia TX1 4x Cortex-A57" ;;
	*stm32*) hw="STMicroelectronics STM32L476RGTx 1x Cortex-M4" ;;
	*fx*) hw="Fujitsu FX700 48x AA64" ;;
	*qc*) hw="Qualcomm 8x AA64" ;;
	*) hw="<unknown>"
    esac

    cat <<EOF

Configuration:
- Benchmark: $bmk_suite
- Toolchain: $compiler + $libc + $linker
- Version: all components were built from their $version
- Target: $(print_gnu_target $target)
- Compiler flags: $bmk_flags
- Hardware: $hw

This benchmarking CI is work-in-progress, and we welcome feedback and suggestions at linaro-toolchain@lists.linaro.org .  In our improvement plans is to add support for SPEC CPU2017 benchmarks and provide "perf report/annotate" data behind these reports.
EOF
}

#========================================================================================
#
#                            INTERESTING_COMMITS PROCEDURES
#
#========================================================================================

# Print out merged version of status-summary.txt files at a given subdir level.
# As an initial approximation, just pick a summary with the biggest number.
merge_status_summary ()
{
    (
    set -euf -o pipefail
    local subdir="$1"

    local cur_file cur best="" best_file

    while read -r cur_file; do
	# Extract a number " N " or " N%"
	cur=$(sed -e "s/.* \([0-9]\+\)[ %].*/\1/" "$cur_file")
	if ! [ "$cur" -le "$best" ]; then
	    best="$cur"
	    best_file="$cur_file"
	fi
    done < <(find "$subdir" -mindepth 2 -maxdepth 2 \
		  -name "status-summary.txt" | sort)

    cat "$best_file"
    )
}

# update_interesting_commits : will update the local icommit repository with
# the new run.
# If it is a regression, single_commit, may be updating first_report variable
#
# Interesting-commits store regression history in the following hierarchy:
# 1. At the top level we have COMPONENT directories.
# 2. At the 2nd level we have SHA1 directories.
# 2a. We also have "git describe" symlinks to SHA1 directories for convenience.
# 3. At the 3rd level we have CI_PROJECT directories.
# 4. At the 4th level we have CI_CONFIG directories, and ...
# 4a. ... status.txt, which contains status of changes from SHA1 across all
#     CI_CONFIGs in CI_PROJECT.
# 5. At the 5th level we have per-build files last_good, details.txt, etc.
update_interesting_commits ()
{
    echo "# ${FUNCNAME[0]}"

    local stage="$1"
    local jira_key="$2"

    # We have successfully identified a bad commit with a good parent!
    # Add $first_bad to interesting commits.

    local subdir3 subdir4 subdir5
    # Commit top-level dir
    subdir3=$(interesting_subdir "$changed_single_component" "$first_bad")
    # Commit project-level dir
    subdir4=$(interesting_subdir "$changed_single_component" "$first_bad" \
				 "$ci_project")
    # Commit config-level dir
    subdir5=$(interesting_subdir "$changed_single_component" "$first_bad" \
				 "$ci_project" "$ci_config")

    if ! [ -d "$icommits/$subdir3" ]; then
	mkdir -p $icommits/$subdir3
	get_current_manifest BUILD_URL > $icommits/$subdir3/first_url
	git -C $icommits add $subdir3/first_url
    fi

    # Update interesting-commits/$component/$first_bad/$ci_project/$ci_config
    mkdir -p "$icommits/$subdir5"
    echo "$(get_current_manifest BUILD_URL)artifact/artifacts" > "$icommits/$subdir5/build_url"
    echo "$last_good" > "$icommits/$subdir5/last_good"
    git -C "$icommits" add "$subdir5/build_url" "$subdir5/last_good"

    if [ "$stage" != "full" ]; then
	return
    fi

    # $icommit/<comp>/sha1/<sha1>/.../status-summary.txt
    $print_result_f --oneline > "$icommits/$subdir5/status-summary.txt"
    merge_status_summary $icommits/$subdir4 \
			 > "$icommits/$subdir4/status-summary.txt"
    merge_status_summary $icommits/$subdir3 \
			 > "$icommits/$subdir3/status-summary.txt"
    git -C "$icommits" add \
	"$subdir5/status-summary.txt" \
	"$subdir4/status-summary.txt" \
	"$subdir3/status-summary.txt"

    $print_result_f --long > "$icommits/$subdir5/details.txt"
    if [ -f $top_artifacts/notify/configure-make.txt ]; then
	cat $top_artifacts/notify/configure-make.txt >> "$icommits/$subdir5/details.txt"
    fi

    # $icommit/<comp>/sha1/<sha1>/<ci_project>/<ci_config>/status.txt
    (
	cat "$icommits/$subdir5/status-summary.txt"
	print_icommits_link "$subdir5/details.txt"
	cat $icommits/$subdir5/build_url
    ) | sed "s/^/* /" > "$icommits/$subdir5/status.txt"

    git -C "$icommits" add "$subdir5/details.txt" "$subdir5/status.txt"

    # FIXME: Remove transisional workaround for summary.txt -> status.txt
    if [ -f "$icommits/$subdir5/summary.txt" ]; then
	git -C "$icommits" rm "$subdir5/summary.txt"
    fi

    # $icommit/<comp>/sha1/<sha1>/<ci_project>/<ci_config>/reproduction_instructions.txt
    local bad_artifacts_url good_artifacts_url
    bad_artifacts_url="$(get_current_manifest BUILD_URL)artifact/artifacts"
    good_artifacts_url="$(get_baseline_manifest BUILD_URL)artifact/artifacts"
    cat > $icommits/$subdir5/reproduction_instructions.txt << EOF
mkdir -p investigate-$changed_single_component-$first_bad
cd investigate-$changed_single_component-$first_bad

# Fetch scripts
git clone https://git.linaro.org/toolchain/jenkins-scripts

# Fetch manifests for bad and good builds
mkdir -p bad/artifacts good/artifacts
curl -o bad/artifacts/manifest.sh $bad_artifacts_url/manifest.sh --fail
curl -o good/artifacts/manifest.sh $good_artifacts_url/manifest.sh --fail

# Reproduce bad build
(cd bad; ../jenkins-scripts/${build_script} @@rr[top_artifacts] artifacts)
# Reproduce good build
(cd good; ../jenkins-scripts/${build_script} @@rr[top_artifacts] artifacts)
EOF
    git -C "$icommits" add $subdir5/reproduction_instructions.txt

    # $icommit/<comp>/sha1/<sha1>/<ci_project>/status.txt
    local ci_config
    while read ci_config; do
	# FIXME: Remove transisional workaround for summary.txt -> status.txt
	if [ -f "$icommits/$subdir4/$ci_config/summary.txt" ]; then
	    echo "* $ci_config"
	    cat $icommits/$subdir4/$ci_config/summary.txt | sed "s/^/** /"
	    continue
	fi

	if ! [ -f "$icommits/$subdir4/$ci_config/status.txt" ]; then
	    continue
	fi

	echo "* $ci_config"
	cat $icommits/$subdir4/$ci_config/status.txt | sed "s/^/*/"
    done < <(cd $icommits/$subdir4; ls) > "$icommits/$subdir4/status.txt"
    git -C "$icommits" add "$subdir4/status.txt"

    # $icommit/<comp>/sha1/<sha1>/status.txt
    local ci_project
    while read ci_project; do
	if ! [ -f "$icommits/$subdir3/$ci_project/status.txt" ]; then
	    continue
	fi

	echo "* $ci_project"
	cat $icommits/$subdir3/$ci_project/status.txt | sed "s/^/*/"
    done < <(cd $icommits/$subdir3; ls) > "$icommits/$subdir3/status.txt"
    git -C "$icommits" add "$subdir3/status.txt"

    # $icommit/<comp>/sha1/<sha1>/commit-log.txt
    $print_commits_f --short > "$icommits/$subdir3/commit-log.txt"
    git -C "$icommits" add "$subdir3/commit-log.txt"

    if $generate_jira; then
	local jira_dir="$subdir3/jira"

	if [ -f "$icommits/$jira_dir/key" ]; then
	    assert_with_msg "Should not have created multiple jira cards" \
			    [ "$jira_key" = "" ]
	    jira_key=$(cat "$icommits/$jira_dir/key")
	fi

	# Recreate jira/ dir with up-to-date info.
	if [ -e "$icommits/$jira_dir" ]; then
	    git -C "$icommits" rm -rf "$jira_dir"
	fi
	mkdir "$icommits/$jira_dir"

	if [ "$jira_key" != "" ]; then
	    echo "$jira_key" > "$icommits/$jira_dir/key"
	    git -C "$icommits" add "$jira_dir/key"
	fi

	# Keep jira/summary in sync with mail-subject.txt
	echo "$($print_commits_f --oneline):" \
	     "$(cat "$icommits/$subdir3/status-summary.txt")" \
	     > "$icommits/$jira_dir/summary"
	git -C "$icommits" add "$jira_dir/summary"

	# FIXME: Unify format with email
	# We stop posting updates to jira cards once they are closed to avoid
	# spamming developers about resolved issues.  Therefore, jira data in
	# interesting-commits.git repo may be more up-to-date than in
	# the actual jira cards.  We add a link to jira/yaml in the card
	# description to make it easy for developers to look at the latest
	# info for closed cards.
	cat > "$icommits/$jira_dir/description" <<EOF
Commit: $($print_commits_f --link)
$(cat "$icommits/$subdir3/commit-log.txt")

$(cat "$icommits/$subdir3/status.txt")

Latest data: $(print_icommits_link "$jira_dir/yaml")
EOF
	git -C "$icommits" add "$jira_dir/description"

	update_jira_card
    fi

    # Generate a $describe_sha1 symlink
    local describe
    describe=$(describe_sha1 "$changed_single_component" "$first_bad" false)
    if [ "$describe" != "" ]; then
	local d
	d=$(dirname "$describe")
	mkdir -p $icommits/$changed_single_component/$d
	local symlink=""
	while [ "$d" != "." ]; do
	    symlink="../$symlink"
	    d=$(dirname "$d")
	done
	symlink="${symlink}sha1/$first_bad"
	# ??? For some reason I don't understand overwriting symlink with
	#     "ln -sf" may create a second [broken] symlink.
	rm -f $icommits/$changed_single_component/$describe
	ln -s $symlink $icommits/$changed_single_component/$describe
	git -C $icommits add $changed_single_component/$describe
    fi
}


# Generate entry in interesting-commits.git and, if $post_icommits,
# push it.  This is applicable only for single-commit case.
# This is called twice -- first time with $stage == init, and second time
# with $stage == full.
# During "init" stage we fetch existing entries and create a very basic entry
# if there isn't one.  This allows us to determine in check_if_first_report()
# whether this is the first build to discover $first_bad as interesting commit.
post_interesting_commits ()
{
    (
    set -euf -o pipefail
    echo "# ${FUNCNAME[0]}"

    local stage="$1"

    if [ "$change_kind" != "single_commit" ]; then
	return
    fi

    # Clone interesting-commits.git repo, which contains a regression
    # summaries for SHA1s.  These are the "first_bad" commits.
    clone_or_update_repo $icommits master \
			 https://git-us.linaro.org/toolchain/ci/interesting-commits.git \
			 auto master

    if ! $post_icommits; then
	dryrun="echo DRYRUN: "
    fi

    local jira_dir jira_key=""
    jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
    jira_dir="$jira_dir/jira"
    if [ "$stage" = "full" ] && $post_jira_card && $first_icommit_to_report \
	   && [ "$dryrun" = "" ]; then
	if ! [ -f "$icommits/$jira_dir/key" ]; then
	    # Create jira card for this interesting commit.
	    # We first create the card,
	    # then add a link to it in interesting-commits.git,
	    # then we update the card's content every time we change entry in
	    # interesting-commits.
	    jira_key=$(create_jira_card)
	else
	    echo "WARNING: jira card already exists $icommits/$jira_dir/key:" \
		 "$(cat "$icommits/$jira_dir/key")"
	fi
    fi

    # Regenerate interesting-commits entry and try to push to upstream.
    # - If failed to push, update icommits again
    #   This can happen if another job pushed to interesting-commits just
    #   before this job.
    # - If successful we consider round-robin-notify.sh to be successful.
    while true; do
	# Reset to master branch
	git -C $icommits remote update -p
	git_clean $icommits refs/remotes/origin/master

	update_interesting_commits "$stage" "$jira_key"

	# Commit changes (if any) to local clone of interesting-commits.git
	git -C $icommits commit \
	    -m "Add entry $first_bad from $(get_current_manifest BUILD_URL)" \
	    || break

	$dryrun git -C $icommits push \
		ssh://git-us.linaro.org/toolchain/ci/interesting-commits.git \
		HEAD:refs/heads/master &
	if wait $!; then
            # Push successful : stop here
	    break
	fi
	# Push failed. update icommit and retry pushing
    done
    )
}

#========================================================================================
#
#                            JIRA RELATED PROCEDURES
#
#========================================================================================

print_jira_template_card ()
{
    # Catch-all case for when project/config IDs change, so that we
    # won't miss notifications.  Forward all that to GNU-692.
    local jira_card="GNU-692"
    case "$ci_project/$ci_config:$changed_single_component" in
	tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;;
	tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;;
	tcwg_kernel/llvm-*:linux) jira_card="LLVM-647" ;;
	tcwg_kernel/llvm-*:*) jira_card="LLVM-646" ;;
	tcwg_bmk-*_speed*/gnu*) jira_card="GNU-689" ;;
	tcwg_bmk-*_size*/gnu*) jira_card="GNU-686" ;;
	tcwg_bmk-*_vect*/gnu*) jira_card="GNU-988" ;;
	tcwg_bmk-*_sve*/gnu*) jira_card="GNU-988" ;;
	tcwg_bmk-*_speed*/llvm*) jira_card="LLVM-651" ;;
	tcwg_bmk-*_size*/llvm*) jira_card="LLVM-650" ;;
	tcwg_bmk-*_vect*/llvm*) jira_card="LLVM-1013" ;;
	tcwg_bmk-*_sve*/llvm*) jira_card="LLVM-1013" ;;
	tcwg_aosp-*/*) jira_card="LLVM-1014" ;;
    esac
    echo "$jira_card"
}

# Create jira card for this interesting commit.
# Link to this card is stored in $icommits/.../jira/key
create_jira_card ()
{
    (
    set -euf -o pipefail

    local template project parent assignee yaml
    template=$(print_jira_template_card)
    project="${template%%-*}"
    parent=$(jipsearch -j "key=$template" -s parent:key \
		 | sed -e "s/.* , //")
    assignee=$(jipsearch -j "key=$template" -s assignee:emailAddress \
		   | sed -e "s/.* , //" || true)
    if [ "$assignee" = "" ]; then
	# The template card is unassigned, so use parent's assignee.
	assignee=$(jipsearch -j "key=$parent" -s assignee:emailAddress \
		       | sed -e "s/.* , //")
    fi

    yaml=$(mktemp)
    # shellcheck disable=SC2064
    trap "rm $yaml" EXIT

    cat > $yaml <<EOF
- Project: $project
  IssueType: Sub-task
  Parent: $parent
  Summary: $changed_single_component:$first_bad
  AssigneeEmail: $assignee
EOF
    local key
    key=$(jipcreate -f $yaml | sed -e "s#.*/##")
    echo "$key"
    )
}

# Print the existing jira card number for this interesting commit.
print_jira_card_key ()
{
    (
    set -euf -o pipefail

    local jira_dir
    jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
    jira_dir="$jira_dir/jira"

    if ! [ -f "$icommits/$jira_dir/key" ]; then
	return 0
    fi

    cat "$icommits/$jira_dir/key"
    )
}

# Update jira card for this interesting commit.
# Link to this card is stored in $icommits/.../jira/key
update_jira_card ()
{
    (
    set -euf -o pipefail
    echo "# ${FUNCNAME[0]}"

    local jira_dir
    jira_dir=$(interesting_subdir "$changed_single_component" "$first_bad")
    jira_dir="$jira_dir/jira"

    local -a components=()
    case "$changed_single_component" in
	binutils) components+=(Binutils) ;;
	gcc) components+=(GCC) ;;
	gdb) components+=(GDB) ;;
	glibc) components+=(Glibc) ;;
	linux) components+=(Linux) ;;
	llvm) components+=(LLVM) ;;
	newlib) components+=(Newlib) ;;
	qemu) components+=(QEMU) ;;
	*) components+=(CI) ;;
    esac
    (IFS=","; echo "${components[*]}") > "$icommits/$jira_dir/components"
    git -C "$icommits" add "$jira_dir/components"

    local commit_date
    commit_date=$(git -C "$changed_single_component" log -n1 \
		      --pretty="%cd" --date=iso "$first_bad")
    date -d "$commit_date" +%Y-%m-%d > "$icommits/$jira_dir/startdate"
    git -C "$icommits" add "$jira_dir/startdate"

    local key project
    key=$(print_jira_card_key)
    if [ -z "$key" ]; then
	echo "WARNING: no existing jira card $icommits/$jira_dir/key"
	return 0
    fi

    project="${key%%-*}"

    local yaml="$icommits/$jira_dir/yaml"

    cat > "$yaml" <<EOF
- Project: $project
  IssueType: Sub-task
  Key: $key
  Summary: |
EOF
    # Summary can have spaces and other special-to-yaml symbols;
    # quote using " |"
    sed -e "s/^/    /" "$icommits/$jira_dir/summary" >> "$yaml"
    cat >> "$yaml" <<EOF
  Components: $(cat "$icommits/$jira_dir/components")
  Start date: $(cat "$icommits/$jira_dir/startdate")
  Description: |
EOF
    sed -e "s/^/    /" "$icommits/$jira_dir/description" >> "$yaml"
    git -C "$icommits" add "$jira_dir/yaml"
    )
}

# Generate notify/jira/* files
generate_jira_dir()
{
    (
    set -euf -o pipefail

    local icommit_entry jira_key=""
    icommit_entry=$($print_last_icommit_f --entry)

    if [ "$icommit_entry" != "" ] && [ -d "$icommit_entry/jira" ]; then
	rsync -a "$icommit_entry/jira/" "$top_artifacts/notify/jira/"
	if [ -f "$top_artifacts/notify/jira/key" ]; then
	    jira_key=$(cat "$top_artifacts/notify/jira/key")
	fi
    else
	mkdir -p "$top_artifacts/notify/jira"
    fi

    if [ "$jira_key" != "" ]; then
	cat > $top_artifacts/notify/jira/comment-card.txt <<EOF
[$jira_key]
$($print_result_f --oneline)
Details: $(print_artifacts_url notify/mail-body.txt/*view*/)
EOF
	cat > $top_artifacts/notify/jira/comment-template.txt <<EOF
[$(print_jira_template_card)]
https://linaro.atlassian.net/browse/$jira_key
$($print_result_f --oneline)
Details: $(print_artifacts_url notify/mail-body.txt/*view*/)
EOF
    else
	cat > $top_artifacts/notify/jira/comment-template.txt << EOF
[$(print_jira_template_card)]
$($print_result_f --oneline)
Details: $(print_artifacts_url notify/mail-body.txt/*view*/)
EOF
    fi
    )
}

# Post update to Jira
post_to_jira ()
{
    (
    set -euf -o pipefail
    echo "# ${FUNCNAME[0]}"

    local post_card_comment=$post_jira_comment
    local post_template_comment=$post_jira_comment

    if $post_jira_card && [ -f $top_artifacts/notify/jira/yaml ]; then
	local key status
	key=$(print_jira_card_key)
	status=$(jipsearch -j "key=$key" -s status:name \
		     | sed -e "s/.* , //")

	# Do not update closed cards to avoid spamming developers about
	# updating resolved issues.  Note that this also skips posting
	# comments below.
	case "$status" in
	    "Closed")
		post_card_comment=false
		# In this case we may still post a comment to the template
		# card if $post_jira_comment is true.
		;;
	    *)
		$dryrun jipcreate -f $top_artifacts/notify/jira/yaml
		post_template_comment=false
		;;
	esac
    fi

    if $post_card_comment \
	    && [ -f $top_artifacts/notify/jira/comment-card.txt ]; then
	echo y | $dryrun jipdate \
			 -f $top_artifacts/notify/jira/comment-card.txt
    fi

    if $post_template_comment \
	    && [ -f $top_artifacts/notify/jira/comment-template.txt ]; then
	echo y | $dryrun jipdate \
			 -f $top_artifacts/notify/jira/comment-template.txt
    fi
    )
}


#========================================================================================
#
#                            MAIL RELATED PROCEDURES
#
#========================================================================================
# Model for mail-recipient.txt (may contain some of the following):
#    <author>, <compiler-mailing-list>, <linaro-toolchain-mailing-list>, <tcwg-validation-mailing-list>
#                                          (generate_mail_recipients : Generic)
#
# Model for mail-subject.txt :
#    [Linaro-TCWG-CI] <diag> after <changes> (generate_mail_subject : Generic)
#
# Model for mail-body.txt :
#    <mail regression details>                (generate_mail_body_regression : Specific to the kind of project)
#
#    <reproduction instructions details>      (generate_mail_body_reproduction_instructions : Generic)
#

print_mail_recipients ()
{
    (
    set -euf -o pipefail

    local c="$changed_single_component"
    if [ "$c" = "" ]; then
	echo "bcc:tcwg-validation@linaro.org"
	return 0
    fi

    local -A emails
    emails["tcwg-validation@linaro.org"]=bcc
    emails["author"]=cc
    emails["committer"]=to

    case "$ci_project/$ci_config:$c" in
	*_fast_*/*:*)
	    # FIXME: testing maintainer-mode
	    emails["author"]=no
	    emails["committer"]=no
	    emails["christophe.lyon@linaro.org"]=to
	    ;;
	tcwg_aosp-*/*:*)
	    # FIXME: stabilize and enable notifications.
	    emails["author"]=no
	    emails["committer"]=no
	    emails["antoine.moynault@linaro.org"]=to
	    ;;
	tcwg_bmk-*/*:*)
	    # FIXME: stabilize and enable notifications.
	    emails["author"]=no
	    emails["committer"]=no
	    emails["maxim.kuvyrkov@linaro.org"]=to
	    ;;
	tcwg_kernel/llvm-*:linux|tcwg_kernel/*:llvm)
	    emails["author"]=no
	    emails["committer"]=no
	    emails["llvm@lists.linux.dev"]=to_postcommit
	    ;;
	tcwg_kernel/*:linux)
	    emails["author"]=no
	    emails["committer"]=no
	    emails["linaro-kernel@lists.linaro.org"]=to_postcommit
	    ;;
	tcwg_kernel/*:*)
	    emails["author"]=no
	    emails["committer"]=no
	    emails["linaro-toolchain@lists.linaro.org"]=to_postcommit
	    ;;
	*/*:gcc)
	    emails["gcc-regression@gcc.gnu.org"]=cc_postcommit
	    ;;
	*/*:gdb)
	    emails["gdb-testers@sourceware.org"]=cc_postcommit
	    ;;
	*/*:*)
	    emails["linaro-toolchain@lists.linaro.org"]=cc_postcommit
	    ;;
    esac

    local c email base_rev cur_rev

    # CC: author
    base_rev=$(get_baseline_git "${c}_rev")
    cur_rev=$(get_current_git "${c}_rev")
    while read -r email; do
	emails["$email"]="${emails[author]}"
    done < <(git -C "$c" log --pretty='%ae' "$base_rev..$cur_rev" || true)


    local precommit_postcommit=postcommit
    if [ "$notify" != precommit ]; then
	# TO: committer
	base_rev=$(get_baseline_git "${c}_rev")
	cur_rev=$(get_current_git "${c}_rev")
	while read -r email; do
	    # shellcheck disable=SC2034
	    emails["$email"]="${emails[committer]}"
	done < <(git -C "$c" log --pretty='%ce' "$base_rev..$cur_rev" || true)
    else
	precommit_postcommit=precommit

	if [ "${notify_email-}" = "" ]; then
	    # TO: precommit submitter
	    # Note that for precommit testing "git log" will specify
	    # tcwg-buildslave@linaro.org as committer, so use patchwork submitter
	    # instead.
	    notify_email="${pw[${c}_patch_submitter]-}"
	fi

	if [ "${notify_email-}" != "" ]; then
	    emails["$notify_email"]="${emails[committer]}"
	fi
    fi

    unset "emails[author]" "emails[committer]"

    local type
    local -a recipients=()
    for email in "${!emails[@]}"; do
	type="${emails[$email]}"
	case "$precommit_postcommit:$type" in
	    precommit:to_precommit)   type=to ;;
	    precommit:to_postcommit)  type=no ;;
	    precommit:cc_precommit)   type=cc ;;
	    precommit:cc_postcommit)  type=no ;;
	    postcommit:to_precommit)  type=no ;;
	    postcommit:to_postcommit) type=to ;;
	    postcommit:cc_precommit)  type=no ;;
	    postcommit:cc_postcommit) type=cc ;;
	esac

	case "$type" in
	    no) ;;
	    to)	recipients+=("$email") ;;
	    *) recipients+=("$type:$email") ;;
	esac
    done

    (IFS=","; echo "${recipients[*]}")
    )
}

##### Model for GNU SPECIFIC mail-body-regression.txt regression details
#
# After commit <>
#
# The following .. slowed down/grew up ..
#
# The configuration is ...
#

##### Generates mail/mail-body.txt file
print_mail_body()
{
    local bad_artifacts_url good_artifacts_url
    bad_artifacts_url="$(get_current_manifest BUILD_URL)artifact/artifacts"
    good_artifacts_url="$(get_baseline_manifest BUILD_URL)artifact/artifacts"

    local key=""
    # We create a Jira card only for single-commit regressions in
    # post-commit CI
    if [ "$change_kind" = "single_commit" ] \
	   && [ "${pw[project]-}" = "" ]; then
	key=$(print_jira_card_key)
	if [ -z "$key" ]; then
	    key=$(print_jira_template_card)
	fi
    fi

    cat << EOF
Dear contributor, our automatic CI has detected problems related to your \
patch(es).  Please find some details below.  If you have any questions, \
please follow up on linaro-toolchain@lists.linaro.org mailing list, Libera's \
#linaro-tcwg channel, or ping your favourite Linaro toolchain developer \
on the usual project channel.

We appreciate that it might be difficult to find the necessary logs or \
reproduce the issue locally. If you can't get what you need from our \
CI within minutes, let us know and we will be happy to help.

EOF

    if [ "$key" != "" ]; then
	cat <<EOF
We track this report status in https://linaro.atlassian.net/browse/$key , \
please let us know if you are looking at the problem and/or when you have a fix.

EOF
    fi

    cat <<EOF
In $($print_config_f --short) after:

$($print_commits_f --short | sed -e 's/^/  | /')

$($print_result_f --short)

The configuration of this build is:
$($print_config_f --long)

-----------------8<--------------------------8<--------------------------8<--------------------------
The information below can be used to reproduce a debug environment:

Current build   : $bad_artifacts_url
Reference build : $good_artifacts_url

EOF

    # FIXME: Remove this warning when we enable maintainer-mode in
    # production.
    if [ "${pw[project]-}" != "" ]; then
	cat <<EOF
Warning: we do not enable maintainer-mode nor automatically update
generated files, which may lead to failures if the patch modifies the
master files.

EOF
    fi

    if [ "$change_kind" != "single_commit" ] \
	   || [ "${pw[project]-}" != "" ]; then
	return
    fi

    cat <<EOF
Reproduce last good and first bad builds: $($print_last_icommit_f --reproduction_instructions_link "$ci_project" "$ci_config")

Full commit : $($print_commits_f --link)

List of configurations that regressed due to this commit :
$($print_last_icommit_f --status)

EOF
}

# Generate notify/mail-*.txt files
generate_mail_files()
{
    (
    set -euf -o pipefail

    print_mail_recipients > $top_artifacts/notify/mail-recipients.txt

    # Keep jira/summary in sync with mail-subject.txt
    echo "[Linaro-TCWG-CI]" \
	 "$($print_commits_f --oneline): $($print_result_f --oneline) on $($print_config_f --oneline)" \
	 > $top_artifacts/notify/mail-subject.txt

    print_mail_body > $top_artifacts/notify/mail-body.txt
    )
}

print_readme_header()
{
    (
    set -euf -o pipefail

    local text_type="$1"

    local msg="How to browse artifacts of this build"
    case $text_type in
	html)
	    cat <<EOF
<!DOCTYPE html>
<html>
<body>
<font color="black">
<h2>$msg</h2>
EOF
	    ;;
	txt)
	    cat <<EOF
$msg

EOF
	    ;;
    esac
    )
}

# print MSG as a link, if appropriate
# $1: type of output (html/txt)
# $2: message (directory or file name)
# #3: directory where $2 resides, used to detertime the file type
print_readme_link()
{
    (
    set -euf -o pipefail

    local text_type="$1"
    local msg="$2"
    local home="$3"

    case $text_type in
	html)
	    # Jenkins webserver need a /*view*/ decoration so that
	    # text files are displayed in the browser.
	    view=""
	    if [ -f "$home/$msg" ]; then
		if file "$home/$msg" | grep -qw text ; then
		    view="/*view*/"
		fi
	    fi
	    echo -n "<a href=\"$msg$view\">$msg</a>"
	    ;;
	txt)
	    echo -n "$msg"
	    ;;
    esac
    )
}
print_readme_footer()
{
    (
    set -euf -o pipefail

    local text_type="$1"
    case $text_type in
	html)
	    cat << EOF
</body>
</html>
EOF
	    ;;
    esac
    )
}

# Provide some hints to users, to help them find their way in our
# artifacts.
# $1: type of text (txt, html)
generate_readme()
{
    (
    set -euf -o pipefail

    local text_type="$1"

    local gnu_text=false
    case "$ci_project" in
	*check*)
	    case "$ci_project" in
		*gnu*|*gcc*|*binutils*|*gdb*|*bootstrap*)
		    gnu_text=true
		    ;;
	    esac
	    ;;
    esac

    local list_start=""
    local list_end=""
    local list_item="- "
    local new_parag=""

    if [ "$text_type" = "html" ]; then
	list_start="<ul>"
	list_end="</ul>"
	list_item="<li>"
	new_parag="<p>"
    fi

    print_readme_header $text_type

    cat << EOF
The artifact directories contain a lot of information related to the
results of this build.
$new_parag
Directories starting with a number contain the logs of each step of
the build.  More synthetic information is available in other directories,
as described below:
$new_parag
$list_start
EOF

    if [ -d $top_artifacts/00-sumfiles ]; then
	cat <<EOF
$list_item$(print_readme_link "$text_type" "00-sumfiles/" "") contains .log and possibly .sum files generated by the
  build. Files with .0 suffix contain the results of the initial full
  testsuite run, files with .1, .2 etc... contain logs restricted to
  the parts (.exp) of the testsuite where we detected regressions.
  .1, .2, .... represent the number of times this subset of the testsuite
  was executed in order to also identify flaky tests.  The last one
  contains what is considered as the results of this build.

EOF
    fi

	cat <<EOF
$list_item$(print_readme_link "$text_type" "git/" "") contains the revision and repository of each toolchain
  component built

$list_item$(print_readme_link "$text_type" "jenkins/" "") contains information useful for the CI maintainers

$list_item$(print_readme_link "$text_type" "notify/" "") contains the material used to build various
  notifications/reports (emails, Jira, LNT, ...)
EOF

    if $gnu_text; then
	cat <<EOF

$list_item$(print_readme_link "$text_type" "sumfiles/" "") contains the .sum files produced by this build.
EOF
    fi

    cat <<EOF
$list_end
$new_parag
If you received a notification about one of your patches causing
problems, the information you received is in $(print_readme_link "$text_type"  "notify/" "") and has
links to other artifacts from this directory.
EOF

    if $gnu_text; then
	local regressions=""
	if [ -f $top_artifacts/notify/regressions.sum ]; then
	    regressions="$(print_readme_link "$text_type" "notify/regressions.sum" "$top_artifacts") and "
	fi
	cat <<EOF
$new_parag
If you are investigating such a problem, you are probably primarily
interested in:
$new_parag
$list_start
$list_item$regressions$(print_readme_link "$text_type" "notify/results.compare.txt" "$top_artifacts") (regression report).

EOF
	if [ -d $top_artifacts/00-sumfiles ]; then
	    cat <<EOF
$list_item$(print_readme_link "$text_type" "00-sumfiles/" "") .log files with detailed errors, to save
  yourself reproducing the problem on your machine.
EOF
	fi

	cat <<EOF
$list_end
EOF
    fi

    # Print the list of files below top_artifacts, some users find
    # this useful.
    cat <<EOF
$new_parag
List of files below:
$new_parag
$list_start
EOF

    while read -r cur_file; do
	echo "$list_item$(print_readme_link "$text_type" "$cur_file" "$top_artifacts")"
    done < <(cd $top_artifacts ; find . -type f | sort)

    cat <<EOF
$list_end
EOF

    print_readme_footer $text_type
    )
}

# Procedure to generate a nice html to publish in jenkins job
generate_jenkins_html_files()
{

    (
    set -euf -o pipefail

    echo "# ${FUNCNAME[0]}"
    if ! $generate_jenkins_html; then
       echo "... Skipping"
       return
    fi

    case "$ci_project" in
	tcwg_bmk-*)
	    (
	    status_file="$top_artifacts/results-vs-prev/csv-results-1/status.csv"
	    if [ -f $status_file ]; then
		# status is one of : success, failed-to-build or failed-to-run
		nb_succeed=$(sort -u $status_file | grep -c ",success$")
		nb_failed=$(sort -u $status_file | grep -c ",failed-to-")
		title="$nb_succeed benchmarks succeeded"
		if [ "$nb_failed" != "0" ]; then
		    title+=", <FONT COLOR=\"orange\">$nb_failed failed<FONT COLOR=\"black\">"
		fi
		cat << EOF
		<!DOCTYPE html>
		<html>
		<body>

		<h2>Status of this run : $title</h2>

		<FONT COLOR="orange">
EOF

		sort -u $status_file | grep ",failed-to-" | cut -d, -f1,3 | \
		    sed -e 's|\(.*\),\(.*\)|<h3> - \1 : \2</h3>|'

		cat << EOF
		<FONT COLOR="black">

		</body>
		</html>
EOF
	    fi
	    ) > $top_artifacts/jenkins/status.html
	    ;;
	*)
	    # no implementation
	    echo "... Skipping"
	    return
	    ;;
    esac
    ) &
    wait $! || true

    generate_readme html > $top_artifacts/README.html
    generate_readme txt > $top_artifacts/README.txt
}


#========================================================================================
#
#                               DASHBOARD RELATED PROCEDURES
#
#========================================================================================
# Calculate a reasonable date to associate with the current results / artifacts
# and prints this date in manifest.sh.
calculate_results_date ()
{
    (
    set -euf -o pipefail

    local c base_d cur_d results_date=0

    # Firstly, set results_date to the max of commit dates of all components.
    for c in $(get_current_manifest "{rr[components]}"); do
	base_d=$(get_baseline_component_date ${c} || true)
	cur_d=$(get_current_component_date ${c} || true)
	if [ x"$base_d" != x"" ]; then
	    if [ x"$cur_d" = x"" ] || [ $cur_d -lt $base_d ]; then
		cur_d="$base_d"
	    fi
	fi
	if [ x"$cur_d" = x"" ]; then
	    continue
	fi

	if [ $cur_d -gt $results_date ]; then
	    results_date="$cur_d"
	fi
    done

    assert_with_msg "Failed to produce results_date" [ $results_date -gt 0 ]

    base_d=$(get_baseline_manifest "{rr[results_date]}")

    # Normally there's a rr[results_date] in baseline manifest.
    # The rr[results_date] was useless at one point (no dashboard for a period
    # of time) and disapeared from the manifest for that time. It is now useful
    # to have it back because we are setting another dashboard backend.
    # This assertion on the existing results blocks any new results.
    # Disabling it.
    #
    # TODO:
    # Once all base-artifacts history are rewritten with the results_date,
    # we will re-enable this assertion.
    #assert_with_msg "Missing rr[results_date] from baseline manifest" \
    #	   [ "$base_d" != "" ]

    if [ "$base_d" != "" ]; then
	if [ $results_date -gt $base_d ]; then
	    # Average between our current results_date and baseline date.
	    # The reason behind average is to spread out dates between bursts
	    # of builds, which can occur when reducing a regression.
	    results_date=$((($results_date + $base_d) / 2))
	elif [ $results_date -eq $base_d ]; then
	    # If the dates are equal, then no point in taking average.
	    # Instead just add some arbitrary, but reasonable, amount.
	    results_date=$(($results_date + 600))
	else
	    # If the baseline date is in the future (e.g., because git commit
	    # dates are weird in one of the components), then add some
	    # arbitrary, but reasonable, amount.
	    results_date=$(($base_d + 600))
	fi
    fi

    # Save results_date in the manifest so that we can fetch it as $base_d
    # above for the next build.
    rr[results_date]="$results_date"
    cat <<EOF | manifest_out
rr[results_date]="$results_date"
EOF
    )
}

generate_dashboard_squad ()
{
    local results_date

    echo "# ${FUNCNAME[0]}"
    if ! $generate_dashboard; then
       echo "... Skipping"
       return
    fi

    results_date="${rr[results_date]}"
    results_date=$(date --utc --iso-8601=seconds --date="@$results_date")

    $scripts/dashboard-generate-squad.sh \
	    --top_artifacts "$top_artifacts" \
	    --baseline_branch "$(get_current_manifest "{rr[baseline_branch]}")" \
	    --components "$(get_current_manifest "{rr[components]}")" \
	    --run_date "$results_date" \
	    --relative_results true \
	    --squad_mode "vs-first"
    echo "... Done"
}

post_dashboard_squad ()
{
    echo "# ${FUNCNAME[0]}"
    if ! $post_dashboard; then
       echo "... Skipping"
       return
    fi

    if ! [ -d $top_artifacts/notify/dashboard/squad-vs-first ]; then
       return
    fi

    $dryrun $top_artifacts/notify/dashboard/squad-vs-first/dashboard-push-squad.sh
    echo "... Done"
}

generate_lnt_report()
{
    (
    set -euf -o pipefail
    local results_date

    echo "# ${FUNCNAME[0]}"
    if ! $generate_lnt; then
       echo "... Skipping"
       return
    fi

    # shellcheck source=lnt-utils
    . $scripts/lnt-utils.sh

    results_date="$(get_current_manifest "{rr[results_date]}")"
    results_date=$(date +"%Y-%m-%d %H:%M:%S" --date "@$results_date")

    local jira_key="-"
    if [ -f "$top_artifacts/notify/jira/key" ]; then
	jira_key=$(cat "$top_artifacts/notify/jira/key")
    fi

    case "$ci_project" in
	tcwg_binutils*|tcwg_bootstrap*|tcwg_gcc*|tcwg_gdb*|tcwg_glibc*|tcwg_gnu*)
	    generate_lnt_gnu_check_report \
		"$(get_current_manifest BUILD_URL)" "$ci_project" "$ci_config" \
		"$results_date" "$jira_key" \
		$top_artifacts/sumfiles \
		$top_artifacts/notify/lnt_report.json
	    ;;
	tcwg_bmk-*)
	    local cc cur_rev describe
	    case "${rr[toolchain]}" in
		llvm) cc=llvm ;;
		gnu) cc=gcc ;;
		*) false ;;
	    esac
	    cur_rev=$(get_current_git ${cc}_rev)
	    describe=$(describe_sha1 "${cc}" "$cur_rev" false)
	    generate_lnt_bmk_report \
		"$(get_current_manifest BUILD_URL)" "$ci_project" "$ci_config" \
		"$results_date" "$jira_key" \
		$top_artifacts/results-vs-prev/csv-results-1/size.csv \
		$top_artifacts/results-vs-prev/csv-results-1/perf.csv \
		$top_artifacts/results-vs-prev/csv-results-1/status.csv \
		$top_artifacts/results-vs-prev/bmk-specific-variability-max.csv \
		$top_artifacts/notify/lnt_report.json
	    ;;
	*)
	    # no lnt support
	    echo "... Skipping"
	    return
	    ;;
    esac



    ) &
    wait $! || true
}

#========================================================================================
#
#                                    MAIN FLOW
#
#========================================================================================

# setup the environment to run notify stage
setup_notify_environment
check_source_changes
setup_stages_to_run

# Initialize icommits
post_interesting_commits init

if [ "$stage" != "full" ]; then
    echo "Init stage ran successfully."
    exit 0
fi

$generate_extra_details_f
calculate_results_date

check_if_first_report

# Update entry with full information
post_interesting_commits full

if $generate_jira; then
    generate_jira_dir
fi

echo "# print all notification files"
if $generate_mail; then
    generate_mail_files
fi

if $generate_jenkins_html; then
    generate_jenkins_html_files
fi

# generate and post Dashboard
echo "# generate dashboard"
generate_dashboard_squad
post_dashboard_squad

generate_lnt_report

if $post_mail; then
    release_notification_files
fi

if $post_gcc_testresults; then
    release_gcc_testresults_files
fi

# Update jira card description
post_to_jira

echo "Full stage ran successfully."