#!/bin/bash

set -ef -o pipefail

scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh

# Relative artifacts are used for generation of manifests and reproduction
# instructions.
rel_artifacts=artifacts
artifacts=$(pwd)/$rel_artifacts

fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"

declare -A rr

# Process bisect-only args
convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"

obligatory_variables bad_git build_script current_project
declare bad_git build_script current_project

BUILD_URL="${BUILD_URL:-$(pwd)}"
replay_log="${replay_log-}"
reproduce_bisect="${reproduce_bisect:-false}"

# Process build args and record them in build-parameters.sh
convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@"
$reproduce_bisect || manifest_pop

obligatory_variables rr[ci_project] rr[ci_config]

verbose="${verbose-true}"

set -u

if $verbose; then set -x; fi

mkdir -p $artifacts/jenkins
touch $artifacts/jenkins/build-name

trap print_traceback EXIT

bad_url="${bad_git%#*}"
bad_branch="${bad_git#*#}"

rebase_workaround=false
rebase_workaround_opts=()

case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
    tcwg_kernel/*-next-*:linux)
	# Workaround linux-next/master rebasing on top of linux-next/stable.
	# Search for regressions against linux-mainline:master (aka linux-next:stable).
	clone_or_update_repo $current_project stable $bad_url
	# Just in case linux-next:stable has advanced between the build and bisect jobs,
	# use merge base between linux-next:stable and $bad_git.
	bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}"
	linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}"
	cat <<EOF | manifest_out
declare -g linux_next_stable=$linux_next_stable
EOF
	echo "Rebase workaround: forcing linux baseline to $linux_next_stable"
	rebase_workaround=true
	rebase_workaround_opts+=("==rr[linux_git]"
				 "$bad_url#$linux_next_stable")
	;;
esac

# Build baseline that we are going to re-use to speed-up bisection.
# (This also confirms that infrastructure is OK.)
echo "Testing baseline (should be success)"
$build_script \
    ^^ $reproduce_bisect \
    %% $rel_artifacts/manifests/build-baseline.sh \
    @@ $rel_artifacts/manifests/build-parameters.sh \
    ==rr[mode] baseline \
    ==rr[update_baseline] push \
    ==rr[top_artifacts] "$rel_artifacts/build-baseline" \
    --verbose "$verbose" \
    "${rebase_workaround_opts[@]}"

baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}"
cat <<EOF | manifest_out
declare -g baseline_rev=$baseline_rev
EOF

ln -f -s "build-baseline" "$artifacts/build-$baseline_rev"
ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh"

case "${rr[ci_project]}" in
    tcwg_gcc_check*|tcwg_gnu_*_check_*)
	(
	# Build up lists of flaky tests.  We do this by comparing
	# just-created baseline vs sumfiles in base-artifacts.
	fails=$(find $rel_artifacts/build-baseline \
		     -path "*-check_regression/fails.sum")
	if ! [ -f "$fails" ]; then
	    exit
	fi

	xfail_short="contrib/testsuite-management/flaky/${rr[ci_project]}-${rr[ci_config]}.xfail"
	xfail="gcc-compare-results/$xfail_short"

	if ! [ -f "$xfail" ]; then
	    echo "@include ../flaky.xfail" > "$xfail"
	fi

	(
	    echo
	    echo "# From $BUILD_URL:"
	    cat "$fails" | sed -e "s/^\([A-Z]\+: \)/flaky \| \1/"
	) >> "$xfail"

	git -C gcc-compare-results add "$xfail_short"
	git -C gcc-compare-results commit -m "From $BUILD_URL"
	git -C gcc-compare-results review -s
	git -C gcc-compare-results pull --rebase gerrit refs/heads/master
	git -C gcc-compare-results push gerrit HEAD:refs/heads/master
	) &
	res=0 && wait $! || res=$?
	# Ignore any failures in the above.
	;;
esac

mkdir -p ./bisect

# Save baseline build state, which we restore before individual bisect tests.
# This ensures that "bad" bisect tests won't affect "good" ones and vice versa.
baseline_exclude=(
    --exclude /bisect/ --exclude "/$rel_artifacts/" --exclude "/$current_project/"
)
rsync -a --del --delete-excluded "${baseline_exclude[@]}" ./ ./bisect/baseline/

cd $current_project

mkdir $artifacts/git-logs

# Make sure the sources are clean before bisecting
git reset --hard

if [ -f "$replay_log" ]; then
    cp "$replay_log" $artifacts/git-logs/bisect_replay.sh
    git bisect replay $artifacts/git-logs/bisect_replay.sh
else
    git bisect start
fi

# Hard-link BISECT_LOG inside $artifacts to guarantee its upload to jenkins.
ln -f "$(pwd)"/.git/BISECT_LOG $artifacts/git-logs/bisect_log

if ! git bisect log | grep -q "^git bisect .* $baseline_rev\$"; then
    git bisect good $baseline_rev
fi

# Bisect script.
#
# With this script we find the first commit that has regressed compared
# to baseline, but not, necessarily, the commit that caused regression in
# $bad_rev.  Consider the scenario:
# - rev_10 produced good result "2000" -- this is current baseline
# - rev_20 completely broke the build (say, result "10")
# - rev_22 fixed the build
# - rev_30 regressed the build to result "1000" -- this is the regression we
#   detected vs "2000" baseline.
#
# The script will identify rev_20 as the first failing commit, which will
# cause the baseline to be reset to rev_20 with metric "10".  When we then
# rebuild master (at rev_30) we will see a /progression/ from "10" to "1000",
# thus missing the regression of "2000" to "1000".
#
# To catch the "2000" to "1000" regression someone would need to manually
# trigger bisect between rev_22 and rev_30.
#
# We reduce the impact of the above by assigning negative values to result
# metric for builds that look very broken, and do not bisect regressions into
# the "negative" side.
cat > $artifacts/test.sh <<EOF
#!/bin/bash

set -euf -o pipefail

rev=\$(git rev-parse HEAD)

if git bisect log | grep -q "^git bisect bad \$rev\\\$"; then
  exit 1
elif git bisect log | grep -q "^git bisect skip \$rev\\\$"; then
  exit 125
elif git bisect log | grep -q "^git bisect good \$rev\\\$"; then
  exit 0
fi

cd ..

# Restore known-good baseline state.
rsync -a --del ${baseline_exclude[@]} ./bisect/baseline/ ./

$build_script \
  ^^ $reproduce_bisect \
  %% $rel_artifacts/manifests/build-\$rev.sh \
  @@ $rel_artifacts/manifests/build-parameters.sh \
  ==rr[mode] bisect \
  ==rr[${current_project}_git] "$bad_url#\$rev" \
  ==rr[top_artifacts] $rel_artifacts/build-\$rev \
  --verbose "$verbose" &
res=0 && wait \$! || res=\$?

git -C $current_project reset --hard

if [ x"\$res" != x"0" ]; then
  if [ -f $rel_artifacts/build-\$rev/trigger-build-$current_project ]; then
    exit 1
  else
    # The build failed due to an uninteresting problem -- a prerequisite
    # failed to build or benchmarking harness went down.  We mark such
    # revisions "skipped", but up to a point.  If we skip more revisions
    # in a row, than half the number of tests necessary to finish the bisect,
    # then we mark such "skipped" revision as "bad".

    # Number of "git bisect skip" in a row
    n_skips=\$(git -C $current_project bisect log | awk '
BEGIN { n_skips=0 }
/git bisect skip/ { n_skips++; next }
/git bisect/ { n_skips=0 }
END { print n_skips }
')
    revs_left=\$(git -C $current_project bisect view --pretty=%H | wc -l)
    # Half the number of steps to finish the bisect
    n_steps_2=\$(echo "n_steps=l(\$revs_left)/l(2); scale=0; n_steps/2" | bc -l)
    if [ \$n_steps_2 -lt 2 ]; then
      # Avoid skipping revisions at the end of the bisect.
      n_steps_2=2
    fi
    if [ \$n_skips -le \$n_steps_2 ]; then
      exit 125
    else
      # We had several skips in a row and still have many revisions to bisect.
      # Mark this one "bad" to progress the bisect.
      exit 1
    fi
  fi
else
  exit 0
fi
EOF
chmod +x $artifacts/test.sh

# Fetch $bad_branch/$bad_rev from $bad_url
prev_rev=$(git rev-parse HEAD)
clone_or_update_repo . "$bad_branch" "$bad_url"
bad_rev="${bad_rev-$(git_rev_parse_long . $bad_branch)}"
cat <<EOF | manifest_out
declare -g bad_rev=$bad_rev
EOF

if [ x"$baseline_rev" = x"$bad_rev" ]; then
    echo "WARNING: Detected regression with no change in sources of $current_project"
    sed -i -e "s/\$/-no_change/" $artifacts/jenkins/build-name
    res=0
else
    # Confirm regression in $bad_rev vs $baseline_rev.
    git checkout --detach $bad_rev
    $artifacts/test.sh &
    res=0 && wait $! || res=$?
fi

# Restore revision previously checked out.  Otherwise "git bisect run"
# below will not use replay info.
git checkout --detach $prev_rev

if [ x"$res" = x"0" ]; then
    if $rebase_workaround; then
	echo "Rebase workaround: no regression between $baseline_rev and $bad_rev"
	sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name
	cat >> $artifacts/trigger-build-rebase <<EOF
linux_git=$bad_url#$baseline_rev
update_baseline=reset
EOF
    else
	echo "WARNING: build for bad_rev $bad_rev succeeded"
	sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name
	# Regressions in speed benchmarking are not stable,
	# so retry with resetting baseline artifacts.
	case "${rr[ci_project]}/${rr[ci_config]}" in
	    tcwg_bmk*/*-O[23]*)
		cat > $artifacts/trigger-build-reset <<EOF
update_baseline=reset
EOF
		;;
	esac
    fi
    echo > $artifacts/jenkins/mail-recipients.txt
    trap "" EXIT
    exit 0
elif [ x"$res" = x"125" ]; then
    # We have confirmed a regression, but not what we have been triggered
    # to bisect.
    echo "WARNING: build for bad_rev $bad_rev showed uninteresting regression"
    sed -i -e "s/\$/-uninteresting/" $artifacts/jenkins/build-name
    echo > $artifacts/jenkins/mail-recipients.txt
    trap "" EXIT
    exit 0
fi

if ! git bisect log | grep -q "^git bisect .* $bad_rev\$"; then
    git bisect bad $bad_rev
    ln -f -s "build-$bad_rev" "$artifacts/build-bad"
    ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh"
fi

# Clone interesting-commits.git repo, which contains a list of SHA1s
# that might cut down bisection time.  Mostly, these are first_bad and
# last_good commits.
interesting_commits="../bisect/interesting-commits"

interesting_commits_url=https://git-us.linaro.org/toolchain/ci/interesting-commits.git
interesting_commits_branch=linaro-local/ci/${rr[ci_project]}
if [ x"$(git ls-remote --heads $interesting_commits_url refs/heads/$interesting_commits_branch | wc -l)" != x"1" ]; then
    interesting_commits_branch=empty
fi
interesting_commits_rev=${interesting_commits_rev-$interesting_commits_branch}
clone_or_update_repo $interesting_commits $interesting_commits_rev $interesting_commits_url auto $interesting_commits_branch
interesting_commits_rev=$(git -C $interesting_commits rev-parse HEAD)
cat <<EOF | manifest_out
declare -g interesting_commits_rev=$interesting_commits_rev
EOF

# Add SHA1 commit $1 to interesting-commits and push the repo.
# If $2 is "regression" then record current configuration as having regressed
# at this commit.
# Ignore failures (since this is cache handling).
push_interesting_commit ()
{
    declare -g push_interesting_commit_result=0
    (
    set -euf -o pipefail

    local sha1="$1"
    local kind="$2"
    local -a configs

    if ! grep -q "^$sha1" $interesting_commits/$current_project; then
	echo "$sha1" >> $interesting_commits/$current_project
    fi

    if [ x"$kind" = x"regression" ]; then
	mapfile -t configs < <(grep "^$sha1" $interesting_commits/$current_project | sed -e "s/^$sha1 *//")
	configs+=("${rr[ci_project]}"/"${rr[ci_config]}")
	mapfile -t configs < <(echo "${configs[@]}" | tr ' ' '\n' | sort -u)
	sed -i -e "s#^$sha1.*\$#$sha1 ${configs[*]}#" $interesting_commits/$current_project
    fi

    git -C $interesting_commits add .
    if [ x"$(git -C $interesting_commits status --short)" = x"" ]; then
	# No file has changed.  We've been here before...
	# E.g., this is a re-occuring regression in linux-next.
	exit 125
    fi
    git -C $interesting_commits commit -m "Add $kind $sha1 from $BUILD_URL

${configs[*]}" &
    local res=0 && wait $! || res=$?

    if [ x"$res" = x"0" ]; then
	# Interesting-commits.git do not have .gitreview, so it's
	# simpler to push via gitolite.
	git_init_linaro_local_remote $interesting_commits baseline false
	git_push $interesting_commits baseline linaro-local/ci/${rr[ci_project]}
    fi
    ) &
    wait $! || push_interesting_commit_result=$?
}

# Print first_bad revision (if detected)
get_first_bad ()
{
    (
    # Allow pipefail to handle error exit codes from git bisect log and grep.
    # Note that child shell inherits settings from parent shell, so we need
    # excplicitly set "+o pipefail".
    set -euf +o pipefail

    git bisect log | tail -n1 | grep "^# first bad commit:" \
	| sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/"
    )
}

# Print revs tested during bisect.
# $1 -- Revision kind -- good/bad/skip.
#       Print all revisions if no $1 given.
print_tested_revs ()
{
    (
    # Allow pipefail to handle error exit codes from git bisect log and grep.
    # Note that child shell inherits settings from parent shell, so we need
    # excplicitly set "+o pipefail".
    set -euf +o pipefail

    local kind="${1-[a-z]*}"
    git bisect log | grep "^git bisect $kind " | sed -e "s/^git bisect $kind //"
    )
}

# Try to reduce bisection range by testing regressions (and their parents)
# identified in other configurations.
touch $interesting_commits/$current_project
# Generate list of commits inside the bisection range.
commits_to_test=$artifacts/git-logs/commits_to_test
git bisect view --pretty=%H > $commits_to_test

# Record commits in the bisect range.  These are commits_to_test plus
# commits that have been tested.
commits_in_range=$artifacts/git-logs/commits_in_range
cp $commits_to_test $commits_in_range
print_tested_revs >> $commits_in_range

# This loop can generate lots of console noise.
set +x
while [ x"$(get_first_bad </dev/null)" = x"" ] && read -a arr; do
    (
    set -euf -o pipefail

    sha1="${arr[0]}"

    # Ignore commits outside of bisection range.
    if ! grep -q "^$sha1\$" $commits_to_test; then
	exit 0
    fi

    # Skip revisions that were already tested.  Good revisions are filtered
    # out in the above $commits_to_test check, and here we filter out
    # "bad" and "skip" revisions.
    if git bisect log | grep -q "^git bisect .* $sha1\$"; then
	exit 0
    fi

    if $verbose; then set -x; fi

    git checkout --detach $sha1
    $artifacts/test.sh &
    res=0 && wait $! || res=$?
    if [ x"$res" = x"0" ]; then
	git bisect good
    elif [ x"$res" = x"125" ]; then
	git bisect skip
    else
	git bisect bad
    fi
    git bisect view --pretty=%H > $commits_to_test
    ) </dev/null
done < <(cat $interesting_commits/$current_project)
if $verbose; then set -x; fi

if [ x"$(get_first_bad)" = x"" ]; then
    git bisect run $artifacts/test.sh &
    res=0 && wait $! || res=$?

    if [ x"$res" = x"0" ]; then
	assert_with_msg "Didn't find first bad commit!" [ x"$(get_first_bad)" != x"" ]
    fi
fi

notify_devs=true
notify_author=""

first_bad=$(get_first_bad)

if [ x"$first_bad" != x"" ] \
       && ! [ -f $artifacts/build-$first_bad/trigger-build-$current_project ]; then
    # First_bad is not a real or "interesting" regression.  Perhaps it was
    # a "skipped" commit marked as "bad.
    first_bad=""
fi

if [ x"$first_bad" != x"" ]; then
    # "git bisect run" succeeded.  Check whether this is an actual regression
    # or bisection artifact.
    last_good=""
    bad_last_good=""
    for sha1 in $(git rev-parse $first_bad^@); do
	# It seems that git-bisect assumes parent commit as "good" on
	# the basis of one of its children being "good".  Therefore we
	# can have a situation when we have parent P with children C1 and C2,
	# and child C1 has a child of its own CB.  Git-bisect tests C2 as
	# "good", and CB as "bad".  From C2 being good it assumes P as "good",
	# and it knows CB is "bad", so git-bisect returns C1 as the first bad
	# commit.
	# To simplify investigations we explicitly test parent of $first_bad.

	# Ignore commits outside of bisection range, but make an exception
	# for commits that weren't tested due to their child tested good.
	# This happens in projects that actively use merges, e.g., linux.
	if ! grep -q "^$sha1\$" $commits_in_range; then
	    child_tested_good=false
	    for tested_good in $(print_tested_revs good); do
		if git merge-base --is-ancestor $sha1 $tested_good; then
		    child_tested_good=true
		    break
		fi
	    done
	    if ! $child_tested_good; then
		continue
	    fi
	fi

	echo "Testing first_bad's parent $sha1 (hoping for success)"
	git checkout --detach "$sha1"
	$artifacts/test.sh &
	res=0 && wait $! || res=$?
	if [ x"$res" = x"0" ]; then
	    last_good=$sha1
	    break
	fi
	bad_last_good=$sha1
    done

    if [ x"$last_good" != x"" ]; then
	# We have successfully identified a bad commit with a good parent!
	# Add both $last_good and $first_bad to interesting commits.
	push_interesting_commit $last_good "last-good"
	push_interesting_commit $first_bad "regression"
	if [ x"$push_interesting_commit_result" = x"125" ]; then
	    notify_devs=false
	fi
    else
	assert_with_msg "Broken bisection range" [ x"$bad_last_good" != x"" ]
	# All parents of $first_bad tested bad.  This is, essencially,
	# same situation as "git bisect" failing, which we handle below.
	first_bad=""
    fi
fi

reset_rev=""
if [ x"$first_bad" != x"" ]; then
    reset_rev="$first_bad"
else
    # "Git bisect" didn't find the first_bad commit or this commit points to
    # an uninteresting regression.
    # Possible reasons include:
    # - We have marked the regressing commit as "skip", which can happen
    #   to tcwg_bmk* projects when benchmarking infra has a problem.
    #
    # - Merge-base of $baseline_rev and $bad_rev is worse than $baseline_rev.
    #   We want to reset baseline to HEAD in this case, so that we catch most
    #   of the commits that introduced change in the result metric.
    #
    # - We have marked a skipped commit as bad to advance bisect, even though
    #   the failure was uninteresting.
    #
    # So, to make at least some progress on narrowing down the regression ...
    # - look for the last commit that tested "good".  If it's not $baseline_rev,
    #   then we have narrowed down the bisection range somewhat.  Therefore,
    #   trigger two builds:
    #   - one to ADVANCE baseline to the last GOOD commit, and
    #   - another to expose the regression again.
    #
    # - If $baseline_rev is the only good commit, then we have got ourselves
    #   into a tricky situation: we can't find the regression and can't make
    #   a good build, which would advance the baseline.  To resolve this,
    #   trigger two builds:
    #   - one to RESET baseline to the last BAD commit, and
    #   - another to advance the baseline and expose another regression.
    #   Note that this approach correctly handles case when merge-base of
    #   $baseline_rev and $bad_rev tests "bad".

    last_good=$(print_tested_revs good | tail -n1)
    assert_with_msg "We should have at least baseline_rev as a good commit" \
		    [ x"$last_good" != x"" ]

    if [ x"$last_good" != x"$baseline_rev" ]; then
	cat > $artifacts/trigger-build-1-advance <<EOF
${current_project}_git=$bad_url#$last_good
EOF
	sed -i -e "s/\$/-advance-baseline/" $artifacts/jenkins/build-name
    else
	# Reset baseline to the earliest bad revision with a true regression.
	# For this we look through tested-bad revisions in reverse order.
	for reset_rev in $(print_tested_revs bad | tac); do
	    if [ -f $artifacts/build-$reset_rev/trigger-build-$current_project ]; then
		break
	    fi
	done
	sed -i -e "s/\$/-reset-baseline/" $artifacts/jenkins/build-name
    fi
    # Don't send any emails.
    notify_devs=false
fi

# Bisect if officially over.
cd ..

# Create trigger-build-* files for subsequent builds.
if ! [ -f $artifacts/build-$bad_rev/trigger-build-$current_project ]; then
    # This can happen *only* when replaying bisects.
    # Otherwise $bad_rev is always tested second to $baseline_rev.
    bad_build=$(find $artifacts -path "$artifacts/build-*/trigger-build-$current_project" | tail -n1)
    assert_with_msg "No bad build during this bisect" [ x"$bad_build" != x"" ]
    mkdir -p $artifacts/build-$bad_rev
    cp $bad_build $artifacts/build-$bad_rev/trigger-build-$current_project
    sed -i -e "s/^\(${current_project}_git=.*\)#.*\$/\1#$bad_rev/" \
	$artifacts/build-$bad_rev/trigger-build-$current_project
fi

if [ x"$reset_rev" != x"" ]; then \
    if ! [ -f $artifacts/build-$reset_rev/trigger-build-$current_project ]; then
	# This is rare, but can happen that $reset_rev hasn't been tested.
	# Workaround by constructing trigger-build file from that of $bad_rev's.
	mkdir -p $artifacts/build-$reset_rev
	cp $artifacts/build-$bad_rev/trigger-build-$current_project \
	   $artifacts/build-$reset_rev/trigger-build-$current_project
	sed -i -e "s/^\(${current_project}_git=.*\)#$bad_rev\$/\1#$reset_rev/" \
	    $artifacts/build-$reset_rev/trigger-build-$current_project
    fi

    # Reset baseline to the regressed commit so that we will catch subsequent
    # regressions (worse than $bad_rev).
    cp $artifacts/build-$reset_rev/trigger-build-$current_project \
       $artifacts/trigger-build-1-reset
    echo "update_baseline=reset" >> $artifacts/trigger-build-1-reset
fi

# Trigger master build now instead of waiting for next timed SCM trigger.
cp $artifacts/build-$bad_rev/trigger-build-$current_project \
   $artifacts/trigger-build-2-default

# Save BISECT_* logs
find "$current_project" -path "$current_project/.git/BISECT_*" -print0 | xargs -0 -I@ mv @ $artifacts/git-logs/

if [ x"$first_bad" != x"" ]; then
    sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name

    ln -f -s "build-$first_bad" "$artifacts/build-first_bad"
    ln -f -s "build-$first_bad.sh" "$artifacts/manifests/build-first_bad.sh"

    ln -f -s "build-$last_good" "$artifacts/build-last_good"
    ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh"
else
    notify_devs=false
fi

case "${rr[ci_project]}" in
    tcwg_gcc_check*|tcwg_gnu_*_check_*)
	# We are building up lists of flaky tests and we need to stabilize
	# GCC and GDB testsuites.  Currently dejagnu testsuites are too
	# flaky for automated regression detection.
	notify_devs=false
	;;
esac

if $notify_devs; then
    occurences="$(cat $current_project/$interesting_commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")"
    if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then
	notify_author=$(git -C $current_project log --pretty="%an <%ae>" -n 1 \
			    "$first_bad")
    fi

    case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
	tcwg_kernel/gnu-*:linux) jira_card="GNU-681" ;;
	tcwg_kernel/gnu-*:*) jira_card="GNU-680" ;;
	tcwg_kernel/llvm-*:linux) jira_card="LLVM-647" ;;
	tcwg_kernel/llvm-*:*) jira_card="LLVM-646" ;;
	tcwg_bmk_*/gnu*-O[23]*) jira_card="GNU-689" ;;
	tcwg_bmk_*/gnu*) jira_card="GNU-686" ;;
	tcwg_bmk_*/llvm-*O[23]*) jira_card="LLVM-651" ;;
	tcwg_bmk_*/llvm-*) jira_card="LLVM-650" ;;
	tcwg_binutils/*) jira_card="GNU-692" ;;
	tcwg_cross/*) jira_card="GNU-692" ;;
	tcwg_gnu/*) jira_card="GNU-692" ;;
	# Catch-all case for when project/config IDs change, so that we
	# won't miss notifications.  Forward all that to GNU-692.
	*) jira_card="GNU-692" ;;
    esac

    rm -f $artifacts/jenkins/mail-body.draft

    if [ x"${TCWG_JIRA_TOKEN+set}" = x"set" ] && [ x"$jira_card" != x"" ]; then
	cat > $artifacts/jenkins/jira-body.txt <<EOF
[$jira_card]
$(cat $artifacts/build-$first_bad/mail/jira-body.txt)

This commit has regressed these CI configurations:
$occurences

Details: ${BUILD_URL}artifact/$rel_artifacts/jenkins/mail-body.txt/*view*/
Even more details: ${BUILD_URL}artifact/$rel_artifacts/
EOF
	if ! [ -f $HOME/.jipdate.yml ]; then
	    cat > $HOME/.jipdate.yml <<EOF
server:
  url: https://linaro.atlassian.net
  token: #TCWG_JIRA_TOKEN#
text-editor: False
username: team-toolchain+tcwg-jira@linaro.org
EOF
	fi
	sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" $HOME/.jipdate.yml
	echo y | jipdate.py -f $artifacts/jenkins/jira-body.txt &
	res=0 && wait $! || res=$?
	if [ $res != 0 ]; then
	    cat >> $artifacts/jenkins/mail-body.draft <<EOF
WARNING: Could not post this report to https://linaro.atlassian.net/browse/$jira_card

EOF
	fi
    fi

    cat >> $artifacts/jenkins/mail-body.draft <<EOF
$(cat $artifacts/build-$first_bad/mail/mail-body.txt)

THIS IS THE END OF INTERESTING STUFF.  BELOW ARE LINKS TO BUILDS, REPRODUCTION INSTRUCTIONS, AND THE RAW COMMIT.

This commit has regressed these CI configurations:
$occurences

First_bad build: \$FIRST_BAD_ARTIFACTS/
Last_good build: \$LAST_GOOD_ARTIFACTS/
Baseline build: \$BASELINE_ARTIFACTS/
Even more details: ${BUILD_URL}artifact/$rel_artifacts/

Reproduce builds:
<cut>
mkdir investigate-$current_project-$first_bad
cd investigate-$current_project-$first_bad

# Fetch scripts
git clone https://git.linaro.org/toolchain/jenkins-scripts

# Fetch manifests and test.sh script
mkdir -p $rel_artifacts/manifests
curl -o $rel_artifacts/manifests/build-baseline.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-baseline.sh --fail
curl -o $rel_artifacts/manifests/build-parameters.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-parameters.sh --fail
curl -o $rel_artifacts/test.sh ${BUILD_URL}artifact/$rel_artifacts/test.sh --fail
chmod +x $rel_artifacts/test.sh

# Reproduce the baseline build (build all pre-requisites)
$build_script @@ $rel_artifacts/manifests/build-baseline.sh

# Save baseline build state (which is then restored in $rel_artifacts/test.sh)
mkdir -p ./bisect
rsync -a --del --delete-excluded ${baseline_exclude[@]} ./ ./bisect/baseline/

cd $current_project

# Reproduce first_bad build
git checkout --detach $first_bad
../$rel_artifacts/test.sh

# Reproduce last_good build
git checkout --detach $last_good
../$rel_artifacts/test.sh

cd ..
</cut>

Full commit (up to 1000 lines):
<cut>
$(git -C $current_project show --stat --patch $first_bad | head -n 1000)
</cut>
EOF

    cp $artifacts/build-$first_bad/mail/mail-subject.txt \
       $artifacts/jenkins/mail-subject.draft

    # Set mail recipients last to preserve catch-error value from .yaml file.
    # Email developers.
    CI_MAIL_RECIPIENTS=("bcc:tcwg-validation@linaro.org")
    case "$notify_author@${rr[ci_project]}/${rr[ci_config]}:$current_project" in
	""@*/*:*) ;;
	*@tcwg_gcc*/*:*)
	    CI_MAIL_RECIPIENTS+=("$notify_author")
	    CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
	    ;;
	*@tcwg_gnu*/*:*)
	    CI_MAIL_RECIPIENTS+=("$notify_author")
	    CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
	    ;;
	*@tcwg_kernel/llvm-*:linux)
	    CI_MAIL_RECIPIENTS+=("$notify_author")
	    CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
	    CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev")
	    CI_MAIL_RECIPIENTS+=("arnd@linaro.org")
	    ;;
	*@tcwg_kernel/llvm-*:llvm)
	    CI_MAIL_RECIPIENTS+=("$notify_author")
	    CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
	    CI_MAIL_RECIPIENTS+=("cc:llvm@lists.linux.dev")
	    ;;
	*@tcwg_kernel/*:*)
	    CI_MAIL_RECIPIENTS+=("$notify_author")
	    CI_MAIL_RECIPIENTS+=("cc:linaro-toolchain@lists.linaro.org")
	    ;;
	*@tcwg_bmk*/*:*)
	# Don't notify patch authors until we improve benchmarking email
	# reporting.
	#CI_MAIL_RECIPIENTS+=("$notify_author")
	CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org")
	;;
	*@*/*:*)
	    CI_MAIL_RECIPIENTS+=("linaro-toolchain@lists.linaro.org")
	    ;;
    esac

    (
	IFS=","
	cat > $artifacts/jenkins/mail-recipients.draft <<EOF
${CI_MAIL_RECIPIENTS[*]}
EOF
    )

    (
	for i in mail-recipients mail-subject mail-body; do
	    FIRST_BAD_ARTIFACTS=${BUILD_URL}artifact/$rel_artifacts/build-$first_bad
	    export FIRST_BAD_ARTIFACTS
	    LAST_GOOD_ARTIFACTS=${BUILD_URL}artifact/$rel_artifacts/build-$last_good
	    export LAST_GOOD_ARTIFACTS
	    BASELINE_ARTIFACTS=${BUILD_URL}artifact/$rel_artifacts/build-baseline
	    export BASELINE_ARTIFACTS
	    envsubst '$FIRST_BAD_ARTIFACTS $LAST_GOOD_ARTIFACTS $BASELINE_ARTIFACTS' \
		     < $artifacts/jenkins/$i.draft > $artifacts/jenkins/$i.txt
	done
    )
else
    # Remove any fail-safe email stubs
    echo > $artifacts/jenkins/mail-recipients.txt
    echo > $artifacts/jenkins/mail-subject.txt
    echo > $artifacts/jenkins/mail-body.txt
fi

trap "" EXIT