#!/bin/bash

set -ef -o pipefail

scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh

# Relative artifacts are used for generation of manifests and reproduction
# instructions.
rel_artifacts=artifacts
artifacts=$(pwd)/$rel_artifacts

fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"

# Process bisect-only args
convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"

obligatory_variables bad_url bad_branch build_script current_project
declare bad_url bad_branch build_script current_project

BUILD_URL="${BUILD_URL:-$(pwd)}"
replay_log="${replay_log-}"
reproduce_bisect="${reproduce_bisect:-false}"

# Process build args and record them in build-parameters.sh
convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@"
$reproduce_bisect || manifest_pop

obligatory_variables rr[ci_project] rr[ci_config]
declare -A rr

verbose="${verbose-true}"

set -u

if $verbose; then set -x; fi

mkdir -p $artifacts/jenkins
touch $artifacts/jenkins/build-name

# shellcheck disable=SC2064
trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/failures" EXIT

rebase_workaround=false
rebase_workaround_opts=()

case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
    tcwg_kernel/*-next-*:linux)
	# Workaround linux-next/master rebasing on top of linux-next/stable.
	# Search for regressions against linux-mainline:master (aka linux-next:stable).
	clone_or_update_repo $current_project stable $bad_url
	# Just in case linux-next:stable has advanced between the build and bisect jobs,
	# use merge base between linux-next:stable and $bad_branch.
	bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}"
	linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}"
	cat <<EOF | manifest_out
declare -g linux_next_stable=$linux_next_stable
EOF
	echo "Rebase workaround: forcing linux baseline to $linux_next_stable"
	rebase_workaround=true
	rebase_workaround_opts+=("==rr[linux_url]" "$bad_url")
	rebase_workaround_opts+=("==rr[linux_branch]" "$linux_next_stable")
	;;
esac

# Build baseline that we are going to re-use to speed-up bisection.
# (This also confirms that infrastructure is OK.)
echo "Testing baseline (should be success)"
$build_script \
    ^^ $reproduce_bisect \
    %% $rel_artifacts/manifests/build-baseline.sh \
    @@ $rel_artifacts/manifests/build-parameters.sh \
    ==rr[mode] "baseline" \
    ==rr[update_baseline] init \
    ==rr[top_artifacts] "$rel_artifacts/build-baseline" \
    --verbose "$verbose" \
    "${rebase_workaround_opts[@]}"

baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}"
cat <<EOF | manifest_out
declare -g baseline_rev=$baseline_rev
EOF

ln -f -s "build-baseline" "$artifacts/build-$baseline_rev"
ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh"

cd $current_project

mkdir $artifacts/git-logs

# Make sure the sources are clean before bisecting
git reset --hard

if [ -f "$replay_log" ]; then
    cp "$replay_log" $artifacts/git-logs/bisect_replay.sh
    git bisect replay $artifacts/git-logs/bisect_replay.sh
else
    git bisect start
fi

# Hard-link BISECT_LOG inside $artifacts to guarantee its upload to jenkins.
ln -f "$(pwd)"/.git/BISECT_LOG $artifacts/git-logs/bisect_log

if ! git bisect log | grep -q "^git bisect .* $baseline_rev\$"; then
    git bisect good $baseline_rev
fi

# Bisect script.
#
# With this script we find the first commit that has regressed compared
# to baseline, but not, necessarily, the commit that caused regression in
# $bad_rev.  Consider the scenario:
# - rev_10 produced good result "2000" -- this is current baseline
# - rev_20 completely broke the build (say, result "10")
# - rev_22 fixed the build
# - rev_30 regressed the build to result "1000" -- this is the regression we
#   detected vs "2000" baseline.
#
# The script will identify rev_20 as the first failing commit, which will
# cause the baseline to be reset to rev_20 with metric "10".  When we then
# rebuild master (at rev_30) we will see a /progression/ from "10" to "1000",
# thus missing the regression of "2000" to "1000".
#
# To catch the "2000" to "1000" regression someone would need to manually
# trigger bisect between rev_22 and rev_30.
#
# TODO: We could skip revisions (exit 125) that are worse than metric
# for $bad_rev (result metric <1000 in the above scenario), so we would
# skip revisions between rev_20 and rev_22.  This might cause other edge
# cases to be handled sub-optimally, though.
cat > $artifacts/test.sh <<EOF
#!/bin/bash

set -euf -o pipefail

rev=\$(git rev-parse HEAD)

if git bisect log | grep -q "^git bisect bad \$rev\\\$"; then
  exit 1
elif git bisect log | grep -q "^git bisect skip \$rev\\\$"; then
  exit 125
elif git bisect log | grep -q "^git bisect good \$rev\\\$"; then
  exit 0
fi

cd ..

$build_script \
  ^^ $reproduce_bisect \
  %% $rel_artifacts/manifests/build-\$rev.sh \
  @@ $rel_artifacts/manifests/build-parameters.sh \
  ==rr[mode] bisect \
  ==rr[${current_project}_url] "$bad_url" \
  ==rr[${current_project}_branch] \$rev \
  ==rr[top_artifacts] $rel_artifacts/build-\$rev \
  --verbose "$verbose" &
res=0 && wait \$! || res=\$?

git -C $current_project reset --hard

if [ x"\$res" != x"0" ]; then
  if [ -f $rel_artifacts/build-\$rev/trigger-build-$current_project ]; then
    exit 1
  else
    exit 125
  fi
else
  exit 0
fi
EOF
chmod +x $artifacts/test.sh

# Fetch $bad_branch/$bad_rev from $bad_url
prev_rev=$(git rev-parse HEAD)
clone_or_update_repo . "$bad_branch" "$bad_url"
bad_rev="${bad_rev-$(git_rev_parse_long . $bad_branch)}"
cat <<EOF | manifest_out
declare -g bad_rev=$bad_rev
EOF

# Confirm regression in $bad_rev vs $baseline_rev.
git checkout --detach $bad_rev
$artifacts/test.sh &
res=0 && wait $! || res=$?
# Restore revision previously checked out.  Otherwise "git bisect run"
# below will not use replay info.
git checkout --detach $prev_rev

if [ x"$res" = x"0" ]; then
    if $rebase_workaround; then
	echo "Rebase workaround: no regression between $baseline_rev and $bad_rev"
	sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name
	project_name="${rr[ci_project]}/${rr[ci_config]}:$current_project"
	case $project_name in
	    tcwg_kernel/llvm-*-next-*:linux)
		cat > $artifacts/trigger-build-rebase <<EOF
llvm_branch=baseline
EOF
		;;
	    tcwg_kernel/gnu-*-next-*:linux)
		cat > $artifacts/trigger-build-rebase <<EOF
binutils_branch=baseline
gcc_branch=baseline
EOF
		;;
	    *) assert_with_msg "Unknown project name: $project_name" false ;;
	esac
	cat >> $artifacts/trigger-build-rebase <<EOF
linux_url=$bad_url
linux_branch=$baseline_rev
update_baseline=reset
EOF
    else
	echo "WARNING: build for bad_rev $bad_rev succeeded"
	sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name
	# Regressions in speed benchmarking are not stable,
	# so retry with resetting baseline artifacts.
	# Retry with default parameters for other cases.
	case "${rr[ci_project]}/${rr[ci_config]}" in
	    tcwg_bmk*/gnu-*-O[23]*)
		cat > $artifacts/trigger-build-reset <<EOF
binutils_branch=baseline
gcc_branch=baseline
glibc_branch=baseline
update_baseline=reset
EOF
		;;
	    tcwg_bmk*/llvm-*-O[23]*)
		cat > $artifacts/trigger-build-reset <<EOF
binutils_branch=baseline
gcc_branch=baseline
glibc_branch=baseline
llvm_branch=baseline
update_baseline=reset
EOF
		;;
	    *)
		cat > $artifacts/trigger-build-retry <<EOF
EOF
		;;
	esac
    fi
    echo > $artifacts/jenkins/mail-recipients.txt
    trap "" EXIT
    exit 0
elif [ x"$res" = x"125" ]; then
    # We have confirmed a regression, but not what we have been triggered
    # to bisect.
    echo "WARNING: build for bad_rev $bad_rev showed uninteresting regression"
    sed -i -e "s/\$/-uninteresting/" $artifacts/jenkins/build-name
    trap "" EXIT
    exit 0
fi

if ! git bisect log | grep -q "^git bisect .* $bad_rev\$"; then
    git bisect bad $bad_rev
    ln -f -s "build-$bad_rev" "$artifacts/build-bad"
    ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh"
fi

# Clone interesting-commits.git repo, which contains a list of SHA1s
# that might cut down bisection time.  Mostly, these are first_bad and
# last_good commits.
interesting_commits_rev=${interesting_commits_rev-linaro-local/ci/${rr[ci_project]}}
clone_or_update_repo ../interesting-commits $interesting_commits_rev https://git-us.linaro.org/toolchain/ci/interesting-commits.git auto $interesting_commits_rev
interesting_commits_rev=$(git -C ../interesting-commits rev-parse HEAD)
cat <<EOF | manifest_out
declare -g interesting_commits_rev=$interesting_commits_rev
EOF

# Add SHA1 commit $1 to interesting-commits and push the repo.
# If $2 is "regression" then record current configuration as having regressed
# at this commit.
# Ignore failures (since this is cache handling).
push_interesting_commit ()
{
    declare -g push_interesting_commit_result=0
    (
    set -euf -o pipefail

    local sha1="$1"
    local kind="$2"
    local -a configs

    if ! grep -q "^$sha1" ../interesting-commits/$current_project; then
	echo "$sha1" >> ../interesting-commits/$current_project
    fi

    if [ x"$kind" = x"regression" ]; then
	mapfile -t configs < <(grep "^$sha1" ../interesting-commits/$current_project | sed -e "s/^$sha1 *//")
	configs+=("${rr[ci_project]}"/"${rr[ci_config]}")
	mapfile -t configs < <(echo "${configs[@]}" | tr ' ' '\n' | sort -u)
	sed -i -e "s#^$sha1.*\$#$sha1 ${configs[*]}#" ../interesting-commits/$current_project
    fi

    git -C ../interesting-commits add .
    if [ x"$(git -C ../interesting-commits status --short)" = x"" ]; then
	# No file has changed.  We've been here before...
	# E.g., this is a re-occuring regression in linux-next.
	exit 125
    fi
    git -C ../interesting-commits commit -m "Add $kind $sha1 from $BUILD_URL

${configs[*]}" &
    local res=0 && wait $! || res=$?

    if [ x"$res" = x"0" ]; then
	git_init_linaro_local_remote ../interesting-commits baseline false
	git_push ../interesting-commits baseline linaro-local/ci/${rr[ci_project]}
    fi
    ) &
    wait $! || push_interesting_commit_result=$?
}

# Print first_bad revision (if detected)
get_first_bad ()
{
    (
    # Allow pipefail to handle error exit codes from git bisect log and grep.
    # Note that child shell inherits settings from parent shell, so we need
    # excplicitly set "+o pipefail".
    set -euf +o pipefail

    git bisect log | tail -n1 | grep "^# first bad commit:" \
	| sed -e "s/^# first bad commit: \[\([0-9a-f]*\)\].*/\1/"
    )
}

print_tested_revs ()
{
    (
    # Allow pipefail to handle error exit codes from git bisect log and grep.
    # Note that child shell inherits settings from parent shell, so we need
    # excplicitly set "+o pipefail".
    set -euf +o pipefail

    local kind="$1"
    git bisect log | grep "^git bisect $1 " | sed -e "s/^git bisect $1 //"
    )
}

# Try to reduce bisection range by testing regressions (and their parents)
# identified in other configurations.
touch ../interesting-commits/$current_project
# This loop can generate lots of console noise.
set +x
while [ x"$(get_first_bad </dev/null)" = x"" ] && read -a arr; do
    (
    set -euf -o pipefail

    sha1="${arr[0]}"
    # Skip $bad_rev and $baseline_rev, these were already tested.
    if git bisect log | grep -q "^git bisect .* $sha1\$"; then
	# shellcheck disable=SC2106
	continue
    fi

    # Maintain "git bisect" invariant: all good revisions are ancestors
    # of all bad revisions.  If we [forcefully] test a revision for
    # which the invariant doesn't hold, then "git bisect" will fail.
    skip=false
    for tested_bad in $(print_tested_revs bad); do
	if ! git merge-base --is-ancestor $sha1 $tested_bad; then
	    skip=true
	fi
    done
    for tested_good in $(print_tested_revs good); do
	if git merge-base --is-ancestor $sha1 $tested_good; then
	    skip=true
	fi
    done
    if $skip; then
	# shellcheck disable=SC2106
	continue
    fi

    if $verbose; then set -x; fi

    git checkout --detach $sha1
    $artifacts/test.sh &
    res=0 && wait $! || res=$?
    if [ x"$res" = x"0" ]; then
	git bisect good
    elif [ x"$res" = x"125" ]; then
	git bisect skip
    else
	git bisect bad
    fi
    ) </dev/null
done < <(tac ../interesting-commits/$current_project)
if $verbose; then set -x; fi

if [ x"$(get_first_bad)" = x"" ]; then
    git bisect run $artifacts/test.sh &
    res=0 && wait $! || res=$?

    if [ x"$res" = x"0" ]; then
	assert_with_msg "Didn't find first bad commit!" [ x"$(get_first_bad)" != x"" ]
    fi
fi

first_bad=$(get_first_bad)
notify_devs=true
if [ x"$first_bad" != x"" ]; then
    # "git bisect run" succeeded.  Check whether this is an actual regression
    # or bisection artifact.
    res=0
    for last_good in $(git rev-parse $first_bad^@); do
	# It seems that git-bisect assumes parent commit as "good" on
	# the basis of one of its children being "good".  Therefore we
	# can have a situation when we have parent P with children C1 and C2,
	# and child C1 has a child of its own CB.  Git-bisect tests C2 as
	# "good", and CB as "bad".  From C2 being good it assumes P as "good",
	# and it knows CB is "bad", so git-bisect returns C1 as the first bad
	# commit.
	# To simplify investigations we explicitly test parent of $first_bad.
	echo "Testing first_bad's parent $last_good (hoping for success)"
	git checkout --detach "$last_good"
	$artifacts/test.sh &
	res=0 && wait $! || res=$?
	if [ x"$res" = x"0" ]; then
	    break
	fi
    done
    # Add both $last_good and $first_bad to interesting commits.
    push_interesting_commit $last_good "last-good"
    push_interesting_commit $first_bad "regression"
    if [ x"$push_interesting_commit_result" = x"125" ]; then
	notify_devs=false
    fi
    if [ x"$res" = x"0" ]; then
	# Success!  Touch $artifacts/first-bad as a marker of successful bisect.
	echo $first_bad > $artifacts/first-bad
    else
	# It seems $last_good was on a path that tested good, even though
	# it itself is bad.
	#
	# We need to be careful to avoid re-trigger loops
	assert_with_msg "Last good is not an ancestor of bad rev!" \
	  git merge-base --is-ancestor $last_good $bad_rev
	if git merge-base --is-ancestor $baseline_rev $last_good; then
	    # $last_good is a child of $baseline_rev, so we can re-trigger
	    # bisection with reduced bisection range.
	    cat > $artifacts/trigger-bisect <<EOF
current_project=$current_project
bad_url=$bad_url
bad_branch=$last_good
EOF
	    sed -i -e "s/\$/-last_good-bad/" $artifacts/jenkins/build-name
	    # Don't send any emails.
	    echo > $artifacts/jenkins/mail-recipients.txt
	    trap "" EXIT
	    exit 0
	fi
	# This case will be handled similar to "git bisect run" failure below.
	# We are going to reset baseline to $first_bad.
    fi
else
    # When "git bisect run" fails, e.g., due to merge-base of $baseline_rev and
    # $bad_rev is worse than $baseline_rev, we want to reset baseline to HEAD,
    # so that we catch most of the commits that introduced change in the result
    # metric.
    first_bad=$(git rev-parse HEAD)
    push_interesting_commit $first_bad "bad-merge-base"
fi
cd ..

# Save BISECT_* logs
find "$current_project" -path "$current_project/.git/BISECT_*" -print0 | xargs -0 -I@ mv @ $artifacts/git-logs/

# Remove any fail-safe email body
rm -f $artifacts/jenkins/mail-body.txt

if [ -f $artifacts/first-bad ]; then
    mkdir -p $artifacts/jenkins
    sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name

    ln -f -s "build-$first_bad" "$artifacts/build-first_bad"
    ln -f -s "build-$first_bad.sh" "$artifacts/manifests/build-first_bad.sh"

    good_name="last_good"
    good_sha1="$last_good"
    bad_name="first_bad"
    bad_sha1="$first_bad"

    ln -f -s "build-$last_good" "$artifacts/build-last_good"
    ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh"

    occurences="$(cat interesting-commits/$current_project | grep "^$first_bad" | sed -e "s/^$first_bad *//" | tr ' ' '\n' | sed "s#^# - #")"
    if [ "$(echo "$occurences" | wc -l)" -le 1 ]; then
	git_log_level="medium"
    else
	git_log_level="short"
    fi

    cat >> $artifacts/jenkins/mail-body.txt <<EOF
Successfully identified regression in *$current_project* in CI configuration ${rr[ci_project]}/${rr[ci_config]}.  So far, this commit has regressed CI configurations:
$occurences

Culprit:
<cut>
$(git -C $current_project log --pretty=$git_log_level -n 1 $first_bad)
</cut>

EOF
else
    good_name="baseline_rev"
    good_sha1="$baseline_rev"
    bad_name="bad"
    bad_sha1="$bad_rev"
    cat >> $artifacts/jenkins/mail-body.txt <<EOF
Could not identify regression in *$current_project* in CI configuration ${rr[ci_project]}/${rr[ci_config]}.  See 'Bisect log' in the links below for bisection details.

EOF
fi

if [ x"${JIRA_PASSWORD+set}" = x"set" ] && [ -f $artifacts/first-bad ]; then
    case "${rr[ci_project]}/${rr[ci_config]}" in
	tcwg_kernel/gnu-*) jira_card="GNU-587" ;;
	tcwg_kernel/llvm-*) jira_card="LLVM-480" ;;
	# _tk1 and _tx1 jobs
	tcwg_bmk_t?1/gnu-*) jira_card="GNU-661" ;;
	tcwg_bmk_t?1/llvm-*) jira_card="LLVM-614" ;;
	tcwg_binutils/*) jira_card="GNU-93" ;;
	tcwg_cross/*) jira_card="GNU-93" ;;
	tcwg_gnu/*) jira_card="GNU-93" ;;
	*) jira_card="" ;;
    esac

    if $notify_devs && [ x"$jira_card" != x"" ]; then
	cat > $artifacts/jenkins/jira-body.txt <<EOF
[$jira_card]
$(cat $artifacts/jenkins/mail-body.txt)

Details: ${BUILD_URL}artifact/$rel_artifacts/jenkins/mail-body.txt/*view*/
Even more details: ${BUILD_URL}artifact/$rel_artifacts/
EOF
	(
            export JIRA_USERNAME=tcwg-jira
            export JIRA_PASSWORD
            export EDITOR=cat
            echo y | jipdate.py -f $artifacts/jenkins/jira-body.txt
        )
    fi
fi

cat >> $artifacts/jenkins/mail-body.txt <<EOF
Results regressed to (for $bad_name == $bad_sha1)
$(cat $artifacts/build-$bad_sha1/results)

from (for $good_name == $good_sha1)
$(cat $artifacts/build-$good_sha1/results)

EOF

good_build_artifacts="Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/"
bad_build_artifacts="Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/"

# Benchmark builds have a results ID that is used to download the results
if test -f "$artifacts/build-$good_sha1/results_id"; then
  cat >> $artifacts/jenkins/mail-body.txt <<EOF
$good_build_artifacts
Results ID of $good_name: $(cat $artifacts/build-$good_sha1/results_id)
$bad_build_artifacts
Results ID of $bad_name: $(cat $artifacts/build-$bad_sha1/results_id)
EOF
# Os vs Os LTO builds have two result IDs per job
elif test -f "$artifacts/build-$good_sha1/results_id-1"; then
  cat >> $artifacts/jenkins/mail-body.txt <<EOF
$good_build_artifacts
Results IDs of $good_name:
$(cat $artifacts/build-$good_sha1/results_id-1)
$(cat $artifacts/build-$good_sha1/results_id-2)
$bad_build_artifacts
Results ID of $bad_name:
$(cat $artifacts/build-$bad_sha1/results_id-1)
$(cat $artifacts/build-$bad_sha1/results_id-2)
EOF
else
  cat >> $artifacts/jenkins/mail-body.txt <<EOF
$good_build_artifacts
$bad_build_artifacts
EOF
fi

cat >> $artifacts/jenkins/mail-body.txt <<EOF
Build top page/logs: ${BUILD_URL}

Configuration details:
$(cat $artifacts/manifests/build-baseline.sh | grep '_url]\|_branch]' | grep -v '="no_')

Reproduce builds:
<cut>
mkdir investigate-$current_project-$bad_sha1
cd investigate-$current_project-$bad_sha1

git clone https://git.linaro.org/toolchain/jenkins-scripts

mkdir -p $rel_artifacts/manifests
curl -o $rel_artifacts/manifests/build-baseline.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-baseline.sh --fail
curl -o $rel_artifacts/manifests/build-parameters.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-parameters.sh --fail
curl -o $rel_artifacts/test.sh ${BUILD_URL}artifact/$rel_artifacts/test.sh --fail
chmod +x $rel_artifacts/test.sh

# Reproduce the baseline build (build all pre-requisites)
$build_script @@ $rel_artifacts/manifests/build-baseline.sh

cd $current_project

# Reproduce $bad_name build
git checkout --detach $bad_sha1
../$rel_artifacts/test.sh

# Reproduce $good_name build
git checkout --detach $good_sha1
../$rel_artifacts/test.sh

cd ..
</cut>

History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}

Artifacts: ${BUILD_URL}artifact/$rel_artifacts/
Build log: ${BUILD_URL}consoleText
EOF

if [ -f $artifacts/first-bad ]; then
    cat >> $artifacts/jenkins/mail-body.txt <<EOF

Full commit (up to 1000 lines):
<cut>
$(git -C $current_project show --stat --patch $first_bad | head -n 1000)
</cut>
EOF
fi

# Set mail recipients last to preserve catch-error value from .yaml file.
# Email developers.
CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org"
case "$current_project" in
    llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, llvm@linaro.org" ;;
    binutils|gcc|glibc|newlib) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, tcwg-gcc@linaro.org" ;;
esac
case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
    tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, clang-built-linux@googlegroups.com" ;;
    tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, clang-built-linux@googlegroups.com" ;;
esac
if $notify_devs; then
    cat > $artifacts/jenkins/mail-recipients.txt <<EOF
$CI_MAIL_RECIPIENTS
EOF
fi

# Reset baseline to the regressed commit so that we will catch subsequent
# regressions (worse than $bad_rev).
cp $artifacts/build-$first_bad/trigger-build-$current_project $artifacts/trigger-build-1-reset
echo "update_baseline=reset" >> $artifacts/trigger-build-1-reset

# Trigger master build now instead of waiting for next timed SCM trigger.
if [ -f $artifacts/build-$bad_rev/trigger-build-$current_project ]; then
    cp $artifacts/build-$bad_rev/trigger-build-$current_project $artifacts/trigger-build-2-default
else
    # When replaying a bisect we often skip the $bad_rev build, so we won't
    # have its artifacts.  Use $first_bad's trigger-build-* instead.
    cp $artifacts/build-$first_bad/trigger-build-$current_project $artifacts/trigger-build-2-default
    sed -i -e "s/$first_bad/$bad_rev/" $artifacts/trigger-build-2-default
fi

trap "" EXIT