#!/bin/bash

set -ef -o pipefail

scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh

artifacts=$(pwd)/artifacts
rm -rf $artifacts
mkdir $artifacts

# Process bisect-only args
convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"

bad_rev="${bad_rev:-default}"
baseline_rev="${baseline_rev:-baseline}"
BUILD_URL="${BUILD_URL:-$(pwd)}"
confirm_failure="${confirm_failure:-false}"

# Process build args and record them in build-manifest.sh
convert_args_to_variables %% artifacts/build-manifest.sh "$@"

obligatory_variables current_project toolchain

linux_config="${linux_config:-defconfig}"
target="${target:-native}"
verbose="${verbose:-true}"

set -u

if $verbose; then set -x; fi

trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/failures" EXIT

if $confirm_failure; then
    # Make sure we are not about to bisect a sporadic failure.
    $scripts/tcwg_kernel-build.sh \
	@@ artifacts/build-manifest.sh \
	--mode "baseline" \
	--current_rev "$bad_rev" \
	--top_artifacts "$artifacts/build-bad" \
	--verbose "$verbose" &
    res=0 && wait $! || res=$?

    bad_rev=$(git -C $current_project rev-parse HEAD)
    if [ x"$res" = x"0" ]; then
	# Build job had a spurious failure.  Re-try.
	cat > $artifacts/trigger-2-build-master <<EOF
current_project=$current_project
current_rev=$bad_rev
EOF
	trap "" EXIT
	exit 0
    fi
    mv "$artifacts/build-bad" "$artifacts/build-$bad_rev-bad"
elif [ x"$bad_rev" = x"default" ]; then
    echo "ERROR: Need explicit --bad_rev"
    exit 1
else
    bad_rev=$(git -C $current_project rev-parse "$bad_rev")
fi

# Build baseline that we are going to re-use to speed-up bisection.
# (This also confirms that infrastructure is OK.)
$scripts/tcwg_kernel-build.sh \
    @@ artifacts/build-manifest.sh \
    --mode "baseline" \
    --current_rev "$baseline_rev" \
    --reset_baseline true \
    --top_artifacts "$artifacts/build-baseline" \
    --verbose "$verbose"

assert ! [ -f artifacts/failures ]

cd $current_project

# Remember $good_rev from the baseline build above.
good_rev=$(git rev-parse HEAD)
baseline_rev="$good_rev"
mv "$artifacts/build-baseline" "$artifacts/build-$good_rev-baseline"

# Bisect script.
#
# With this script we find the first commit that has regressed compared
# to baseline, but not, necessarily, the commit that caused regression in
# $bad_rev.  Consider the scenario:
# - rev_10 produced good result "2000" -- this is current baseline
# - rev_20 completely broke the build (say, result "10")
# - rev_22 fixed the build
# - rev_30 regressed the build to result "1000" -- this is the regression we
#   detected vs "2000" baseline.
#
# The script will identify rev_20 as the first failing commit, which will
# cause the baseline to be reset to rev_20 with metric "10".  When we then
# rebuild master (at rev_30) we will see a /progression/ from "10" to "1000",
# thus missing the regression of "2000" to "1000".
#
# To catch the "2000" to "1000" regression someone would need to manually
# trigger bisect between rev_22 and rev_30.
#
# TODO: We could skip revisions (exit 125) that are worse than metric
# for $bad_rev (result metric <1000 in the above scenario), so we would
# skip revisions between rev_20 and rev_22.  This might cause other edge
# cases to be handled sub-optimally, though.
cat > ../bisect-run.sh <<EOF
#!/bin/sh
rev=\$(git rev-parse HEAD)
cd ..
$scripts/tcwg_kernel-build.sh \
  @@ artifacts/build-manifest.sh \
  --mode bisect \
  --top_artifacts $artifacts/build-\$rev \
  --verbose "$verbose" &
res=0 && wait \$! || res=\$?
if [ -f $artifacts/build-\$rev/failures ]; then
  exit 1
elif [ x"\$res" != x"0" ]; then
  exit 125
else
  exit 0
fi
EOF
chmod +x ../bisect-run.sh

# Workaround linux-next/master rebasing on top of linux-next/stable.
# Try to find $good_rev that is "good" compared to baseline
# and that is an ancesstor of both $baseline_rev and $bad_rev.
merge_base=$(git merge-base $bad_rev HEAD)
origin=$(git remote get-url origin)
origin=$(basename "$origin")
if [ x"$merge_base" != x"$good_rev" -a x"$origin" = x"linux-next.git" ]; then
    git checkout --detach "$merge_base"
    ../bisect-run.sh &
    res=0 && wait $! || res=$?
    if [ x"$res" != x"0" ]; then
	# With bad merge-base below bisect will fail.
	# Check if we can use linux-next/stable as our last resort.
	git checkout --detach refs/remotes/origin/stable
	../bisect-run.sh &
	res=0 && wait $! || res=$?
	if [ x"$res" = x"0" ]; then
	    good_rev=$(git rev-parse HEAD)
	else
	    :
	    # Below bisect will fail :-(
	fi
    else
	:
	# With good merge-base below bisect should succeed.
    fi
fi

if ! $confirm_failure; then
    git checkout --detach $bad_rev
    ../bisect-run.sh &
    res=0 && wait $! || res=$?

    if [ x"$res" = x"0" ]; then
	echo "ERROR: build for bad_rev $bad_rev succeeded"
	exit 1
    fi

    mv "$artifacts/build-$bad_rev" "$artifacts/build-$bad_rev-bad"
fi

git bisect start $bad_rev $good_rev 2>&1 | tee $artifacts/bisect.log

# "git bisect run" can fail (exit with non-zero) in a number of cases:
# - on trivial bisects (e.g., between HEAD^ and HEAD),
# - when merge-base between baseline and bad is worse than baseline,
# - something else?
# In all these cases we want to reset baseline to HEAD, so that we catch
# most of the commits that introduced change in the result metric.
git bisect run ../bisect-run.sh 2>&1 | tee -a $artifacts/bisect.log &
res=0 && wait $! || res=$?

if [ x"$res" = x"0" ]; then
    first_bad=$(cat .git/BISECT_RUN | head -n 1 | grep "is the first bad commit" | cut -d" " -f 1)
    assert [ x"$first_bad" != x"" ]
    echo $first_bad > $artifacts/first-bad
else
    first_bad=$(git rev-parse HEAD)
    if ! [ -f .git/BISECT_LOG ]; then
	# It seems this was a trivial bisect with $bad_rev^ == $good_rev.
	first_bad=$bad_rev
	echo $first_bad > $artifacts/first-bad
    fi
fi
cd ..

# Save BISECT_* logs
mkdir $artifacts/git-logs
find "$current_project" -path "$current_project/.git/BISECT_*" -print0 | xargs -0 -I@ mv @ $artifacts/git-logs/

# Reset baseline to the regressed commit so that we will catch subsequent
# regressions (worse than $bad_rev).
cat > $artifacts/trigger-1-reset-baseline <<EOF
current_project=$current_project
current_rev=$first_bad
reset_baseline=true
EOF

if [ -f artifacts/first-bad ]; then
    first_bad=$(cat artifacts/first-bad)
    touch build-name
    sed -i -e "s/\$/-$first_bad/" build-name

    if [ x"$first_bad" != x"$bad_rev" ]; then
	mv "$artifacts/build-$first_bad" "$artifacts/build-$first_bad-first_bad"
    else
	mv "$artifacts/build-$first_bad-bad" "$artifacts/build-$first_bad-first_bad"
	ln -s "build-$first_bad-first_bad" "$artifacts/build-$first_bad-bad"
    fi

    last_good=$(git -C $current_project rev-parse $first_bad^)
    if [ x"$last_good" != x"$baseline_rev" ]; then
	mv "$artifacts/build-$last_good" "$artifacts/build-$last_good-last_good"
    else
	mv "$artifacts/build-$last_good-baseline" "$artifacts/build-$last_good-last_good"
	ln -s "build-$last_good-last_good" "$artifacts/build-$last_good-baseline"
    fi

    # Email developers on successful bisects.
    CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org"
    case "$toolchain:$current_project" in
        gnu:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;;
        llvm:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
        llvm:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;;
    esac

    cat > artifacts/ci_mail_parameters.sh <<EOF
CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS"
CI_MAIL_BODY="Successfully identified regression in $current_project on $target @ $toolchain-${rr[${toolchain}_version]} @ linux-${rr[linux_version]}-$linux_config between bad $bad_rev and baseline $baseline_rev .

Culprit:
<cut>
$(git -C $current_project log -n 1 $first_bad | tr '"' "'")
</cut>

Results regressed from (for $last_good)

<cut>
$(cat artifacts/build-$last_good-last_good/results)
</cut>

to (for $first_bad)

<cut>
$(cat artifacts/build-$first_bad-first_bad/results)
</cut>

Artifacts of first_bad build: ${BUILD_URL}artifact/artifacts/build-$first_bad-first_bad/
Artifacts of last_good build: ${BUILD_URL}artifact/artifacts/build-$last_good-last_good/

Reproduce builds:
<cut>
mkdir investigate-$current_project-$first_bad
cd investigate-$current_project-$first_bad
git clone https://git.linaro.org/toolchain/jenkins-scripts
wget ${BUILD_URL}artifact/artifacts/build-manifest.sh
mkdir reproduce-good
cd reproduce-good
../jenkins-scripts/tcwg_kernel-build.sh @@ ../build-manifest.sh ==rr[${current_project}_rev] $last_good
cd ..
mkdir reproduce-bad
cd reproduce-bad
../jenkins-scripts/tcwg_kernel-build.sh @@ ../build-manifest.sh ==rr[${current_project}_rev] $first_bad
cd ..
</cut>

History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/tcwg_kernel/$toolchain-${rr[${toolchain}_version]}-$target-${rr[linux_version]}-$linux_config

Bisect log: ${BUILD_URL}artifact/artifacts/bisect.log/*view*/
Artifacts: ${BUILD_URL}artifact/artifacts/
Build URL: $BUILD_URL
Build log: ${BUILD_URL}consoleText

Full commit:
<cut>
$(git -C $current_project show $first_bad | tr '"' "'")
</cut>
"
EOF
    chmod +x artifacts/ci_mail_parameters.sh
fi

# Trigger master build now instead of waiting for next timed SCM trigger.
cat > $artifacts/trigger-2-build-master <<EOF
current_project=$current_project
EOF

trap "" EXIT