[WIP] Precommit supportnew-flaky

Change-Id: I1c94ba22edce24826ce246e2dadc2532435f48e2

5 files changed, 769 insertions, 72 deletions

diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index ce225e9d..83de76fd 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -34,7 +34,7 @@ assert_with_msg ()
     local failure_message=$1
     shift
 
-    eval "$@" || (echo "$failure_message" && exit 1)
+    eval "$@" || (echo "$failure_message" >&2 && exit 1)
     )
 }
 
@@ -518,6 +518,14 @@ clone_or_update_repo_no_checkout ()
 	# Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock
 	# These occur when builds are aborted during "git remote update" or similar.
 	find "$dir/.git" -name "*.lock" -delete
+
+	# Recover from any previous am/cherry-pick/rebase.
+	# In pre-commit CI we apply patches with "git am", which can fail
+	# and leave clone in a bad state.
+	local i
+	for i in am cherry-pick rebase; do
+	    git -C "$dir" "$i" --abort &>/dev/null || true
+	done
     fi
 
     git_set_remote "$dir" "$remote" "$url" "$single_branch"
@@ -944,9 +952,18 @@ get_baseline_git ()
 {
     (
     set -euf -o pipefail
+
+    local base_artifacts
+
+    if test_array rr && [[ -v rr[base_artifacts] ]]; then
+      base_artifacts=${rr[base_artifacts]}
+    else
+      base_artifacts="base-artifacts"
+    fi
+
     assert_with_msg "ERROR: No $1 in baseline git" \
-		    [ -f "${rr[base_artifacts]}/git/$1" ]
-    cat "${rr[base_artifacts]}/git/$1"
+		    [ -f "$base_artifacts/git/$1" ]
+    cat "$base_artifacts/git/$1"
     )
 }
 
@@ -1793,3 +1810,241 @@ describe_sha1 ()
 	echo "$component#$(git -C "$component" rev-parse --short)"
     fi
 }
+
+pw_init ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+
+    git -C "$project" config pw.server \
+	"https://patchwork.sourceware.org/api/1.2/"
+    git -C "$project" config pw.project "$project"
+
+    set +x
+    local token="$2"
+    if [ "$token" != "" ]; then
+	git -C "$project" config pw.token "$token"
+    fi
+    )
+}
+
+pw_yaml_get ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local section="$2"
+    local id="$3"
+    local match_field="$4"
+    local match_value="$5"
+    local get_field="$6"
+    local match_stop="${7-}"
+    local match_num="${8-}"
+
+    # Reduce noise in the logs
+    set +x
+
+    local yaml
+    yaml=$(mktemp)
+    # shellcheck disable=SC2064
+    trap "rm -f $yaml" EXIT
+
+    local -a cmd
+    case "$id" in
+	"--owner"*)
+	    # shellcheck disable=SC2206
+	    cmd=(list $id)
+	    ;;
+	*)
+	    cmd=(show "$id")
+	    ;;
+    esac
+
+    git -C "$project" pw "$section" "${cmd[@]}" -f yaml > "$yaml"
+
+    local len
+    len=$(shyaml get-length < "$yaml")
+    while [ "$len" -gt "0" ]; do
+	len=$(($len - 1))
+	if [ "$match_value" != ".*" ]; then
+	    if shyaml get-value "$len.$match_field" < "$yaml" \
+		    | grep "$match_value" >/dev/null; then
+		if [ "$match_num" = "" ] || [ "$match_num" = "0" ]; then
+		    shyaml get-value "$len.$get_field" < "$yaml"
+		    return 0
+		fi
+		match_num=$(($match_num - 1))
+	    fi
+	else
+	    # Special case for $match_value == ".*".
+	    # Only check that $match_field exist, and don't look at the value.
+	    # This handles empty values without involving grep, which can't
+	    # match EOF generated by empty value.
+	    if shyaml get-value "$len.$match_field" < "$yaml" >/dev/null; then
+		if [ "$match_num" = "" ] || [ "$match_num" = "0" ]; then
+		    shyaml get-value "$len.$get_field" < "$yaml"
+		    return 0
+		fi
+		match_num=$(($match_num - 1))
+	    fi
+	fi
+	if [ "$match_stop" != "" ] \
+	       && shyaml get-value "$len.$match_field" < "$yaml" \
+		   | grep "$match_stop" >/dev/null; then
+	    return 1
+	fi
+    done
+
+    assert_with_msg "Missing $match_field == $match_value" false
+    )
+}
+
+pw_series_complete_p ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local series_id="$2"
+
+    local value
+    value=$(pw_yaml_get "$project" series "$series_id" property Complete value)
+    if [ "$value" = "True" ]; then
+	return 0
+    fi
+    return 1
+    )
+}
+
+pw_get_patch_from_bundle_or_series ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local bundle_or_series="$2"
+    local id="$3"
+    local num="$4"
+
+    local patch_id
+    patch_id=$(pw_yaml_get "$project" $bundle_or_series "$id" property ".*" \
+			   value Patches "$num" | cut -d" " -f 1)
+    if [ "$patch_id" = "" ]; then
+	return 1
+    fi
+    assert_with_msg "Invalid patch_id $patch_id" \
+		    git -C "$project" pw patch show "$patch_id" > /dev/null
+    echo "$patch_id"
+    )
+}
+
+pw_bundle_name ()
+{
+    (
+    set -euf -o pipefail
+    local action="$1"
+    local ci_bot="$2"
+
+    # Patchworks limits bundle names to 50 symbols, so reduce $ci_bot name
+    # to 39 bytes: 30 bytes of the original $ci_bot + "-" + 8-byte hash.
+    if [ ${#ci_bot} -gt "39" ]; then
+	local hash
+	hash=$(echo "$ci_bot" | md5sum | cut -b 1-8)
+	ci_bot="${ci_bot::29}-$hash"
+    fi
+
+    # Then append 1 byte of "-" + at most 9 bytes of $action.
+    assert_with_msg "action too long" \
+		    [ ${#action} -le 9 ]
+    ci_bot="$ci_bot-$action"
+
+    assert_with_msg "bundle name too long" \
+		    [ ${#ci_bot} -le 49 ]
+
+    echo "$ci_bot"
+    )
+}
+
+pw_bundle_id ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local name="$2"
+
+    pw_yaml_get "$project" bundle "--owner linaro-tcwg-bot" name "$name" id
+    )
+}
+
+pw_bundle_has_patch_p ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local bundle_id="$2"
+    local patch_id="$3"
+
+    if [ "$bundle_id" = "" ]; then
+	return 1
+    fi
+
+    pw_yaml_get "$project" bundle "$bundle_id" value "^$patch_id " value \
+		"^[^0-9]" >/dev/null 2>&1
+    )
+}
+
+pw_ci_bot_bundle_check_and_add ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local bundle_type="$2"
+    local ci_bot="$3"
+    local patch_id="$4"
+    local skip_check="$5"
+
+    local bundle_name bundle_id
+    bundle_name=$(pw_bundle_name "$bundle_type" "$ci_bot")
+    bundle_id=$(pw_bundle_id "$project" "$bundle_name" || true)
+
+    if [ "$bundle_id" != "" ]; then
+	if ! $skip_check \
+		&& pw_bundle_has_patch_p \
+		       "$project" "$bundle_id" "$patch_id"; then
+	    return 1
+	fi
+	git -C "$project" pw bundle add -f yaml "$bundle_id" "$patch_id"
+    else
+	git -C "$project" pw bundle create --public -f yaml \
+	    "$bundle_name" "$patch_id"
+    fi
+    )
+}
+
+pw_ci_bot_bundle_remove ()
+{
+    (
+    set -euf -o pipefail
+    local project="$1"
+    local bundle_type="$2"
+    local ci_bot="$3"
+    local patch_id="$4"
+    local allow_delete="${5-true}"
+
+    local bundle_name bundle_id
+    bundle_name=$(pw_bundle_name "$bundle_type" "$ci_bot")
+    bundle_id=$(pw_bundle_id "$project" "$bundle_name" || true)
+
+    if ! pw_bundle_has_patch_p "$project" "$bundle_id" "$patch_id"; then
+	return
+    fi
+
+    local only_patch
+    only_patch=$(pw_yaml_get "$project" bundle "$bundle_id" property ".*" \
+			     property Patches)
+    if [ "$only_patch" != "Patches" ]; then
+	git -C "$project" pw bundle remove -f yaml "$bundle_id" "$patch_id"
+    elif $allow_delete; then
+	git -C "$project" pw bundle delete -f yaml "$bundle_id"
+    fi
+    )
+}
diff --git a/pw-apply.sh b/pw-apply.sh
new file mode 100755
index 00000000..67cf6d20
--- /dev/null
+++ b/pw-apply.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -euf -o pipefail +x
+
+scripts=$(dirname $0)
+# shellcheck source=jenkins-helpers.sh
+. $scripts/jenkins-helpers.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables ci_bot project pw_url pw_token pw_dir build_url
+declare              ci_bot project pw_url pw_token pw_dir build_url
+
+verbose="${verbose-true}"
+
+if $verbose; then
+    set -x
+fi
+
+case "$pw_url" in
+    "pw://series/"*)
+	series_id=$(echo "$pw_url" | cut -d/ -f 4)
+	;;
+    *) assert_with_msg "pw_url does not match pw://series/*" false ;;
+esac
+
+retrigger=false
+case "$pw_url/" in
+    "pw://series/"*"/retrigger/"*) retrigger=true ;;
+esac
+
+url=$(get_baseline_git "${project}_url")
+rev=$(get_baseline_git "${project}_rev")
+echo "Fetching baseline $url#$rev"
+clone_or_update_repo "$project" "$rev" "$url" > /dev/null
+
+(set +x; pw_init "$project" "$pw_token")
+
+if ! pw_series_complete_p "$project" "$series_id"; then
+    echo "ERROR: Series $series_id is not complete"
+    exit 2
+fi
+
+num_patch=0
+while true; do
+    patch_id=$(pw_get_patch_from_bundle_or_series \
+		   "$project" series "$series_id" "$num_patch" || true)
+    if [ "$patch_id" = "" ]; then
+	break
+    fi
+
+    pw_ci_bot_bundle_check_and_add \
+	"$project" triggered "$ci_bot" "$patch_id" "$retrigger" &
+    res=0 && wait $! || res=$?
+    if [ $res = 0 ]; then
+	break
+    fi
+
+    num_patch=$(($num_patch + 1))
+done
+
+if [ "$patch_id" = "" ]; then
+    echo "ERROR: All patches in series $series_id are already tested"
+    exit 3
+fi
+
+# Remove the patch from queue bundle, unless it is the last one.
+# We avoid deleting and re-creatthe queued bundles to conserve bundle IDs.
+pw_ci_bot_bundle_remove "$project" queued "$ci_bot" "$patch_id" false
+
+clone_or_update_repo glibc-cicd main \
+		     https://gitlab.com/djdelorie/glibc-cicd.git > /dev/null
+(
+    set +x;
+    if [ "$pw_token" != "" ]; then
+	cat >> glibc-cicd/cicd-config.py <<EOF
+
+patchwork_token = "$pw_token"
+EOF
+    fi
+)
+pw_check_cmd=(glibc-cicd/check.py --patch_id "$patch_id" --context "$ci_bot")
+
+mkdir -p "$pw_dir"
+cat > "$pw_dir/$project" <<EOF
+project="$project"
+patch_id="$patch_id"
+ci_bot="$ci_bot"
+pw_check_cmd=(${pw_check_cmd[@]})
+build_url="$build_url"
+EOF
+
+$scripts/pw-report.sh --check triggered --result pass --pw_dir "$pw_dir"
+
+if ! git -C "$project" pw series apply "$series_id"; then
+    $scripts/pw-report.sh --check apply --result fail --pw_dir "$pw_dir"
+    echo "ERROR: Series $series_id did not apply"
+    exit 4
+fi
+
+git -C "$project" checkout --detach HEAD~$num_patch
+
+$scripts/pw-report.sh --check apply --result pass --pw_dir "$pw_dir"
+
diff --git a/pw-report.sh b/pw-report.sh
new file mode 100755
index 00000000..f14ebf46
--- /dev/null
+++ b/pw-report.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -euf -o pipefail +x
+
+scripts=$(dirname $0)
+# shellcheck source=jenkins-helpers.sh
+. $scripts/jenkins-helpers.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables check result pw_dir
+declare              check result pw_dir
+
+verbose="${verbose-true}"
+
+if $verbose; then
+    set -x
+fi
+
+if ! [ -d "$pw_dir" ]; then
+    exit 0
+fi
+
+bundle_types=({appl,test}-{fail,pass})
+# ... and "triggered" -- all bundle types are <= 9 characters long.
+
+while IFS= read -r -d '' pw_file; do
+    (
+	source "$pw_file"
+	obligatory_variables project patch_id ci_bot pw_check_cmd build_url
+	declare              project patch_id ci_bot pw_check_cmd build_url
+
+	case "$ci_bot" in
+	    tcwg_*_check) pw_check_cmd=(echo DRYRUN: "${pw_check_cmd[@]}") ;;
+	esac
+
+	case "$check-$result" in
+	    triggered-*)
+		# Remove $patch_id from *-pass and *-fail bundles.
+		for i in "${bundle_types[@]}"; do
+		    pw_ci_bot_bundle_remove "$project" "$i" "$ci_bot" \
+					    "$patch_id"
+		done
+		"${pw_check_cmd[@]}" --state pending \
+				     --description "Test started" \
+				     --url "${build_url}console"
+		;;
+	    apply-fail|apply-pass|test-fail|test-pass)
+		if [ "$check" = "apply" ]; then
+		    # Make apply-pass and apply-fail fit into 9 characters.
+		    check="appl"
+		fi
+		pw_ci_bot_bundle_check_and_add \
+		    "$project" "$check-$result" "$ci_bot" "$patch_id" true
+
+		case "$check-$result" in
+		    appl-fail)
+			desc="Patch failed to apply"
+			url="${build_url}artifact/artifacts/jenkins/pw-apply.log"
+			state="fail"
+			;;
+		    appl-pass)
+			desc="Patch applied"
+			url="${build_url}console"
+			state="pending"
+			;;
+		    test-fail)
+			desc="Testing failed"
+			url="${build_url}artifact/artifacts/"
+			state="fail"
+			;;
+		    test-pass)
+			desc="Testing passed"
+			url="$build_url"
+			state="success"
+			;;
+		esac
+		"${pw_check_cmd[@]}" --state "$state" \
+				     --description "$desc" --url "$url"
+		;;
+	    test-ignore)
+		# Testing failed due to sporadic failure.  Allow retrigger testing
+		# of this patch in the next round.
+		pw_ci_bot_bundle_remove "$project" "triggered" "$ci_bot" \
+					"$patch_id"
+		desc="Build did not complete; will be retriggered"
+		"${pw_check_cmd[@]}" --state warning \
+				     --description "$desc" --url "$build_url"
+		;;
+	esac
+    )
+done < <(find "$pw_dir" -type f -print0)
diff --git a/pw-trigger.sh b/pw-trigger.sh
new file mode 100755
index 00000000..1d8c1181
--- /dev/null
+++ b/pw-trigger.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -euf -o pipefail +x
+
+scripts=$(dirname $0)
+# shellcheck source=jenkins-helpers.sh
+. $scripts/jenkins-helpers.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables ci_bot project pw_token out_dir
+declare              ci_bot project pw_token out_dir
+
+verbose="${verbose-true}"
+
+if $verbose; then
+    set -x
+fi
+
+(set +x; pw_init "$project" "$pw_token")
+
+# Delete patches from the queue bundle.  The fact that we are here means
+# that the jenkins queue is empty, and no more patches will be tested.
+# This can happen if a new version of series was uploaded and we tested
+# a different top-of-series patch.
+# We avoid deleting and re-creatthe queued bundles to conserve bundle IDs.
+queued_name=$(pw_bundle_name queued "$ci_bot")
+queued_id=$(pw_bundle_id "$project" "$queued_name" || true)
+while [ "$queued_id" != "" ]; do
+    patch_id=$(pw_get_patch_from_bundle_or_series \
+		   "$project" bundle "$queued_id" 0)
+    git -C "$project" pw bundle remove -f yaml "$queued_id" "$patch_id" &
+    if ! wait $!; then
+	# "bundle remove" will refuse to delete the last patch.
+	break
+    fi
+done
+
+triggered_name=$(pw_bundle_name triggered "$ci_bot")
+triggered_id=$(pw_bundle_id "$project" "$triggered_name" || true)
+
+yaml=$(mktemp)
+trap "rm -f $yaml" EXIT
+
+git -C "$project" pw series list -f yaml > "$yaml"
+mkdir -p "$out_dir"
+
+len=$(shyaml get-length < "$yaml")
+i=$len
+j=0
+# Go through series with the oldest first.
+while [ "$i" -gt "0" ]; do
+    i=$(($i - 1 ))
+    series_id=$(shyaml get-value $i.id < "$yaml")
+
+    if ! pw_series_complete_p "$project" "$series_id"; then
+	continue
+    fi
+
+    num_patch=0
+    while true; do
+	patch_id=$(pw_get_patch_from_bundle_or_series \
+		       "$project" series "$series_id" "$num_patch" || true)
+	if [ "$patch_id" = "" ]; then
+	    break
+	fi
+
+	if ! pw_bundle_has_patch_p "$project" "$triggered_id" "$patch_id"; then
+	    break
+	fi
+	num_patch=$(($num_patch + 1))
+    done
+
+    if [ "$patch_id" = "" ]; then
+	continue
+    fi
+
+    j=$(($j + 1))
+    cat > "$out_dir/trigger-precommit-$project-$j-$series_id" <<EOF
+${project}_git=pw://series/$series_id
+EOF
+
+    pw_ci_bot_bundle_check_and_add \
+	"$project" queued "$ci_bot" "$patch_id" true
+done
+
+echo "Processed $len series and created $j pre-commit triggers"
diff --git a/round-robin.sh b/round-robin.sh
index ed5e1a78..b8832d7e 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -58,7 +58,7 @@ rr[breakup_changed_components]=breakup_changed_components
 
 # Abe's repository and branch to use for the build.
 rr[abe_repo]="https://git-us.linaro.org/toolchain/abe.git"
-rr[abe_branch]="master"
+rr[abe_branch]="new-flaky"
 
 # Host compiler defaults to /usr/bin/gcc and g++
 rr[host_cc]="/usr/bin/gcc"
@@ -160,14 +160,24 @@ clone_repo ()
 
     local url branch
 
-    if [ x"${rr[${project}_git]}" != x"baseline" ]; then
-	# Fetch and checkout from the specified repo.
-	url="${rr[${project}_git]%#*}"
-	branch="${rr[${project}_git]#*#}"
-    else
-	url=$(get_baseline_git ${project}_url)
-	branch=$(get_baseline_git ${project}_rev)
-    fi
+    case "${rr[${project}_git]}" in
+	*"#"*)
+	    # Fetch from specified remote repo.
+	    url="${rr[${project}_git]%#*}"
+	    branch="${rr[${project}_git]#*#}"
+	    ;;
+	"baseline")
+	    # Fetch from remote repo specified in the baseline.
+	    url=$(get_baseline_git ${project}_url)
+	    branch=$(get_baseline_git ${project}_rev)
+	    ;;
+	*)
+	    # Use revision in the existing local repo.
+	    # Most likely it is "HEAD" in precommit testing.
+	    url=""
+	    branch="${rr[${project}_git]}"
+	    ;;
+    esac
 
     # Allow manifest override for $url
     url="${rr[${project}_url]-$url}"
@@ -175,27 +185,36 @@ clone_repo ()
     branch="${rr[${project}_rev]-$branch}"
 
     if [ x"${rr[mode]}" = x"bisect" ]; then
-	# Cleanup current checkout in bisect mode.
+	# FIXME: We should be able to merge this into below "local repo" clause.
+	# In bisect builds we should specify HEAD for the bisected project,
+	# and it should be OK to fetch "baseline" for all other components.
+	git_clean "$project"
+    elif [ "$url" = "" ]; then
+	# In local mode -- clean the project directory.
 	git_clean "$project"
+	# Don't use git_checkout(), which prefers remote resolution of refs.
+	git -C "$project" checkout --detach "$branch"
     else
-	clone_or_update_repo_no_checkout "$project" "$url" auto "" origin \
-					 > /dev/null
-	git_checkout "$project" "$branch" origin
+	clone_or_update_repo "$project" "$branch" "$url" > /dev/null
     fi
 
     local cur_rev debug_project_date
     cur_rev=$(git -C $project rev-parse HEAD)
     debug_project_date=$(git -C $project show --no-patch --pretty="%ct # %cr" HEAD)
 
-    # Store git info info in the manifest.
-    cat <<EOF | manifest_out
+    # Store git info in the manifest and git data into artifacts.
+    # Git data in artifacts is then used by subsequent builds to fetch
+    # baseline commits.
+    if [ "$url" != "" ]; then
+	cat <<EOF | manifest_out
 rr[${project}_url]=$url
+EOF
+	echo "$url" | set_current_git ${project}_url
+    fi
+    cat <<EOF | manifest_out
 rr[${project}_rev]=$cur_rev
 rr[debug_${project}_date]=$debug_project_date
 EOF
-    # Store baseline git data.  This is then used by
-    # subsequent builds to fetch baseline commits.
-    echo "$url" | set_current_git ${project}_url
     echo "$cur_rev" | set_current_git ${project}_rev
     )
 }
@@ -206,7 +225,7 @@ prepare_abe ()
     (
     set -euf -o pipefail
 
-    clone_or_update_repo abe ${rr[abe_branch]}  ${rr[abe_repo]} > /dev/null
+    clone_or_update_repo abe ${rr[abe_branch]} ${rr[abe_repo]} > /dev/null
 
     # We use our modified version of GCC's comparison script
     clone_or_update_repo gcc-compare-results master \
@@ -379,43 +398,41 @@ build_abe ()
 	mkdir "$run_step_top_artifacts/sumfiles" \
 	      "$run_step_top_artifacts/00-sumfiles"
 
-	# Construct $xfails from
-	# - failed tests from base-artifacts/sumfiles/ and
-	# - flaky tests seen in history of base-artifacts/sumfiles/flaky.xfail.
-	# These are passed to ABE to speed-up test convergence
-	# and to .sum comparison in tcwg_gnu-build.sh:no_regression_p().
-	local xfails="$run_step_top_artifacts/sumfiles/xfails.xfail"
-
+	# Construct $baseline_fails from base-artifacts/sumfiles/.
+	# These and $flaky_tests are passed to ABE to speed-up test convergence
+	# and then to .sum comparison in tcwg_gnu-build.sh:no_regression_p().
+	local baseline_fails="$run_step_artifacts/baseline.xfail"
 	if [ -d base-artifacts/sumfiles ]; then
 	    gcc-compare-results/contrib/testsuite-management/validate_failures.py \
 		--build_dir=base-artifacts/sumfiles --produce_manifest \
-		--manifest "$xfails"
+		--manifest "$baseline_fails"
 	else
-	    touch "$xfails"
+	    touch "$baseline_fails"
 	fi
 
 	# Fetch flaky tests from base-artifacts history.
-	local history_flaky history_root=""
+	local history_flaky history_root="" flaky_tests
+	flaky_tests=$(mktemp)
 	while read history_flaky; do
 	    if [ "$history_root" = "" ]; then
 		history_root="$history_flaky"
 		continue
 	    fi
 
-	    (echo; cat "$history_flaky") >> "$xfails"
+	    (echo; cat "$history_flaky") >> "$flaky_tests"
 	done < <(get_git_history 0 base-artifacts sumfiles/flaky.xfail)
 	rm -rf "$history_root"
 
-	# Record flaky tests in current run in $flaky_tests; these will then be
-	# included in subsequent runs as $xfails.
+	# Record flaky tests in current run in $new_flaky.
 	local new_flaky="$run_step_artifacts/flaky.xfail"
-	cat > "$new_flaky" <<EOF
-# From ${BUILD_URL-$(pwd)}:
-EOF
+
+	# ABE will re-write the $new_flaky file to contain entries only
+	# for the new flaky tests detected in this run.
+	cp "$flaky_tests" "$new_flaky"
 
 	rerun_failed_tests=("--rerun-failed-tests"
 			    "--gcc-compare-results" "$PWD/gcc-compare-results"
-			    "--expected-failures" "$xfails"
+			    "--expected-failures" "$baseline_fails"
 			    "--flaky-failures" "$new_flaky")
     fi
 
@@ -647,23 +664,50 @@ EOF
 	    sumfiles="$run_step_top_artifacts/00-sumfiles"
 	fi
 
-	# Post-process flaky tests: add "flaky" attibute, and set the entry
-	# to expire in 1 month.  The upside of expiration is that we will
-	# have an up-to-date list of flaky tests, which we can address.
-	# The downside is that we will spend a bit more CPU cycles
-	# to re-detect flaky tests.
-	local expire
-	expire=$(date -d "now+1 month" +%Y%m%d)
-	expire="expire=$expire"
-
-	cp "$new_flaky" "$new_flaky.bak"
-	sed -i -e "s/^\([A-Z]\+: \)/flaky,$expire \| \1/" "$new_flaky"
-	if ! diff -q "$new_flaky" "$new_flaky.bak" > /dev/null; then
+	local xfails="$sumfiles/xfails.xfail"
+
+	if [ -s "$new_flaky" ]; then
+	    # Mark new flaky entries to expire in 1 month.
+	    # The upside of expiration is that we will
+	    # have an up-to-date list of flaky tests, which we can address.
+	    # The downside is that we will spend a bit more CPU cycles
+	    # to re-detect flaky tests.
+	    local expire
+	    expire=$(date -d "now+1 month" +%Y%m%d)
+	    expire="expire=$expire"
+	    sed -i -e "s#^flaky | #flaky,$expire | #" "$new_flaky"
+
 	    # Move flaky fails to the sumfiles so that they will be
 	    # fetched by next run's get_git_history().
-	    mv "$new_flaky" "$sumfiles/flaky.xfail"
+	    echo "# From ${BUILD_URL-$(pwd)}:" > "$sumfiles/flaky.xfail"
+	    cat "$new_flaky" >> "$sumfiles/flaky.xfail"
+
+	    # Add newly-detected flaky tests to the xfails to be used in
+	    # tcwg_gnu-build.sh:no_regression_p().
+	    # Strictly speaking, this is not necessary, since all tests
+	    # detected as flaky in the current run will show up as PASSed
+	    # test in the final merged .sum files.
+	    cat "$sumfiles/flaky.xfail" >> "$xfails"
 	fi
-	rm "$new_flaky.bak"
+
+	# Add known flaky tests and baseline_fails to the xfails to be used in
+	# tcwg_gnu-build.sh:no_regression_p().
+	#
+	# Note #1: We generate $baseline_fails without regard for flaky
+	# tests.  Therefore, validate_failures in no_regression_p() will
+	# see same tests with and without flaky attributes.
+	# Validate_failure uses python sets to store results, so the first
+	# entry wins.  Therefore, we need to put lists of flaky tests before
+	# lists of expected fails -- $baseline_fails.
+	#
+	# Note #2: Order of expired and non-expired flaky tests is less of
+	# an issue because expired entries are discarded by validate_failures
+	# before adding them to the ResultSet.  Still, for uniformity, we
+	# put fresh flaky entries before older ones.
+	cat "$flaky_tests" >> "$xfails"
+	cat "$baseline_fails" >> "$xfails"
+
+	rm "$new_flaky" "$baseline_fails"
     fi
 
     # Wait for logs to compress.
@@ -843,14 +887,16 @@ create_trigger_files ()
 
     if [ ${#changed_components[@]} -gt 1 ] \
 	   || { [ x"${rr[mode]}" = x"bisect" ] \
-		    && ! [ $score -lt 0 ] 2>/dev/null; }; then
+		    && ! [ "$score" -lt 0 ] 2>/dev/null; }; then
 	# If we have several changed components, then trigger individual builds
 	# for these components to narrow down the problem to a single component.
 	# Also generate trigger-build-* files as a favor to
 	# round-robin-bisect.sh script to distinguish "bad" from "skip" builds
 	# and to provide templates for triggering follow-up builds.
-	# Note that the negative "! [ $score -lt 0 ]" condition is to handle
-	# non-numeric scores like "all" and "boot".
+	# Note that we can't use "[ "$score" -ge 0 ]" condition above and below
+	# because it will give wrong answer for non-numeric scores like "all"
+	# and "boot".  "-lt" comparison, however, will produce correct answer,
+	# albeit, with an occasional harmless error message.
 	local -a update_components
 
 	while read -a update_components; do
@@ -860,6 +906,8 @@ create_trigger_files ()
 	    for c in ${rr[components]}; do
 		if echo "${update_components[@]}" | tr ' ' '\n' \
 			| grep "^$c\$" >/dev/null; then
+		    assert_with_msg "Should never happen for precommit builds" \
+				    [ "${rr[${c}_git]}" != "HEAD" ]
 		    echo "${c}_git=${rr[${c}_git]}"
 		fi
 	    done > $trigger_dest/trigger-build-$update_components2
@@ -875,14 +923,9 @@ create_trigger_files ()
 	    fi
 	done < <(${rr[breakup_changed_components]})
     elif [ ${#changed_components[@]} = 1 ] \
-	     && ! [ $score -lt 0 ] 2>/dev/null; then
+	     && ! [ "$score" -lt 0 ] 2>/dev/null; then
 	local single_component="${changed_components[0]}"
 
-	if [ x"${rr[update_baseline]}" = x"ignore" ]; then
-	    # Don't trigger bisect for "ignore" builds.
-	    return
-	fi
-
 	# Trigger bisect failures in a single changed component in all steps
 	# with a positive result.  If $score is less-than 0, then
 	# the regression is not very interesting, so don't bisect.
@@ -902,7 +945,107 @@ EOF
 }
 
 
-# Check if current build regressed compared to the baseline
+# Check if current build regressed compared to the baseline.
+#
+# As inputs we have:
+# $score -- last line of artifacts/results
+# $res   -- exit code of no_regression_p()
+#
+# Additionally, ${rr[update_baseline]} and ${rr[mode]} affect how the rest
+# of the build will proceed -- via create_trigger_files().
+#
+# Decision matrix:
+#
+# 1. score >= 0 && res == 0: return 0
+#    OK; new results good for baseline;
+#    update baseline if != ignore (e.g., not bisect test or precommit)
+#      - round-robin-notify.sh compares results in artifacts/ vs
+#        base-artifacts#HEAD^
+#    don't update baseline if == ignore (e.g., bisect test or precommit)
+#      - round-robin-notify.sh compares results in artifacts/ vs
+#        base-artifacts#HEAD
+#    send precommit feedback if precommit testing is active;
+#    (see [R] below) send regression report if notify==onregression
+#    push baseline if != ignore (e.g., not precommit build);
+#    trigger pre-commit testing if SCM build;
+#
+# 2. score >= 0 && res == E: return E
+#    sporadic fail; wait for next build; don't trigger anything;
+#    don't update baseline even with force or init;
+#    Can happen in, e.g., tcwg_native_build if git server fails while
+#    cloning repo for one of the components;
+#
+# 3. score >= 0 && res != {0,E}: return I (or 0 if update_baseline==force)
+#    regression (or expected regression if update_baseline==force);
+#    if update_baseline == onsuccess -- trigger reduce or bisect;
+#    [R] if update_baseline == force -- succeed, return 0, treat as (1).
+#    if update_baseline == ignore -- nothing
+#      - if mode==bisect, then create trigger-* files as favor for
+#        round-robin-bisect.sh script.
+#      - if pre-commit testing is active, then send notifications
+#        similar to "update_baseline==force".  See [P] below.
+#    case
+#
+# 4. score <  0 && res == 0: return E
+#    should not happen; baseline is bad and current build is no better;
+#    don't update the baseline or reduce or bisect; send error-mail
+#
+# 5. score <  0 && res == E: return E
+#    sporadic fail; wait for next build; don't trigger anything;
+#    don't update baseline even with force or init.
+#
+# 6. score <  0 && res != {0,E}: return I
+#    uninteresting fail; trigger reduce, but no bisect
+#    don't update baseline even with force or init.
+#
+# 7. score == -E: return E
+#    special case indicating a sporadic failure; same as (5) but without
+#    calling no_regression_p() to get "res".
+#
+# "E" -- EXTERNAL_FAILURE; "I" -- INTERNAL_FAILURE
+#
+# Notes:
+# - If score < 0 then we never update baseline -- even with force or init.
+#   This allows us to always have a reasonable "interesting" baseline,
+#   which is necessary for pre-commit testing.  E.g., we don't want to
+#   pre-commit test upstream patches against a baseline that doesn't build.
+#
+# - If we return E -- exit, do nothing, wait for the next timed trigger.
+#   Hopefully by that time infra fixes itself.
+#   In the .yaml files we have a choice on how to implement this:
+#   a. Mark the build as FAILURE, which will prevent any subsequent steps
+#      from running.  Pros: simple, doesn't require conditional-steps;
+#      cons: the build is shown as "red" failure.
+#   b. Mark the build as UNSTABLE, and then condition all subsequent steps
+#      to run only for current-status==SUCCESS, which is what we are doing
+#      now.  Pros: the build is shown as "yellow"; cons: all subsequent
+#      steps need to be wrapped in conditional-step.
+#
+# - [P] It's not clear how to send notifications for failed pre-commit
+#   builds.  At the moment we run trigger-followup-builds for failed
+#   builds and exit, thus avoiding notify-and-push step for failed builds.
+#   It seems we need to
+#   a. Split notify-and-push into "notify" and "push"
+#   b. Move "notify" before trigger-followup-builds and leave "push"
+#      after trigger-followup-builds.
+#   c. We need to make sure "notify" does not send anything when it is
+#      running after a failed build.
+#   d. Trigger of pre-commit testing needs to happen only after "push",
+#      so we know that we have a good recent baseline.
+#
+# - If we return E during precommit testing -- we should, ideally,
+#   remove the patch from "triggered" bundle, so that we will process
+#   it again later.
+#
+# - If we return N != {0, I, E}, then the exit was due to script error,
+#   so send error-mail and follow case of returning "E".
+#
+# - If we returned "0", then build should be marked as SUCCESS.
+#
+# - If we returned "I", then build should be marked FAILURE
+#
+# - If we returned "E" or anything else, then build should be marked
+#   as UNSTABLE.
 check_regression ()
 {
     (
@@ -911,6 +1054,8 @@ check_regression ()
     local score
     score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)
 
+    # FIXME: notify-round-robin.sh should get information on status of
+    # the build via exit code -- 0, I, E -- not via check-regression-status.txt.
     mkdir -p ${rr[top_artifacts]}/notify
     if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then
 	echo "ERROR: We have encountered some infrastructure problem (e.g.,"
@@ -920,7 +1065,7 @@ check_regression ()
 	# Exit now and don't update baseline artifacts.
 	# By not creating trigger-build-* files, we signal
 	# round-robin-bisect.sh to skip this build/revision.
-	exit $EXTERNAL_FAIL
+	return $EXTERNAL_FAIL
     fi
 
     local res
@@ -945,11 +1090,22 @@ check_regression ()
     echo "$res" > ${rr[top_artifacts]}/notify/check-regression-status.txt
 
     if [ $res = 0 ]; then
-	# All good, no regression
-	return
+	if [ "$score" -lt 0 ] 2>/dev/null; then
+	    # Case (4) in the comment above.
+	    if [ -d ${rr[top_artifacts]}/jenkins ]; then
+		echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
+                     > artifacts/jenkins/error-mail-recipients.txt
+                echo -e "${BUILD_URL-}\nERROR: case (4) in check_regression" \
+                     >> artifacts/jenkins/error-mail-body.txt
+	    fi
+	    return $EXTERNAL_FAIL
+	else
+	    # All good, no regression
+	    return 0
+	fi
     elif [ $res = $EXTERNAL_FAIL ]; then
 	# Comparison failed to produce a meaningful result
-	exit $EXTERNAL_FAIL
+	return $EXTERNAL_FAIL
     fi
 
     assert_with_msg "no_regression_p should succeed in init baseline mode" \
@@ -957,7 +1113,8 @@ check_regression ()
 
     # We've got a regression.  Generate trigger-* files.
     local trigger_dest
-    if [ x"${rr[update_baseline]}" != x"force" ]; then
+    if [ "${rr[update_baseline]}" = "onsuccess" ] \
+	   || [ "${rr[mode]}" = "bisect" ]; then
 	# We are seeing a failure, so instead of updating baseline start
 	# reducing/bisecting the failure.  Create trigger-* files at top level
 	# where jenkins expects them -- and trigger the followup builds.
@@ -971,10 +1128,12 @@ check_regression ()
 
     create_trigger_files "$trigger_dest" "$score"
 
-    if [ x"${rr[update_baseline]}" != x"force" ]; then
-	echo "Detected a regression!"
-	false
+    if [ "${rr[update_baseline]}" = "force" ]; then
+	return 0
     fi
+
+    echo "Detected a regression!"
+    return $INTERNAL_FAIL
     )
 }
 
@@ -1108,7 +1267,7 @@ update_baseline ()
     set -euf -o pipefail
 
     if [ x"${rr[update_baseline]}" = x"ignore" ]; then
-	return
+	return 0
     fi
 
     trim_base_artifacts