jenkins-helpers.sh: Improve run_step functionality.

This is partly fix, partly cleanup.  At the moment in bisection
mode artifacts/results are not populated until first step successfully
executes.  This means that for bisections of binutils, gcc, and llvm
regressions the results file is empty if the build of these projects
fails (i.e., no successful steps at all).  This causes empty clauses
in the bisection emails due to artifacts/results file being empty.

We could have fixed this by adding
===
Starting Y build
<value of successful previous step>
===
to the top of every build step, but instead we refactor and cleanup
run_step support to handle constant-value result entries.

In addition to current "stop_on_fail" mode we add "skip_on_fail" and
"reset_on_fail" modes, which simplify error handling for most build
steps.

Change-Id: Ic2d1a15fa490ec25be741cdaf718442260c52e6d

3 files changed, 79 insertions, 73 deletions

diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index 78d44da4..b4b1b112 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -1031,6 +1031,7 @@ run_step_init ()
     run_step_count="0"
     run_step_prev_step=""
     run_step_active=false
+    run_step_status=0
     run_step_artifacts=""
 
     if [ x"$run_step_base_artifacts" != x"" ]; then
@@ -1040,11 +1041,13 @@ run_step_init ()
     fi
 
     rm -f $run_step_top_artifacts/console.log
+    rm -f $run_step_top_artifacts/results
 }
 
-# Run execution step
+# Run execution step and handle its failure as requested
 # This function manages
-#   1. step skipping -- skip steps before START_AT and after FINISH_AT
+#   1. step skipping -- skip steps before START_AT and after FINISH_AT,
+#      as well as "skip_on_fail" steps during failure.
 #   2. artifact handling -- create/clean artifact directories per step.
 #      Also, copy baseline artifacts for steps before START_AT to simulate
 #      skipped steps.
@@ -1054,9 +1057,28 @@ run_step_init ()
 #   3. logging -- dump stdout and and stderr output of step commands
 #      into per-step console.log files, and, also, into the top-level
 #      console.log file.
+#   4. result handling -- output provided success result to artifacts/results
+#      for successful steps.  Special value "x" means to let the step itself
+#      update artifacts/results.  Results are written to artifacts/results
+#      for both skipped and executed steps as long as $run_step_status doesn't
+#      indicate failure.
+#   5. in run mode "stop_on_fail" -- kill script if $run_step_status indicates
+#      failure, otherwise run the step and kill script on failure.
+#   6. in run mode "skip_on_fail" -- skip step if $run_step_status indicates
+#      a failure, otherwise run the step and set $run_step_status to the exit
+#      status of the step.  Normally, a series of "skip_on_fail" steps should
+#      be followed by "reset_on_fail" step that handles the cumulative result.
+#   7. in run mode "reset_on_fail" -- run the step regardless and set
+#      $run_step_status to the exit status of the step (thus resetting it).
+# $1: Run mode: stop_on_fail, skip_on_fail, reset_on_fail.
+# $2: Result to write to artifacts/results in the absence of failures.
 # $@: Step command and its arguments
 run_step ()
 {
+    local run_mode="$1"
+    local success_result="$2"
+    shift 2
+
     local -a step
     local pretty_step
 
@@ -1078,8 +1100,42 @@ run_step ()
     mkdir -p "$run_step_artifacts"
 
     if $run_step_active; then
-	echo "RUNNING ${step[@]}; see tail -f $run_step_artifacts/console.log"
-	eval "if $run_step_verbose; then set -x; else set +x; fi; ${step[@]}" 2>&1 | tee -a $run_step_top_artifacts/console.log > $run_step_artifacts/console.log
+	local skip=false
+	case "$run_step_status:$run_mode" in
+	    0:*) ;;
+	    *:stop_on_fail)
+		echo "STOPPING before ${step[@]} due to previous failure"
+		false
+		;;
+	    *:skip_on_fail)
+		echo "SKIPPING ${step[@]} due to previous failure"
+		skip=true
+		;;
+	    *:reset_on_fail)
+		echo "HANDLING previous failure in ${step[@]}"
+		;;
+	    *)
+		assert false
+	esac
+
+	if ! $skip; then
+	    echo "RUNNING ${step[@]}; see tail -f $run_step_artifacts/console.log"
+	    eval "if $run_step_verbose; then set -x; else set +x; fi; ${step[@]}" 2>&1 | tee -a $run_step_top_artifacts/console.log > $run_step_artifacts/console.log &
+	    wait $! || run_step_status=$?
+
+	    case "$run_step_status:$run_mode" in
+		0:*) ;;
+		*:stop_on_fail|*:reset_on_fail)
+		    echo "STOPPING at ${step[@]} due to failure"
+		    false
+		    ;;
+		*:skip_on_fail)
+		    echo "CARRYING ON after failure in ${step[@]}"
+		    ;;
+		*)
+		    assert false
+	    esac
+	fi
     elif $run_step_use_baseline; then
 	echo "COPYING BASE-ARTIFACTS for ${step[@]}"
 	local base_artifacts
@@ -1091,6 +1147,13 @@ run_step ()
 	echo "SKIPPING ${step[@]}"
     fi
 
+    if [ x"$run_step_status" = x"0" -a x"$success_result" != x"x" ]; then
+	cat >> $run_step_top_artifacts/results <<EOF
+${step[@]}:
+$success_result
+EOF
+    fi
+
     if [ x"$pretty_step" = x"$run_step_finish_at" ]; then
 	run_step_active=false
     fi
diff --git a/round-robin.sh b/round-robin.sh
index ac1ff3b3..8f4fdae2 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -53,10 +53,6 @@ reset_artifacts ()
 	      $run_step_artifacts/console.log \
 	      "${rr[top_artifacts]}/jenkins/*"
 
-    # Set success metric to worst possible (-10).
-    echo "Starting build" > ${rr[top_artifacts]}/results
-    echo "-10" >> ${rr[top_artifacts]}/results
-
     local single_branch=${rr[baseline_branch]}
 
     # Clone base-artifacts so that run_step can rsync artifacts for skipped
@@ -176,7 +172,7 @@ EOF
 
 # Build ABE component
 # $1: Component -- ABE component to build.
-build_abe_1 ()
+build_abe ()
 {
     (
     set -euf -o pipefail
@@ -243,42 +239,8 @@ build_abe_1 ()
     )
 }
 
-# Build binutils or GCC using ABE
-# $1: Component -- Binutils or stage1 GCC.
-build_abe ()
-{
-    (
-    set -euf -o pipefail
-
-    local component="$1"
-
-    # Attempt to build GCC only if binutils build (in previous step) was OK.
-    if [ x"$component" != x"binutils" -a ! -f $run_step_prev_artifacts/build-ok ]; then
-	return
-    fi
-
-    build_abe_1 $component &
-    local res=0; wait $! || res=$?
-
-    # Update artifacts on success.
-    if [ $res = 0 ]; then
-	echo "Finished build_abe $component" >> ${rr[top_artifacts]}/results
-	case "$component" in
-	    binutils) echo "-9" >> ${rr[top_artifacts]}/results ;;
-	    stage1) echo "-8" >> ${rr[top_artifacts]}/results ;;
-	    linux) echo "-7" >> ${rr[top_artifacts]}/results ;;
-	    glibc) echo "-6" >> ${rr[top_artifacts]}/results ;;
-	    stage2) echo "-5" >> ${rr[top_artifacts]}/results ;;
-	    *) assert false ;;
-	esac
-
-	touch $run_step_artifacts/build-ok
-    fi
-    )
-}
-
 # Build LLVM
-build_llvm_1 ()
+build_llvm ()
 {
     (
     set -euf -o pipefail
@@ -329,24 +291,6 @@ EOF
     )
 }
 
-# Build LLVM
-build_llvm ()
-{
-    (
-    set -euf -o pipefail
-
-    build_llvm_1 &
-    local res=0; wait $! || res=$?
-
-    # Update artifacts on success.
-    if [ $res = 0 ]; then
-	echo "Finished build_llvm" >> ${rr[top_artifacts]}/results
-	echo "-1" >> ${rr[top_artifacts]}/results
-	touch $run_step_artifacts/build-ok
-    fi
-    )
-}
-
 # Check if current build regressed compared to the baseline
 # (unless ${rr[reset_baseline]} is set).
 check_regression ()
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index a8188fce..6eda2248 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -182,8 +182,7 @@ count_linux_objs ()
     (
     set -euf -o pipefail
 
-    # Run only if compiler was successfully built in the previous step.
-    if [ -f $run_step_prev_artifacts/build-ok ]; then
+    if true; then
 	build_linux &
 	local res=0; wait $! || res=$?
 
@@ -244,20 +243,20 @@ no_regression_p ()
     )
 }
 
-run_step reset_artifacts
+run_step stop_on_fail -10 reset_artifacts
 case "${rr[toolchain]}" in
     gnu)
-	run_step prepare_abe
-	run_step build_abe binutils
-	run_step build_abe stage1
+	run_step stop_on_fail x prepare_abe
+	run_step skip_on_fail -9 build_abe binutils
+	run_step skip_on_fail -8 build_abe stage1
 	;;
     llvm)
-	run_step build_llvm
+	run_step skip_on_fail -1 build_llvm
 	;;
 esac
-run_step count_linux_objs
-run_step check_regression
-run_step update_baseline
-run_step push_baseline
+run_step skip_on_fail x count_linux_objs
+run_step reset_on_fail x check_regression
+run_step stop_on_fail x update_baseline
+run_step stop_on_fail x push_baseline
 
 trap "" EXIT