From 473a300827fed0abff85a3c336c66825f612ab59 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Wed, 1 Sep 2021 07:55:07 +0000
Subject: tcwg_gnu-build.sh: Conserve disk space in artifacts

... by removing top-level console.log, which is too huge to be useful,
and by compressing per-step console.log files.

Also don't generate results with compare_tests when we have only
partial results.  It outputs a lot of missing PASSes and other
expected differences due to a partial run.

Change-Id: I3a29698fe3f6dccc844c0bf3b6034b013533377a
---
 jenkins-helpers.sh |  9 +++++----
 tcwg_gnu-build.sh  | 50 ++++++++++++++++++++++++++------------------------
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index 26151090..ff3f2035 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -1243,6 +1243,7 @@ finishing at step \"$run_step_finish_at\""
     run_step_top_artifacts=$(cd "$run_step_top_artifacts"; pwd)
 
     rm -f $run_step_top_artifacts/console.log
+    rm -f $run_step_top_artifacts/console.log.xz
     rm -f $run_step_top_artifacts/results
 }
 
@@ -1256,8 +1257,7 @@ finishing at step \"$run_step_finish_at\""
 #      Step commands have $run_step_artifacts pointing to artifact directory
 #      for current step.
 #   3. logging -- dump stdout and and stderr output of step commands
-#      into per-step console.log files, and, also, into the top-level
-#      console.log file.
+#      into per-step console.log files
 #   4. result handling -- output provided success result to artifacts/results
 #      for successful steps.  Special value "x" means to let the step itself
 #      update artifacts/results.  Results are written to artifacts/results
@@ -1339,7 +1339,7 @@ run_step ()
 	    local log_url=""
 	    if [ -v BUILD_URL ]; then
 	        # Link to jenkins, valid once the job has finished
-	        log_url="(${BUILD_URL}artifact/artifacts/$full_step_name/console.log)"
+	        log_url="(${BUILD_URL}artifact/artifacts/$full_step_name/console.log.xz)"
 	    fi
 
 	    rm -rf "$run_step_artifacts"
@@ -1347,8 +1347,9 @@ run_step ()
 
 	    echo "RUNNING ${step[*]}; see tail -f $run_step_artifacts/console.log" $log_url
 	    run_step_status=0
-	    eval "if $run_step_verbose; then set -x; else set +x; fi; ${step[*]}" 2>&1 | ts -s "%T" | tee -a $run_step_top_artifacts/console.log > $run_step_artifacts/console.log &
+	    eval "if $run_step_verbose; then set -x; else set +x; fi; ${step[*]}" 2>&1 | ts -s "%T" > $run_step_artifacts/console.log &
 	    wait $! || run_step_status=$?
+	    xz $run_step_artifacts/console.log
 
 	    case "$run_step_status:$run_mode" in
 		0:*) ;;
diff --git a/tcwg_gnu-build.sh b/tcwg_gnu-build.sh
index 588a9c5f..892a2662 100755
--- a/tcwg_gnu-build.sh
+++ b/tcwg_gnu-build.sh
@@ -132,18 +132,6 @@ no_regression_p ()
     # We use our modified version of GCC's comparison script
     clone_or_update_repo gcc-compare-results master https://git.linaro.org/toolchain/gcc-compare-results.git
 
-    # (defined by init_step in jenkins-helpers)
-    # shellcheck disable=SC2154
-    gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
-				      $sumfiles_base $sumfiles_new \
-	| tee $run_step_artifacts/results.compare1 &
-    res=0 && wait $! || res=$?
-
-    local xfail="gcc-compare-results/contrib/testsuite-management/flaky"
-    if [ -f "$xfail/${rr[ci_project]}-${rr[ci_config]}.xfail" ]; then
-	xfail="$xfail/${rr[ci_project]}-${rr[ci_config]}"
-    fi
-
     local ignore_ERRORs_opt=""
     if [ ${#runtestflags[@]} != 0 ] \
 	   || { [ x"${rr[mode]}" = x"baseline" ] \
@@ -156,27 +144,41 @@ no_regression_p ()
 	# (rr[mode]=baseline, rr[update_baseline]=push) so that ERRORs
 	# don't creep into automatically-generated flaky XFAILs.
 	ignore_ERRORs_opt="--ignore_ERRORs"
+	echo "NOT COMPARING PARTIAL RESULTS" \
+	     > $run_step_artifacts/results.compare2
+    else
+	# Compare results using compare_tests only when we compare complete
+	# testsuites.
+	# (defined by init_step in jenkins-helpers)
+	# shellcheck disable=SC2154
+	gcc-compare-results/compare_tests -compr none -pass-thresh 0.9 \
+					  $sumfiles_base $sumfiles_new \
+	    | tee $run_step_artifacts/results.compare2 &
+	res=0 && wait $! || res=$?
+    fi
+
+    local xfail="gcc-compare-results/contrib/testsuite-management/flaky"
+    if [ -f "$xfail/${rr[ci_project]}-${rr[ci_config]}.xfail" ]; then
+	xfail="$xfail/${rr[ci_project]}-${rr[ci_config]}"
     fi
 
     gcc-compare-results/contrib/testsuite-management/validate_failures.py \
 	--manifest=$xfail.xfail --clean_build=$sumfiles_base \
 	--build_dir=$sumfiles_new $ignore_ERRORs_opt \
-	| tee $run_step_artifacts/results.compare2 &
+	| tee $run_step_artifacts/results.compare &
     res=0 && wait $! || res=$?
 
     if [ $res != 0 ]; then
 	local reg_lines
-	for i in 1 2; do
-	    reg_lines=$(cat $run_step_artifacts/results.compare$i | wc -l)
-	    reg_lines=$(($reg_lines-100))
-	    cat $run_step_artifacts/results.compare$i | sed -e "s/^/# /" \
-		| (head -n100; cat >/dev/null) \
-		      > $run_step_artifacts/results.regressions
-	    if [ $reg_lines -gt 0 ]; then
-		echo "# ... and $reg_lines more entries" \
-		     >> $run_step_artifacts/results.regressions
-	    fi
-	done
+	reg_lines=$(cat $run_step_artifacts/results.compare | wc -l)
+	reg_lines=$(($reg_lines-100))
+	cat $run_step_artifacts/results.compare | sed -e "s/^/# /" \
+	    | (head -n100; cat >/dev/null) \
+		  > $run_step_artifacts/results.regressions
+	if [ $reg_lines -gt 0 ]; then
+	    echo "# ... and $reg_lines more entries" \
+		 >> $run_step_artifacts/results.regressions
+	fi
 
 	if [ $res = 2 ]; then
 	    # Result comparison found regressions (exit code 2)
-- 
cgit v1.2.3