jenkins-logs.sh: Upload logs during EXIT trap handler.

We upload the logs and results at the very end of the script, but in
some cases we exit earlier (for instance when the first cross-compiler
build failed). In such cases we wouldn't upload the logs, relying on
the Jenkins artifacts to debug the problem.

This patch moves the upload code to a dedicated function, which
is called by the EXIT trap handler.

We must be careful to also remove the locks if needed, so we create an
EXIT trap handler to perform both actions: upload the
logs/results/manifest and clean the locks.

This also fixes a potential problem where a build failure would not
upload the manifest (in case the failing build went far enough to
generate the manifest).

Change-Id: I45d8db3993a986b529756454e5c022105f4c4deb

1 files changed, 76 insertions, 80 deletions

diff --git a/jenkins.sh b/jenkins.sh
index 9f0e5328..2f7d99fa 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -39,6 +39,74 @@ read_var()
    grep "^${var}=" "${artifact_list}" | cut -d = -f 2-
 }
 
+upload_to_logserver()
+{
+    if test x"${logserver}" != x""; then
+	logs_dir=$(mktemp -d)
+	sums=""
+	# We use the global 'status' variable
+
+	# Copy over the build logs
+	makelog_list=($(read_var ${user_workspace}/build-artifacts.txt 'log_make_[^=]*'))
+	if test x"${canadian}" = x"true"; then
+            makelog_list+=($(read_var ${user_workspace}/canadian-build1-artifacts.txt 'log_make_[^=]*'))
+            makelog_list+=($(read_var ${user_workspace}/canadian-build2-artifacts.txt 'log_make_[^=]*'))
+	fi
+	for makelog in "${makelog_list[@]}"; do
+            cp ${makelog} ${logs_dir}/ || status=1
+	done
+
+	# Copy the manifest, if present. Print an error message if not.
+	manifest="$(read_var ${user_workspace}/build-artifacts.txt manifest)"
+	if test x"${manifest}" = x; then
+	    echo "ERROR: No manifest file, build probably failed!"
+	else
+	    scp $logopts ${manifest} ${logserver}:${basedir}/${dir}/ || status=1
+	fi
+
+	# Copy stdout and stderr output from abe.
+	cp build.out build.err ${logs_dir}/ || status=1
+	if $runtests; then
+	    cp check.out check.err ${logs_dir}/ || status=1
+	fi
+
+	# Find all the test result files.
+	if test x"${runtests}" = x"true"; then
+	    sums="$(read_var ${user_workspace}/check-artifacts.txt 'dj_sum_[^=]*')"
+	fi
+
+	# Copy the test results, if any
+	if test x"${sums}" != x; then
+	    test_logs=""
+	    for s in ${sums}; do
+		test_logs="$test_logs ${s%.sum}.log"
+	    done
+
+	    cp ${sums} ${test_logs} ${logs_dir}/ || status=1
+
+	    # Copy over the logs from make check, which we need to find testcase errors.
+            for check in $(read_var ${user_workspace}/check-artifacts.txt 'log_check_[^=]*'); do
+		cp ${check} ${logs_dir}/ || status=1
+            done
+	fi
+
+	xz ${logs_dir}/* || status=1
+	scp $logopts ${logs_dir}/* ${logserver}:${basedir}/${dir}/ || status=1
+	echo "Uploaded build logs and test results to ${logserver}:$logport:${basedir}/${dir}/ with status: $status"
+
+	rm -rf ${logs_dir} || status=1
+    fi
+}
+
+exit_handler()
+{
+    upload_to_logserver
+    if $clean_lock_on_exit; then
+	ssh $logopts $logserver rm -f $basedir/$dir.lock
+    fi
+    exit $status
+}
+
 if test $# -lt 1; then
     echo "ERROR: No options for build!"
     usage
@@ -99,6 +167,9 @@ excludecheck_opt=""
 # --logname gcc-<sha1>
 rebuild=true
 
+# Whether to remove locks upon exit
+clean_lock_on_exit=false
+
 orig_parameters="$@"
 
 OPTS="$(getopt -o s:g:w:o:l:rt:b:h -l override:,gcc-branch:,snapshots:,gitrepo:,abedir:,workspace:,options:,logserver:,logname:,languages:,runtests,target:,testcontainer:,bootstrap,help,excludecheck:,norebuild,extraconfig:,send-results-to: -- "$@")"
@@ -159,8 +230,11 @@ logport="$(echo $logserver_spec | cut -s -d: -f 2)"
 
 logopts="${logport:+-o Port=$logport}"
 
+trap "exit_handler" EXIT
+
 # Check status of logs on $logserver and rebuild if appropriate.
 [ x"$logserver" != x"" ] && ssh $logopts $logserver mkdir -p $(dirname $basedir/$dir)
+
 # Loop and wait until we successfully grabbed the lock.  The while condition is,
 # effectively, "while true;" with a provision to skip if $logserver is not set.
 while [ x"$logserver" != x"" ]; do
@@ -217,7 +291,7 @@ while [ x"$logserver" != x"" ]; do
 
     if [ x"$(ssh $logopts $logserver cat $basedir/$dir.lock)" \
 	= x"$(hostname)-$$-$BUILD_URL" ]; then
-	trap "ssh $logopts $logserver rm -f $basedir/$dir.lock" 0 1 2 3 5 9 13 15
+	clean_lock_on_exit=true
 	# Hurray!  Break from the loop and go ahead with the build!
 	break
     fi
@@ -411,45 +485,6 @@ fi
 
 echo "Build parameters: $orig_parameters"
 
-manifest="$(read_var ${user_workspace}/build-artifacts.txt manifest)"
-if test x"${manifest}" = x; then
-    echo "ERROR: No manifest file, build probably failed!"
-fi
-
-# This becomes the path on the remote file server    
-if test x"${logserver}" != x""; then
-    # Re-eval $dir as we now have full range of variables available.
-    eval dir="$logname"
-    ssh $logopts ${logserver} mkdir -p ${basedir}/${dir}
-    if test x"${manifest}" != x; then
-	scp $logopts ${manifest} ${logserver}:${basedir}/${dir}/
-    fi
-
-# If 'make check' works, we get .sum files with the results. These we
-# convert to JUNIT format, which is what Jenkins wants it's results
-# in. We then cat them to the console, as that seems to be the only
-# way to get the results into Jenkins.
-#if test x"${sums}" != x; then
-#    for i in ${sums}; do
-#	name="$(basename $i)"
-#	${abe_dir}/sum2junit.sh $i $user_workspace/${name}.junit
-#	cp $i ${user_workspace}/results/${dir}
-#    done
-#    junits="$(find ${user_workspace} -name *.junit)"
-#    if test x"${junits}" = x; then
-#	echo "Bummer, no junit files yet..."
-#    fi
-#else
-#    echo "Bummer, no test results yet..."
-#fi
-#touch $user_workspace/*.junit
-fi
-
-# Find all the test result files.
-if test x"${runtests}" = x"true"; then
-    sums="$(read_var ${user_workspace}/check-artifacts.txt 'dj_sum_[^=]*')"
-fi
-
 # Canadian Crosses are a win32 hosted cross toolchain built on a Linux
 # machine.
 if test x"${canadian}" = x"true"; then
@@ -464,45 +499,7 @@ if test x"${canadian}" = x"true"; then
 fi
 
 # This setups all the files needed by tcwgweb
-if test x"${logserver}" != x"" && test x"${sums}" != x -o x"${runtests}" != x"true"; then
-    logs_dir=$(mktemp -d)
-
-    if test x"${sums}" != x; then
-	test_logs=""
-	for s in ${sums}; do
-	    test_logs="$test_logs ${s%.sum}.log"
-	done
-
-	cp ${sums} ${test_logs} ${logs_dir}/ || status=1
-	
-	# Copy over the logs from make check, which we need to find testcase errors.
-        for check in $(read_var ${user_workspace}/check-artifacts.txt 'log_check_[^=]*'); do
-	    cp ${check} ${logs_dir}/ || status=1
-        done
-    fi
-
-    # Copy over the build logs
-    makelog_list=($(read_var ${user_workspace}/build-artifacts.txt 'log_make_[^=]*'))
-    if test x"${canadian}" = x"true"; then
-        makelog_list+=($(read_var ${user_workspace}/canadian-build1-artifacts.txt 'log_make_[^=]*'))
-        makelog_list+=($(read_var ${user_workspace}/canadian-build2-artifacts.txt 'log_make_[^=]*'))
-    fi
-    for makelog in "${makelog_list[@]}"; do
-        cp ${makelog} ${logs_dir}/ || status=1
-    done
-
-    # Copy stdout and stderr output from abe.
-    cp build.out build.err ${logs_dir}/ || status=1
-    if $runtests; then
-	cp check.out check.err ${logs_dir}/ || status=1
-    fi
-
-    xz ${logs_dir}/* || status=1
-    scp $logopts ${logs_dir}/* ${logserver}:${basedir}/${dir}/ || status=1
-    rm -rf ${logs_dir} || status=1
-
-    echo "Uploaded test results and build logs to ${logserver}:$logport:${basedir}/${dir}/ with status: $status"
-
+if test x"${logserver}" != x"" ; then
     if test x"${tarsrc}" = xtrue -a x"${release}" != x; then
 	allfiles="$(ls ${user_snapshots}/*${release}*.xz)"
 	srcfiles="$(echo ${allfiles} | egrep -v "arm|aarch")"
@@ -516,7 +513,6 @@ if test x"${logserver}" != x"" && test x"${sums}" != x -o x"${runtests}" != x"tr
 	scp $logopts ${binfiles} ${logserver}:/work/space/binaries/ || status=1
 	rm -f ${binfiles} || status=1
     fi
-
 fi
 
 exit $status