summaryrefslogtreecommitdiff
path: root/jenkins.sh
blob: bb3d7e0f1fa8b5e636ede1094a16da5f740bd1b5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
#!/bin/bash
# 
#   Copyright (C) 2013, 2014, 2015, 2016 Linaro, Inc
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
# 
set -e
set -o pipefail

# Improve debug logs
PRGNAME=$(basename $0)
PS4='+ $PRGNAME: ${FUNCNAME+"$FUNCNAME : "}$LINENO: '

usage()
{
    ret=1
    [ x"$1" != x ] && ret=$1
    cat << EOF
  jenkins.sh [--help] [-s snapshot dir] [g git reference dir] [--abedir path] [-w workspace]
EOF
    exit $ret
}

read_var()
{
   local artifact_list=$1
   local var=$2
   if [ -f "${artifact_list}" ]; then
       grep "^${var}=" "${artifact_list}" | cut -d = -f 2-
   else
       echo ""
   fi
}

upload_to_logserver()
{
    if test x"${logserver}" != x""; then
	logs_dir=$(mktemp -d)
	sums=""
	# We use the global 'status' variable

	ssh $logopts $logserver mkdir -p $basedir/$dir || status=1

	# Copy over the build logs
	mapfile -t makelog_list < <(\
    read_var ${user_workspace}/build-artifacts.txt 'log_make_[^=]*')
	# shellcheck disable=SC2154
	if test x"${canadian}" = x"true"; then
            mapfile -t -O "${#makelog_list[@]}" makelog_list < <(\
              read_var ${user_workspace}/canadian-build1-artifacts.txt 'log_make_[^=]*')
            mapfile -t -O "${#makelog_list[@]}" makelog_list < <(\
              read_var ${user_workspace}/canadian-build2-artifacts.txt 'log_make_[^=]*')
	fi
	for makelog in "${makelog_list[@]}"; do
            cp ${makelog} ${logs_dir}/ || status=1
	done

	# Copy the manifest, if present. Print an error message if not.
	manifest="$(read_var ${user_workspace}/build-artifacts.txt manifest)"
	if test x"${manifest}" = x; then
	    echo "ERROR: No manifest file, build probably failed!"
	else
	    scp $logopts ${manifest} ${logserver}:${basedir}/${dir}/ || status=1
	fi

	# Copy stdout and stderr output from abe.
	cp build.out build.err ${logs_dir}/ || status=1
	if $runtests; then
	    cp check.out check.err ${logs_dir}/ || status=1
	fi

	# Find all the test result files.
	if test x"${runtests}" = x"true"; then
	    # Check is check-artifacts.txt is not empty otherwise the
	    # process gets stuck waiting for stdin
	    if test -s ${user_workspace}/check-artifacts.txt ; then
		ret=0
		sums="$(read_var ${user_workspace}/check-artifacts.txt 'dj_sum_[^=]*')" || ret=$?
		if test $ret -gt 0; then
		    echo "WARNING: couldn't find any .sum file in the artifacts"
		    cat ${user_workspace}/check-artifacts.txt
		    status=1
		fi
	    else
		status=1
	    fi
	fi

	# Copy the test results, if any
	if test x"${sums}" != x; then
	    test_logs=""
	    for s in ${sums}; do
		test_logs="$test_logs ${s%.sum}.log"
	    done

	    cp ${sums} ${test_logs} ${logs_dir}/ || status=1

	    # Copy over the logs from make check, which we need to find testcase errors.
            for check in $(read_var ${user_workspace}/check-artifacts.txt 'log_check_[^=]*'); do
		cp ${check} ${logs_dir}/ || status=1
            done
	fi

	xz ${logs_dir}/* || status=1
	scp $logopts ${logs_dir}/* ${logserver}:${basedir}/${dir}/ || status=1
	echo "Uploaded build logs and test results to ${logserver}:$logport:${basedir}/${dir}/ with status: $status"

	rm -rf ${logs_dir} || status=1
    fi
}

exit_handler()
{
    if $something_to_upload; then
	upload_to_logserver
    fi
    if $clean_lock_on_exit; then
	ssh $logopts $logserver rm -f $basedir/$dir.lock
    fi
    exit $status
}

if test $# -lt 1; then
    echo "ERROR: No options for build!"
    usage
fi

# Directory of ABE source files
abe_dir=

# This is where all the builds go
user_workspace="${WORKSPACE}"

# This is an optional directory for the reference copy of the git repositories.
git_reference="${HOME}/snapshots-ref"

# Override default versions of components
change=""

# Server to store results on.
logserver=""

# Template of logs' directory name
logname=""

# Compiler languages to build
languages=default

# Whether to run tests
runtests=false

# Target to build for
target=""

# ABE's --testcontainer= option
testcontainer_opt=""

# Whether attempt bootstrap
try_bootstrap=false

# The release version string, usually a date
releasestr=

# This is a string of optional extra arguments to pass to abe at runtime
user_options=""

# Return status
status=0

# Whether to exclude some component from 'make check'
excludecheck_opt=""

# Whether to rebuild the toolchain even if logs are already present.
# Note that the check is done on logserver/logname pair, so logname should not
# be relying on variables that this script sets for match to succeed.
# In practice, --norebuild option should be accompanied by something like
# --logname gcc-<sha1>
rebuild=true

# Whether to remove locks upon exit
clean_lock_on_exit=false

# Whether we have some logs to upload. False if we didn't build
# anything because the build was already performed.
something_to_upload=true

orig_parameters=( "$@" )

getopt -o s:g:w:o:l:rt:b:h -l override:,gcc-branch:,snapshots:,gitrepo:,abedir:,workspace:,options:,logserver:,logname:,languages:,runtests,target:,testcontainer:,bootstrap,help,excludecheck:,norebuild,extraconfig:,send-results-to: -Q -- "$@"
while test $# -gt 0; do
    case $1 in
	--gcc-branch) change="$change gcc=$2"; shift ;;
	--override) change="$change $2"; shift ;;
	--extraconfig) change="${change} --extraconfig $2"; shift ;;
        -s|--snapshots) user_snapshots=$2; shift ;;
        -g|--gitrepo) git_reference=$2; shift ;;
        --abedir) abe_dir=$2; shift ;;
	-t|--target) target=$2; shift ;;
	--testcontainer) testcontainer_opt=" --testcontainer $2"; shift ;;
        -w|--workspace) user_workspace=$2; shift ;;
        -o|--options) user_options=$2; shift ;;
        --logserver) logserver=$2; shift ;;
        --logname) logname=$2; shift ;;
        -l|--languages) languages=$2; shift ;;
        -r|--runtests) runtests="true" ;;
        -b|--bootstrap) try_bootstrap="true" ;;
	--excludecheck) excludecheck_opt="$excludecheck_opt --excludecheck $2"; shift ;;
	--send-results-to) change="${change} --send-results-to $2"; shift ;;
	--norebuild) rebuild=false ;;
	-h|--help) usage 0 ;;
        *) usage ;;
    esac
    shift
done

if test x"${abe_dir}" = x; then
    echo "Error: --abedir missing"
    usage
fi

if test x"${user_workspace}" = x; then
    echo "Error: user_workspace is not defined. Make sure WORKSPACE is defined or use --workspace option"
    usage
fi

# set default values for options to make life easier
user_snapshots="${user_workspace}/snapshots"

if [ x"$logserver" = x"" ] && [ x"$logname" != x"" ]; then
    echo "ERROR: \$logname is not provided, but \$logserver is set to $logserver"
    exit 1
fi
# Now that all variables from $logname template are known, calculate log dir.
declare dir
# Eval here because logname can include other variables at this point.
eval dir="$logname"

# Split $logserver into "server:port:path".
basedir="${logserver##*:}"
logserver_spec="${logserver%:*}"

logserver="$(echo $logserver_spec | cut -d: -f 1)"
logport="$(echo $logserver_spec | cut -s -d: -f 2)"

logopts="${logport:+-o Port=$logport}"

trap "exit_handler" EXIT

# Check status of logs on $logserver and rebuild if appropriate.
[ x"$logserver" != x"" ] && ssh $logopts $logserver mkdir -p "$(dirname $basedir/$dir)"

# Loop and wait until we successfully grabbed the lock.  The while condition is,
# effectively, "while true;" with a provision to skip if $logserver is not set.
while [ x"$logserver" != x"" ]; do
    # Non-blocking read lock, and check whether logs already exist.
    log_status=$(ssh $logopts $logserver flock -ns $basedir/$dir.lock -c \
	"\"if [ -e $basedir/$dir ]; then exit 0; else exit 2; fi\""; echo $?)

    case $log_status in
	0)
	    echo "Logs are already present in $logserver:$logport:$basedir/$dir"
	    if ! $rebuild; then
		something_to_upload=false
		exit 0
	    fi
	    echo "But we are asked to rebuild them anyway"
	    ;;
	1)
	    echo "Can't obtain read lock; waiting for another build to finish"
	    sleep 60
	    continue
	    ;;
	2)
	    echo "Logs don't exist in $basedir/$dir, trying to rebuild"
	    ;;
	*)
	    echo "ERROR: Unexpected status of logs: $log_status"
	    exit 1
	    ;;
    esac

    # Acquire the lock for the duration of the build.  The lock is released
    # in the "trap" cleanup below on signal or normal exit.
    # Note that the ssh command will be running in the background for the
    # duration of the build (spot "&" at its end).  Ssh command will exit
    # when lock file is deleted by the "trap" cleanup.
    # We place a unique marker into the lock file to check on our side who
    # has the lock, since we can't inspect return value of the ssh command.
    #
    # Note on '-tt': We forcefully allocate pseudo-tty for the flock command
    # so that flock dies (through SIGHUP) and releases the lock when this
    # script is cancelled or terminated for whatever reason.  Without SIGHUP
    # reaching flock we risk situations when a lock will hang forever preventing
    # any subsequent builds to progress.  There are a couple of options as to
    # exactly how enable delivery of SIGHUP (e.g., set +m), and 'ssh -tt' seems
    # like the simplest one.
    ssh -tt $logopts $logserver flock -nx $basedir/$dir.lock -c \
	"\"echo $(hostname)-$$-$BUILD_URL > $basedir/$dir.lock; while [ -e $basedir/$dir.lock ]; do sleep 10; done\"" &
    pid=$!
    # This is borderline fragile, since we are giving the above ssh command
    # a fixed period of time (10sec) to connect to $logserver and populate
    # $basedir/$dir.lock.  In practice, $logserver is a fast-ish machine,
    # which serves connections quickly.  In the worst-case scenario, we will
    # just retry a couple of times in this loop.
    sleep 10

    if [ x"$(ssh $logopts $logserver cat $basedir/$dir.lock)" \
	= x"$(hostname)-$$-$BUILD_URL" ]; then
	clean_lock_on_exit=true
	# Hurray!  Break from the loop and go ahead with the build!
	break
    fi

    kill $pid || true
done

# Test the config parameters from the Jenkins Build Now page

# See if we're supposed to build a source tarball
# (if tarsrc is set true, or user used --tarsrc option)
# shellcheck disable=SC2154
if test x"${tarsrc}" = xtrue -o "$(echo $user_options | grep -c -- --tarsrc)" -gt 0; then
    tars="--tarsrc"
fi

# See if we're supposed to build a binary tarball
# shellcheck disable=SC2154
if test x"${tarbin}" = xtrue -o "$(echo $user_options | grep -c -- --tarbin)" -gt 0; then
    tars="${tars} --tarbin "
fi

# Set the release string if specefied
if ! test x"${release}" = xsnapshot -o x"${release}"; then
    releasestr="--release ${release}"
fi
if test "$(echo $user_options | grep -c -- --release)" -gt 0; then
    release="$(echo  $user_options | grep -o -- "--release [a-zA-Z0-9]* " | cut -d ' ' -f 2)"
    releasestr="--release ${release}"
fi

# Get the versions of dependant components to use
# shellcheck disable=SC2154
if test x"${gmp_snapshot}" != x"latest" -a x"${gmp_snapshot}" != x; then
    change="${change} gmp=${gmp_snapshot}"
fi
# shellcheck disable=SC2154
if test x"${mpc_snapshot}" != x"latest" -a x"${mpc_snapshot}" != x; then
    change="${change} mpc=${mpc_snapshot}"
fi
# shellcheck disable=SC2154
if test x"${mpfr_snapshot}" != x"latest" -a x"${mpfr_snapshot}" != x; then
    change="${change} mpfr=${mpfr_snapshot}"
fi

# shellcheck disable=SC2154
if test x"${binutils_snapshot}" != x"latest" -a x"${binutils_snapshot}" != x; then
    change="${change} binutils=${binutils_snapshot}"
fi
# shellcheck disable=SC2154
if test x"${linux_snapshot}" != x"latest" -a x"${linux_snapshot}" != x; then
    change="${change} linux=${linux_snapshot}"
fi

if test x"${target}" != x"native" -a x"${target}" != x; then
    platform="--target ${target}"
fi

# shellcheck disable=SC2154
if test x"${libc}" != x; then
    # ELF based targets are bare metal only
    case ${target} in
	arm*-none-*)
	    change="${change} --set libc=newlib"
	    ;;
	*)
	    change="${change} --set libc=${libc}"
	    ;;
    esac
fi

# Create a build directory
if test -d ${user_workspace}/_build; then
    rm -fr ${user_workspace}/_build
fi
mkdir -p ${user_workspace}/_build

# Use the newly created build directory
pushd ${user_workspace}/_build

# Configure Abe itself. Force the use of bash instead of the Ubuntu
# default of dash as some configure scripts go into an infinite loop with
# dash. Not good...
export CONFIG_SHELL="/bin/bash"
# shellcheck disable=SC2154
if test x"${debug}" = x"true"; then
    export CONFIG_SHELL="/bin/bash -x"
fi

# Print some information about the build machine
echo Running on "$(hostname)"
uname -a
lsb_release -a
lscpu
cat /proc/meminfo

$CONFIG_SHELL ${abe_dir}/configure --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${git_reference} --with-languages=${languages} --enable-schroot-test

# load commonly used varibles set by configure
if test -e "${PWD}/host.conf"; then
    # shellcheck disable=SC1090
    . "${PWD}/host.conf"
fi

# We used to delete *.sum files, but this should not be necessary. This
# check is transitional, and will be removed later.
if [ -n "$(find ${user_workspace} -name \*.sum)" ]; then
   echo "Found *.sum files, but workspace should be empty!"
   exit 1
fi

if test x"${try_bootstrap}" = xtrue; then
    # Attempt to bootstrap GCC is build and target are compatible
    build1="$(grep "^build=" host.conf | sed -e "s/build=\(.*\)-\(.*\)-\(.*\)-\(.*\)/\1-\3-\4/")"
    target1="$(echo ${target} | sed -e "s/\(.*\)-\(.*\)-\(.*\)-\(.*\)/\1-\3-\4/")"
    if test x"${build1}" = x"${target1}" -o x"${platform}" = x""; then
	try_bootstrap="--set buildconfig=bootstrap"
    else
	try_bootstrap=""
    fi
else
    try_bootstrap=""
fi

# Checkout all sources now to avoid grabbing lock for 1-2h while building and
# testing runs.  We configure ABE to use reference snapshots, which are shared
# across all builds and are updated by an external process.  The lock protects
# us from looking into an inconsistent state of reference snapshots.
(
    flock -s 9
    $CONFIG_SHELL ${abe_dir}/abe.sh ${platform} ${change} --checkout all
) 9>${git_reference}.lock

# Now we build the cross compiler, for a native compiler this becomes
# the stage2 bootstrap build.
ret=0
$CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/build-artifacts.txt --disable update ${tars} ${releasestr} ${platform} ${change} ${try_bootstrap} --timeout 100 --build all --disable make_docs > build.out 2> >(tee build.err >&2) || ret=$?

# If abe returned an error, make jenkins see this as a build failure
if test $ret -gt 0; then
    echo "================= TAIL OF LOG: BEGIN ================="
    tail -n 50 build.out
    echo "================= TAIL OF LOG: FINISH ================="
    exit 1
fi

# if runtests is true, then run make check after the build completes
if $runtests; then
    # check that expect is working, and dump some debug info then exit if not
    if ! echo "spawn true" | /usr/bin/expect -f - >/dev/null; then
        echo "expect cannot spawn processes. Aborting make check."
        echo "some debug info follows..."
        echo "running: ls -l /dev/ptmx"
        ls -l /dev/ptmx
        echo "running: ls -l /dev/pts"
        ls -l /dev/pts
        echo "running: grep devpts /proc/mounts"
        grep devpts /proc/mounts
        exit 1
    fi

    check="--check all"
    check="${check}${excludecheck_opt}${testcontainer_opt}"

    ret=0
    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/check-artifacts.txt --disable update ${check} ${tars} ${releasestr} ${platform} ${change} ${try_bootstrap} --timeout 100 --build all --disable make_docs > check.out 2> >(tee check.err >&2) || ret=$?

    # If abe returned an error, make jenkins see this as a build failure
    if test $ret -gt 0; then
	echo "================= TAIL OF LOG: BEGIN ================="
	tail -n 50 check.out
	echo "================= TAIL OF LOG: FINISH ================="
	exit 1
    fi
fi

# Create the BUILD-INFO file for Jenkins.
cat << EOF > ${user_workspace}/BUILD-INFO.txt
Format-Version: 0.5

Files-Pattern: *
License-Type: open
EOF

echo "Build parameters: ${orig_parameters[*]}"

# Canadian Crosses are a win32 hosted cross toolchain built on a Linux
# machine.
if test x"${canadian}" = x"true"; then
    $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/canadian-build1-artifacts.txt --disable update ${change} ${platform} --build all
    distro="$(lsb_release -sc)"
    # Ubuntu Lucid uses an older version of Mingw32
    if test x"${distro}" = x"lucid"; then
	$CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/canadian-build2-artifacts.txt --disable update ${change} ${tars} --host=i586-mingw32msvc ${platform} --build all
    else
	$CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/canadian-build2-artifacts.txt --disable update ${change} ${tars} --host=i686-w64-mingw32 ${platform} --build all
    fi
fi

# This setups all the files needed by tcwgweb
if test x"${logserver}" != x"" ; then
    if test x"${tarsrc}" = xtrue -a x"${release}" != x; then
	allfiles="$(ls ${user_snapshots}/*${release}*.xz)"
	srcfiles="$(echo ${allfiles} | grep -E -v "arm|aarch")"
	scp $logopts ${srcfiles} ${logserver}:/home/abe/var/snapshots/ || status=1
	rm -f ${srcfiles} || status=1
    fi

    if test x"${tarbin}" = xtrue -a x"${release}" != x; then
	allfiles="$(ls ${user_snapshots}/*${release}*.xz)"
	binfiles="$(echo ${allfiles} | grep -E "arm|aarch")"
	scp $logopts ${binfiles} ${logserver}:/work/space/binaries/ || status=1
	rm -f ${binfiles} || status=1
    fi
fi

exit $status