1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
|
#!/bin/bash
set -euf -o pipefail
scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh
# shellcheck source=round-robin.sh
. $scripts/round-robin.sh
convert_args_to_variables "$@"
obligatory_variables rr[ci_project] rr[ci_config]
declare -A rr
# All config about configure flags, simulator, pretty names, .... is
# implemented in this file
# shellcheck source=tcwg_gnu-config.sh
. $scripts/tcwg_gnu-config.sh
# Execution mode: build or bisect,
rr[mode]="${rr[mode]-build}"
# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
rr[update_baseline]="${rr[update_baseline]-ignore}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
# {toolchain_ver}-{target}[-{type_of_test}]
IFS=- read -a ci_config <<EOF
${rr[ci_config]}
EOF
# Toolchain version -- master or release
toolchain_ver=${toolchain_ver-${ci_config[0]}}
ci_target=${ci_target-${ci_config[1]}}
# optional type_of_test contains the type of action to perform in this test
# campaign: bootstrap, bootstrap_lto, ...
type_of_test=${type_of_test-${ci_config[2]-}}
case "${rr[ci_project]}" in
tcwg_gcc_*|tcwg_bootstrap_*)
rr[target]="${rr[target]-native}"
rr[components]="gcc"
;;
tcwg_binutils_*)
rr[target]="${rr[target]-native}"
rr[components]="binutils"
;;
tcwg_glibc_*)
rr[target]="${rr[target]-native}"
rr[components]="glibc"
;;
tcwg_gdb_*)
rr[target]="${rr[target]-native}"
rr[components]="gdb"
;;
tcwg_gnu_native_*)
rr[target]="${rr[target]-native}"
rr[components]="binutils gcc linux glibc gdb"
;;
tcwg_gnu_cross_*)
rr[target]="${rr[target]-$ci_target}"
rr[components]="binutils gcc linux glibc gdb qemu"
;;
tcwg_gnu_embed_*)
rr[target]="${rr[target]-$ci_target}"
rr[components]="binutils gcc newlib gdb qemu"
;;
tcwg_gnu_mingw_*)
rr[target]="${rr[target]-$ci_target}"
rr[components]="binutils gcc mingw"
;;
*) assert_with_msg "Unknown ci_project: ${rr[ci_project]}" false ;;
esac
rr[major]=3
rr[minor]=0
case "${rr[ci_project]}" in
tcwg_gnu_native_fast_check_gdb)
# Tests that run quickly and have stable results. This list is taken from
# Sourceware's builder:
# https://sourceware.org/git/?p=builder.git;a=blob;f=builder/master.cfg;h=612b3177f0e5#l2407
testsuites=(gdb.base/break-always.exp
gdb.base/break-caller-line.exp
gdb.base/break-entry.exp
gdb.base/break.exp
gdb.base/break-fun-addr.exp
gdb.base/break-idempotent.exp
gdb.base/break-include.exp
gdb.base/break-inline.exp
gdb.base/break-main-file-remove-fail.exp
gdb.base/break-on-linker-gcd-function.exp
gdb.base/breakpoint-in-ro-region.exp
gdb.base/breakpoint-shadow.exp
gdb.base/break-probes.exp
gdb.gdb/unittest.exp
)
;;
tcwg_gnu_native_fast_check_gcc)
# Use a small subset of GCC testsuite until we have made the
# results stable.
testsuites=(compile.exp
execute.exp
)
;;
esac
runtestflags=()
if test_array testsuites; then
# shellcheck disable=SC2154
for i in "${testsuites[@]}"; do
runtestflags+=(--set "runtestflags=$i")
done
fi
# Use baseline branches by default.
for c in ${rr[components]}; do
rr[${c}_git]=${rr[${c}_git]-baseline}
done
start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"
if $verbose2; then set -x; fi
trap print_traceback EXIT
# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
case "${rr[mode]}" in
"bisect")
case "$(print_single_updated_component):${rr[ci_project]}" in
*:tcwg_gnu_mingw_*) default_start_at="clean_sysroot" ;;
binutils:*) default_start_at="build_abe-binutils" ;;
gcc:tcwg_gnu_embed_*) default_start_at="build_abe-stage1" ;;
gcc:tcwg_gnu_cross_*) default_start_at="build_abe-stage1" ;;
gcc:tcwg_gnu_native_*) default_start_at="build_abe-gcc" ;;
gcc:tcwg_gcc_*) default_start_at="build_abe-gcc" ;;
gcc:tcwg_bootstrap_*) default_start_at="build_abe-${type_of_test#check_}" ;;
glibc:tcwg_glibc_*) default_start_at="build_abe-glibc" ;;
linux:*|glibc:*|newlib:*) default_start_at="clean_sysroot" ;;
gdb:*) default_start_at="build_abe-gdb" ;;
qemu:*) default_start_at="build_abe-qemu" ;;
*) assert_with_msg "Trying to bisecting unknown component: $(print_single_updated_component)" false ;;
esac
;;
esac
if [ x"$start_at" = x"default" ]; then
start_at="$default_start_at"
fi
if [ x"$finish_at" = x"default" ]; then
finish_at="$default_finish_at"
fi
run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"
# Initialize sysroot directory in ABE's build/ with system headers
# so that [native] glibc can find headers to build against.
init_abe_sysroot ()
{
(
set -euf -o pipefail
local abe_sysroot gnu_target
abe_sysroot=$(print_abe_sysroot)
gnu_target=$(print_gnu_target "$ci_target")
rm -rf "$abe_sysroot/usr/include/"
mkdir -p "$abe_sysroot/usr/include/"
rsync -a "/usr/include/" "$abe_sysroot/usr/include/"
# Debian/Ubuntu have arch headers under target-specific directory.
rsync -a "/usr/include/$gnu_target/" "$abe_sysroot/usr/include/"
)
}
# Build mingw parts: headers, crt and libs.
build_mingw ()
{
(
set -euf -o pipefail
local todo="$1"
if [ "$todo" = "headers" ]; then
clone_repo mingw
fi
local abe_sysroot gnu_host gnu_target
abe_sysroot=$(print_abe_sysroot)
gnu_host=$(print_gnu_target native)
gnu_target=$(print_gnu_target "$ci_target")
PATH="$(pwd)/abe/builds/destdir/$gnu_host/bin:$PATH"
export PATH
local dir="mingw-$todo"
rm -rf "$dir"
mkdir "$dir"
cd "$dir"
if [ "$todo" = "headers" ]; then
../mingw/mingw-w64-headers/configure \
--prefix="$abe_sysroot" --host="$gnu_target"
elif [ "$todo" = "crt" ]; then
../mingw/mingw-w64-crt/configure \
--prefix="$abe_sysroot" --host="$gnu_target" \
--enable-libarm64 \
--disable-lib32 \
--disable-lib64 \
--disable-libarm32 \
--disable-shared \
--with-default-msvcrt=msvcrt
elif [ "$todo" = "libs" ]; then
../mingw/configure \
--prefix="$abe_sysroot" --host="$gnu_target" \
--enable-libarm64 \
--disable-lib32 \
--disable-lib64 \
--disable-libarm32 \
--disable-shared \
--with-libraries=winpthreads \
--with-default-msvcrt=msvcrt
fi
make -j"$(nproc --all)"
make install
if [ "$todo" = "headers" ]; then
# ??? I'm not sure whether this is a hack or an adorable percularity
# of building a mingw toolchain. The point of this symlink is that
# it's easiest for stage1 GCC mingw compiler to find headers in mingw/
# directory instead of the actual sysroot.
rm -rf "$abe_sysroot/../mingw"
ln -s "$abe_sysroot" "$abe_sysroot/../mingw"
fi
)
}
# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
{
(
set -euf -o pipefail
no_build_regression_p "$@"
local ref_artifacts=$1
local new_artifacts=$2
local sumfiles_base=$ref_artifacts/sumfiles
local sumfiles_new=$new_artifacts/sumfiles
local xfails="$sumfiles_new/xfails.xfail"
# Remove files generated here (in case we are re-generating results in
# round-robin-baseline.sh).
rm -f "$xfails"
if ! [ -d $sumfiles_base ]; then
return 0
elif ! [ -d $sumfiles_new ]; then
return 1
fi
cat > "$xfails" <<EOF
# This file contains three sections:
# - newly detected flaky tests (if any)
# - known flaky tests (from baseline)
# - known failures (from baseline)
#
EOF
local flaky_tests="$run_step_artifacts/flaky.xfail"
local baseline_fails="$run_step_artifacts/baseline.xfail"
build_abe_check_xfails "$flaky_tests" "$baseline_fails"
# Add known flaky tests and baseline_fails to the xfails.
#
# Note #1: We generate $baseline_fails without regard for flaky
# tests. Therefore, validate_failures in no_regression_p() will
# see same tests with and without flaky attributes.
# Validate_failure uses python sets to store results, so the first
# entry wins. Therefore, we need to put lists of flaky tests before
# lists of expected fails -- $baseline_fails.
#
# Note #2: Order of expired and non-expired flaky tests is less of
# an issue because expired entries are discarded by validate_failures
# before adding them to the ResultSet. Still, for uniformity, we
# put fresh flaky entries before older ones.
cat "$flaky_tests" >> "$xfails"
echo "# Known failures (from baseline)" >> "$xfails"
cat "$baseline_fails" >> "$xfails"
# Set and save result_expiry_date so that we use the same expiration date
# when re-generating results in round-robin-baseline.sh.
if [ "${rr[result_expiry_date]-}" = "" ]; then
rr[result_expiry_date]=$(date +%Y%m%d)
# finalize_manifest() will not see declaration because we are running
# in a sub-shell. Update manifest manually.
cat <<EOF | manifest_out
rr[result_expiry_date]="${rr[result_expiry_date]}"
EOF
fi
local validate_failures="gcc-compare-results/contrib/testsuite-management/validate_failures.py"
"$validate_failures" --manifest="$xfails" \
--expiry_date="${rr[result_expiry_date]}" \
--build_dir=$sumfiles_new --verbosity=1 \
> $run_step_artifacts/fails.sum &
local res
res=0 && wait $! || res=$?
if [ $res != 0 ]; then
# Add a short marker to record the status (used by Jenkins build-name)
local n_regressions
n_regressions=$(grep -c "^[A-Z]\+:" $run_step_artifacts/fails.sum \
|| true)
echo "# $n_regressions regressions" \
> $run_step_artifacts/results.regressions
if [ $res = 2 ]; then
# Result comparison found regressions (exit code 2)
#
# Exit code 1 means that the script has failed to process
# .sum files. This likely indicates malformed or very unusual
# results.
printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
local exp
while read exp; do
printf "++testsuites %s " $exp >> $run_step_artifacts/extra-bisect-params
done < <(cat $run_step_artifacts/fails.sum \
| awk '/^Running .* \.\.\./ { print $2 }')
printf "\n" >> $run_step_artifacts/extra-bisect-params
fi
fi
return $res
)
}
# Implement rr[breakup_changed_components] hook.
tcwg_gnu_breakup_changed_components ()
{
(
set -euf -o pipefail
local cc=""
case "${rr[ci_project]}" in
*_check_*)
# Changes to "foo" of check_foo projects tend to cause the most
# regressions.
# Breakup changed components into "foo" and the rest of components
# to reduce the number of builds.
cc=$(echo "${rr[ci_project]}" | sed -e "s/.*_check_\(.*\)/\1/")
;;
esac
if print_changed_components "\n" | grep "^$cc\$" >/dev/null; then
echo "$cc"
print_changed_components "\n" | grep -v "^$cc\$" | tr '\n' ' ' \
| sed -e "s/ \$//g"
echo
else
print_changed_components "\n"
fi
)
}
rr[breakup_changed_components]=tcwg_gnu_breakup_changed_components
# Define gnu_data[] as needed below
settings_for_ci_project_and_config "${rr[ci_project]}" "${rr[ci_config]}"
run_step stop_on_fail -10 reset_artifacts
run_step stop_on_fail x prepare_abe
case "${rr[ci_project]}" in
tcwg_gnu_cross_build|tcwg_gnu_embed_build)
run_step skip_on_fail 0 true
run_step skip_on_fail 1 build_abe binutils
run_step skip_on_fail 2 build_abe stage1 -- \
${gnu_data[gcc_override_configure]}
run_step skip_on_fail x clean_sysroot
case "${rr[components]}" in
*glibc*)
run_step skip_on_fail 3 build_abe linux
run_step skip_on_fail 4 build_abe glibc
;;
*newlib*)
run_step skip_on_fail 4 build_abe newlib
;;
esac
run_step skip_on_fail 5 build_abe stage2 -- \
${gnu_data[gcc_override_configure]}
run_step skip_on_fail 6 build_abe gdb
run_step skip_on_fail 7 build_abe qemu
;;
tcwg_gnu_cross_check_*|tcwg_gnu_embed_check_*)
run_step skip_on_fail -8 build_abe binutils
run_step skip_on_fail -7 build_abe stage1 -- \
${gnu_data[gcc_override_configure]}
run_step skip_on_fail x clean_sysroot
case "${rr[components]}" in
*glibc*)
run_step skip_on_fail -6 build_abe linux
run_step skip_on_fail -5 build_abe glibc
;;
*newlib*)
run_step skip_on_fail -5 build_abe newlib
;;
esac
run_step skip_on_fail -4 build_abe stage2 -- \
${gnu_data[gcc_override_configure]}
run_step skip_on_fail -3 build_abe gdb
run_step skip_on_fail -2 build_abe qemu
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe "check_${rr[ci_project]#*check_}" -- "${runtestflags[@]}" \
${gnu_data[gcc_override_configure]} \
${gnu_data[gcc_target_board_options]} \
${gnu_data[qemu_cpu]}
;;
tcwg_gnu_native_build)
run_step skip_on_fail 0 true
run_step skip_on_fail 1 build_abe binutils
run_step skip_on_fail 2 build_abe gcc
run_step skip_on_fail x clean_sysroot
run_step skip_on_fail 4 build_abe linux
run_step skip_on_fail 5 build_abe glibc
run_step skip_on_fail 6 build_abe gdb
;;
tcwg_gnu_native_check_*|tcwg_gnu_native_fast_check_*)
component="${rr[ci_project]#tcwg_gnu_native_}"
component="${component#fast_}"
declare -a abe_arguments=()
if [ "${component}" = check_gdb ]; then
abe_arguments=(--set check_buffer_workaround=gdb-read1)
fi
run_step skip_on_fail -8 build_abe binutils
run_step skip_on_fail -7 build_abe gcc
run_step skip_on_fail x clean_sysroot
run_step skip_on_fail -5 build_abe linux
run_step skip_on_fail -4 build_abe glibc
run_step skip_on_fail -3 build_abe gdb
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe "$component" -- "${runtestflags[@]}" \
"${abe_arguments[@]}"
;;
tcwg_bootstrap_build)
run_step skip_on_fail 0 true
run_step skip_on_fail 1 build_abe ${type_of_test}
;;
tcwg_bootstrap_check)
run_step skip_on_fail -2 build_abe ${type_of_test#check_}
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe ${type_of_test} -- "${runtestflags[@]}"
;;
tcwg_binutils_build|tcwg_gcc_build|tcwg_gdb_build)
run_step skip_on_fail 0 true
run_step skip_on_fail 1 build_abe "${rr[components]}"
;;
tcwg_binutils_check|tcwg_gcc_check)
run_step skip_on_fail -2 build_abe "${rr[components]}"
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe "check_${rr[components]}" \
-- "${runtestflags[@]}"
;;
tcwg_gdb_check)
# GDB's testsuite has a limitation where it can only find debug info
# within the installation prefix. To allow it to find the
# distro-installed debug info for ld.so, use /usr as the prefix. We
# don't need to actually install GDB there though, so disable the
# install step.
run_step skip_on_fail -2 build_abe "${rr[components]}" \
-- --prefix /usr --disable install
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe "check_${rr[components]}" \
-- "${runtestflags[@]}" --set check_buffer_workaround=gdb-read1
;;
tcwg_glibc_build)
run_step skip_on_fail 0 init_abe_sysroot
# ABE tries to copy gcc runtime libraries on glibc install, which
# fails when we don't build gcc. Workaround by not installing glibc.
run_step skip_on_fail 1 build_abe glibc -- --disable install
;;
tcwg_glibc_check)
run_step skip_on_fail -3 init_abe_sysroot
run_step skip_on_fail -2 build_abe glibc -- --disable install
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe check_glibc \
-- "${runtestflags[@]}" --disable install
;;
tcwg_gnu_mingw_build)
run_step skip_on_fail 0 true
run_step skip_on_fail x clean_sysroot
run_step skip_on_fail 1 build_abe binutils
run_step skip_on_fail 2 build_mingw headers
run_step skip_on_fail 3 build_abe stage1
run_step skip_on_fail 4 build_mingw crt
run_step skip_on_fail 5 build_mingw libs
run_step skip_on_fail 6 build_abe stage2
#run_step skip_on_fail 7 build_abe gdb
;;
tcwg_gnu_mingw_check_*)
run_step skip_on_fail x clean_sysroot
run_step skip_on_fail -8 build_abe binutils
run_step skip_on_fail -7 build_mingw headers
run_step skip_on_fail -6 build_abe stage1
run_step skip_on_fail -5 build_mingw crt
run_step skip_on_fail -4 build_mingw libs
run_step skip_on_fail -3 build_abe stage2
#run_step skip_on_fail -2 build_abe gdb
run_step skip_on_fail -1 build_abe dejagnu
run_step skip_on_fail 0 build_abe "${rr[ci_project]#tcwg_gnu_mingw_}" \
-- "${runtestflags[@]}"
;;
esac
run_step reset_on_fail x check_regression
trap "" EXIT
|