From e7b992985284c335eaec9a333c0a80c92f7a93ec Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 12 Feb 2019 08:34:25 +0000 Subject: tcwg_kernel-bisect.sh: Use rr[ci_config] instead of its sub-items Replaces uses of ${rr[linux_version]} and $toolchain with ${rr[ci_config]}, which makes code more generic. Change-Id: I27030c467b3bdb448aa2dbed36272b93726789cb --- tcwg_kernel-bisect.sh | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index 45a6aaf0..8ab8a014 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -29,10 +29,6 @@ obligatory_variables rr[current_project] rr[ci_config] current_project=${rr[current_project]} -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then - obligatory_variables toolchain -fi - verbose="${verbose:-true}" set -u @@ -43,21 +39,22 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/fai rebase_workaround=false -if [ x"${rr[ci_project]}" = x"tcwg_kernel" ] \ - && [ x"$current_project" = x"linux" -a x"${rr[linux_version]}" = x"next" ]; then - # Workaround linux-next/master rebasing on top of linux-next/stable. - # Search for regressions against linux-mainline:master (aka linux-next:stable). - clone_or_update_repo $current_project stable ${rr[linux_url]} - # Just in case linux-next:stable has advanced between the build and bisect jobs, - # use merge base between linux-next:stable and $bad_rev. - linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}" - cat < $artifacts/jenkins/mail-recipients.txt < Date: Tue, 12 Feb 2019 08:37:08 +0000 Subject: tcwg_kernel-bisect.sh: Detect linker and make errors in the logs "ERROR:" detects linker errors and "] Error " detects make errors Change-Id: I91fe470d2cf81487945e2e55796c0dfd3f5dc74c --- tcwg_kernel-bisect.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index 8ab8a014..bac608bf 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -268,6 +268,10 @@ Could not identify regression in $current_project for CI configuration ${rr[ci_c EOF fi +# In log scan for errors below +# - ": error:" detects compiler errors from GCC and Clang +# - "^ERROR:" detects linker errors +# - "] Error " detects GNU make errors cat >> $artifacts/jenkins/mail-body.txt < Date: Tue, 12 Feb 2019 09:12:36 +0000 Subject: [rr-many 1/N]: Support testing multiple projects in the same build This patch series is both a major cleanup and an optimization of round-robin jobs. We switch from testing each component (binutils, gcc, linux) in separate builds in round-robin fashion to testing all components in the same build, and, if there's a failure, re-triggering builds for individual components like we are doing now. Then if individual build fails, it triggers bisect on that one component. The cleanup part is that jenkins triggers are greatly simplified and we can remove all trigger jobs (of which we had hundreds). The optimization part is that successful builds (which is the majority) now consume N times less resources, where N is the number of components -- 3 in case of binutils, gcc, linux, -- because all components are updated and tested in the same build. This patch is the first in the series, and it replaces uses of $current_project and $current_branch with "baseline" keyword of components' branches. I.e., for testing LLVM in LLVM+Kernel jobs we currently use ==rr[llvm_branch] master ==rr[linux_branch] master \ ==rr[current_project] llvm ==rr[current_branch] SHA1 and it will change to ==rr[llvm_branch] SHA1 ==rr[linux_branch] baseline . IMO, the latter is significantly more intuitive. Change-Id: Ib47dcac1571381f870150e4872ec37be5d900a07 --- round-robin.sh | 46 +++++++++++++++++++++++++++++++++++++++------- tcwg_kernel-bisect.sh | 8 +++----- tcwg_kernel-build.sh | 29 +++++++++++++++++------------ 3 files changed, 59 insertions(+), 24 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index e4ff41a0..1fdd27de 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -7,6 +7,11 @@ #rr[PROJECT_url] # PROJECT's git branch or SHA1 revision parsable by git rev-parse. +# A special value "baseline" means that PROJECT is not being updated +# in this build, and its baseline branch should be used. +# In a successful build "update_baseline" step will update baseline +# branches of all PROJECTs to the current values, thus setting +# a baseline for the next build. #rr[PROJECT_branch] # PROJECT's git SHA1 revision. These are mostly used in manifests. @@ -16,13 +21,6 @@ # be present in all above git repos (if ${rr[init_configuration]} is false). #rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}" -# PROJECT that we are testing in this build. Use ${rr[current_branch]} for -# this project, and ${rr[baseline_branch]} for all other projects. -#rr[current_project]="$current_project" - -# Git branch or SHA1 revision of ${rr[current_project]} to test. -#rr[current_branch]="$current_branch" - # Run mode: bisect or non-bisect. In bisect mode we do a couple things # slightly differently (e.g., don't touch repo in clone_repo() ). #rr[mode]="$mode" @@ -40,6 +38,40 @@ # to initialize baseline branches in git repos. #rr[init_configuration]=false +# Print round-robin components that are being updated in this build +# (the ones using non-baseline branches). +print_updated_components () +{ + ( + set -euf -o pipefail + + local delim="" + local c + for c in ${rr[components]}; do + if [ x"${rr[${c}_branch]}" != x"baseline" ]; then + printf "%s%s" "$delim" "$c" + delim=${1- } + fi + done + ) +} + +# Print the single round-robin component being updated in this build. +# Print nothing if multiple components are being updated. +print_single_updated_component () +{ + ( + set -euf -o pipefail + + local -a updated_components + updated_components=($(print_updated_components)) + + if [ ${#updated_components[@]} = 1 ]; then + echo "${updated_components[0]}" + fi + ) +} + # Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most # important artifact, since it records the metric of how successful the build # is. diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index bac608bf..ef1cb2f5 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -16,7 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*" convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" -obligatory_variables bad_rev baseline_rev build_script rr[ci_project] +obligatory_variables bad_rev baseline_rev build_script current_project BUILD_URL="${BUILD_URL:-$(pwd)}" reproduce_bisect="${reproduce_bisect:-false}" @@ -25,9 +25,7 @@ reproduce_bisect="${reproduce_bisect:-false}" convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@" $reproduce_bisect || manifest_pop -obligatory_variables rr[current_project] rr[ci_config] - -current_project=${rr[current_project]} +obligatory_variables rr[ci_project] rr[ci_config] verbose="${verbose:-true}" @@ -64,7 +62,7 @@ $build_script \ %% $rel_artifacts/manifests/build-baseline.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ ==rr[mode] "baseline" \ - ==rr[current_branch] "$baseline_rev" \ + ==rr[${current_project}_branch] "$baseline_rev" \ ==rr[reset_baseline] true \ ==rr[top_artifacts] "$rel_artifacts/build-baseline" \ --verbose "$verbose" diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index ab169e5b..dabc2995 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -4,6 +4,7 @@ set -euf -o pipefail scripts=$(dirname $0) . $scripts/jenkins-helpers.sh +. $scripts/round-robin.sh convert_args_to_variables "$@" @@ -12,18 +13,10 @@ obligatory_variables rr[ci_config] # Execution mode: baseline, bisect, continue, jenkins-full rr[mode]="${rr[mode]-baseline}" -case "${rr[mode]}" in - "jenkins-full") - obligatory_variables rr[current_project] rr[current_branch] - ;; -esac - # Set custom revision for one of the projects, and use baseline revisions # for all other projects. rr[ci_project]="${rr[ci_project]-tcwg_kernel}" rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}" -rr[current_branch]="${rr[current_branch]-default}" -rr[current_project]="${rr[current_project]-none}" rr[reset_baseline]="${rr[reset_baseline]-false}" rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}" @@ -35,6 +28,18 @@ rr[toolchain]=${rr[toolchain]-${ci_config[0]}} rr[target]=${rr[target]-${ci_config[2]}} rr[linux_config]=${rr[linux_config]-${ci_config[4]}} +case "${rr[toolchain]}" in + llvm) rr[components]="llvm linux" ;; + gnu) rr[components]="binutils gcc linux" ;; + *) assert false ;; +esac + +# Use baseline branches by default. +for c in ${rr[components]}; do + rr[${c}_branch]=${rr[${c}_branch]-baseline} + obligatory_variables rr[${c}_url] +done + start_at="${start_at-default}" finish_at="${finish_at-default}" verbose="${verbose-true}" @@ -53,12 +58,13 @@ case "${rr[mode]}" in default_finish_at="update_baseline" ;; "bisect") - case "${rr[toolchain]}:${rr[current_project]}" in + case "${rr[toolchain]}:$(print_single_updated_component)" in gnu:binutils) default_start_at="build_abe-binutils" ;; gnu:gcc) default_start_at="build_abe-stage1" ;; gnu:linux) default_start_at="count_linux_objs" ;; llvm:llvm) default_start_at="build_llvm" ;; llvm:linux) default_start_at="count_linux_objs" ;; + *) assert false ;; esac default_finish_at="check_regression" ;; @@ -81,8 +87,6 @@ fi run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose" -. $scripts/round-robin.sh - # Build Linux kernel build_linux () { @@ -114,7 +118,8 @@ build_linux () # Otherwise the compiler is new in every build and we would # only clobber ccache volume. local ccache="" - if [ x"${rr[mode]}" != x"jenkins-full" -a x"${rr[current_project]}" = x"linux" ]; then + if [ x"${rr[mode]}" != x"jenkins-full" \ + -a x"$(print_single_updated_component)" = x"linux" ]; then ccache="ccache" fi cat > $(pwd)/bin/${rr[target]}-cc < Date: Tue, 12 Feb 2019 09:27:42 +0000 Subject: [rr-many 2/N] Update and simplify round-robin.sh: clone_repo() With simpler meaning of rr[COMPONENT_branch] to be either "baseline" (instead of "default", overrides from ${rr[current_project]}, etc.) we only need to resolve "baseline" value. Change-Id: I5f32949587688d152402b6b64ed4a37ac062019e --- round-robin.sh | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index 1fdd27de..bd3d886f 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -123,24 +123,15 @@ clone_repo () fi local branch - # Select the branch to build. - # ${rr[current_branch]} specifies branch for ${rr[current_project]}, - # and everything else uses baseline branch. - if [ x"$project" = x"${rr[current_project]}" ]; then - # Use the tip of the branch tracked in the current configuration. - if [ x"${rr[current_branch]}" = x"default" ]; then - branch=${rr[${rr[current_project]}_branch]} - elif [ x"${rr[current_branch]}" = x"baseline" ]; then - branch="refs/remotes/baseline/${rr[baseline_branch]}" - else - branch=${rr[current_branch]} - fi - elif ${rr[init_configuration]-false}; then - branch="${rr[${project}_branch]}" - else + + # Resolve "baseline" branch specifier. + if [ x"${rr[${project}_branch]}" = x"baseline" ]; then branch="refs/remotes/baseline/${rr[baseline_branch]}" + else + branch="${rr[${project}_branch]}" fi + # Allow manifest override branch="${rr[${project}_rev]-$branch}" # Decide on whether to use read-only or read-write mode for @@ -148,7 +139,7 @@ clone_repo () # developers without ssh keys on Linaro git servers to reproduce builds # in --mode "baseline". local read_only="true" - if [ x"$project" = x"${rr[current_project]}" -a \ + if [ x"${rr[${project}_branch]}" != x"baseline" -a \ x"${rr[mode]}" = x"jenkins-full" ]; then read_only=false fi -- cgit v1.2.3 From ddddcf637bc3f1dcff1ca774f19b4fd35bdfd3a9 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 12 Feb 2019 09:32:55 +0000 Subject: [rr-many 3/N] Rework trigger files for follow-up builds. Now that we reliably reach check_regression (due to previous improvements in run_step support) move creation of trigger files here from clone_repo. If updating multiple components -- create trigger files for individual builds. I.e., trigger-build-llvm: llvm_branch=SHA1_LLVM linux_branch=baseline trigger-build-linux: llvm_branch=baseline linux_branch=SHA1_LINUX If updating a single component -- create trigger-bisect like before. In the tcwg_kernel-bisect.sh script we start using trigger files produced by the build script, thus simplifying and generalizing bisect script. Change-Id: Iaee02ce0f8fbb3f9f5dbee805089ff31f1859cb5 --- round-robin.sh | 40 ++++++++++++++++++++++++++-------------- tcwg_kernel-bisect.sh | 19 ++++++------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index bd3d886f..57da594b 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -157,20 +157,6 @@ clone_repo () cat < ${rr[top_artifacts]}/trigger-bisect-on-failure < ${rr[top_artifacts]}/distance-to-baseline - fi ) } @@ -341,6 +327,32 @@ check_regression () set -euf -o pipefail if ! ${rr[reset_baseline]} && ! no_regression_p; then + local single_component=$(print_single_updated_component) + + if [ x"$single_component" = x"" -o x"${rr[mode]}" = x"bisect" ]; then + local c + for c in $(print_updated_components); do + local c2 + for c2 in ${rr[components]}; do + if [ x"$c" = x"$c2" ]; then + echo "${c2}_branch=${rr[${c}_branch]}" + else + echo "${c2}_branch=baseline" + fi + done >> ${rr[top_artifacts]}/trigger-build-$c + done + else + local baseline_rev cur_rev + baseline_rev=$(git_rev_parse_long $single_component ${rr[baseline_branch]} baseline) + cur_rev=$(git -C $single_component rev-parse HEAD) + + cat > ${rr[top_artifacts]}/trigger-bisect < $artifacts/trigger-2-build-master < $artifacts/jenkins/mail-recipients.txt @@ -199,7 +197,7 @@ if [ x"$res" = x"0" ]; then # We need to be careful to avoid re-trigger loops, so verify that # last_good is an ancestor of bad_rev. assert git merge-base --is-ancestor $last_good $bad_rev - cat > $artifacts/trigger-0-bisect-again < $artifacts/trigger-bisect < $artifacts/trigger-1-reset-baseline <> $artifacts/trigger-build-1-reset # Trigger master build now instead of waiting for next timed SCM trigger. -cat > $artifacts/trigger-2-build-master < Date: Tue, 12 Feb 2019 09:36:58 +0000 Subject: [rr-many 4/N] Update round-robin.sh: push_baseline() Make push_baseline handle multiple projects being updated. Change-Id: I9e6d2740154ac1987092c37ac09179e18b7f4da3 --- round-robin.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index 57da594b..7fcfb44f 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -407,7 +407,7 @@ $(cat ${rr[top_artifacts]}/results)" ) } -# Push to ${rr[current_project]} baseline branch and to base-artifacts repo. +# Push to baseline branches and to base-artifacts repo. push_baseline () { ( @@ -415,6 +415,10 @@ push_baseline () git_init_linaro_local_remote base-artifacts baseline false git_push base-artifacts baseline ${rr[baseline_branch]} - git_push ${rr[current_project]} baseline ${rr[baseline_branch]} + + local c + for c in $(print_updated_components); do + git_push $c baseline ${rr[baseline_branch]} + done ) } -- cgit v1.2.3 From 86b171c5fddc46efdae13448c60c18fe632ff1d9 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 12 Feb 2019 09:40:44 +0000 Subject: [rr-many 5/N] Update update_baseline Re-work commit title to handle multiple projects Change-Id: I5532a36e961734246d6c1b028d70a3f9906b8180 --- round-robin.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index 7fcfb44f..379b4971 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -396,12 +396,20 @@ update_baseline () # in base-artifacts.git repo (though they will be uploaded to jenkins). rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ - local build_rev - build_rev=$(git -C ${rr[current_project]} rev-parse HEAD) - cd base-artifacts - git add . - git commit $amend -m "${rr[current_project]}-$build_rev: $(tail -n1 ${rr[top_artifacts]}/results) + local single_component + single_component=$(print_single_updated_component) + if [ x"$single_component" != x"" ]; then + local single_rev + single_rev=$(git -C $single_component rev-parse HEAD) + msg_title="$msg_title: $single_component-$single_rev" + else + msg_title="$msg_title: $(print_updated_components "-")" + fi + msg_title="$msg_title: $(tail -n1 ${rr[top_artifacts]}/results)" + + git -C base-artifacts add . + git -C base-artifacts commit $amend -m "$msg_title $(cat ${rr[top_artifacts]}/results)" ) -- cgit v1.2.3 From bf550484f240557949aecf547e9363ef3dbc4f25 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 12 Feb 2019 09:46:05 +0000 Subject: [rr-many 6/N] Improve update_baseline Be more deliberate in preserving "first-bad" commit in the base-artifacts repo. Create a marker file reset-baseline for such builds. Also prepend the commit message title with number of pending commits, which is roughly twice the number of pending regressions. Change-Id: I14e16d893cab12637b71037665c8aee9b6ef24f1 --- round-robin.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/round-robin.sh b/round-robin.sh index 379b4971..8d903903 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -387,7 +387,8 @@ update_baseline () # For every regression we want to keep artifacts for the first-bad # build, so discard one less than needed. - if [ x"$prev_head" != x"" ]; then + if [ x"$prev_head" != x"" -a -f base-artifacts/reset-baseline ] \ + && ! ${rr[reset_baseline]}; then git -C base-artifacts reset --hard $prev_head fi @@ -396,6 +397,23 @@ update_baseline () # in base-artifacts.git repo (though they will be uploaded to jenkins). rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ + local rev_count + if [ x"$amend" = x"" ]; then + rev_count=$(git -C base-artifacts rev-list --count HEAD) + else + rev_count="0" + fi + + local msg_title="$rev_count" + + if ${rr[reset_baseline]}; then + # Create a marker for builds that reset baselines (these are builds + # for bisected regressions). + touch base-artifacts/reset-baseline + msg_title="$msg_title: first-bad" + else + msg_title="$msg_title: last-good" + fi local single_component single_component=$(print_single_updated_component) -- cgit v1.2.3 From 84ddec275438d5460a1b16048941f328e6c6ece3 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 19 Feb 2019 09:56:31 +0000 Subject: round-robin.sh: Add asserts to troubleshoot strange failures We are seeing strange behavior with base-artifacts -- results file is not being updated, so we get corrupted baselines. It's unclear why or how this happens, so add these asserts. If nothing else, these asserts will prevent push_baseline from pushing bogus results to the base-artifacts repo. The following script can be used to identified branches with bogus results: === set -euf -o pipefail git reset --hard git clean -fd branches=($(git branch -r | grep linaro-local/ci)) for branch in "${branches[@]}"; do git checkout --detach $branch > /dev/null 2>&1 while git rev-parse HEAD^ >/dev/null 2>&1; do if [ x"$(git diff HEAD HEAD^ -- results)" = x"" ]; then echo "$branch: $(git rev-parse HEAD)" git log --oneline HEAD | head -n2 tail -n1 results fi git checkout --detach HEAD^ > /dev/null 2>&1 done done === Change-Id: I840041f63dffc5aeb3a72a4cbd6859ea0191d336 --- round-robin.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/round-robin.sh b/round-robin.sh index 8d903903..caa6b47a 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -430,6 +430,17 @@ update_baseline () git -C base-artifacts commit $amend -m "$msg_title $(cat ${rr[top_artifacts]}/results)" + + # We saw strange behavior with base-artifacts/results not being updated + # or git-add'ed properly. Add a couple of asserts to catch such problems. + if [ x"$(diff -rup -x .git ${rr[top_artifacts]}/ base-artifacts/ | wc -l)" != x"0" ] \ + || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then + cd base-artifacts + git status + cat results + ls -la + assert false + fi ) } -- cgit v1.2.3 From fafb6d725bdfaac632ab0ff31c468b1665da1d91 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 19 Feb 2019 10:11:52 +0000 Subject: tcwg_kernel-bisect.sh: Fix handling of spurious failures For linux-next rebase workaround we reset baseline to linux-next:stable, and for spurious failures we re-trigger build with default parameters. Change-Id: I448870d101b7dfd554ecc4a91e71ce74538ecec6 --- tcwg_kernel-bisect.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index dee14845..7fb28b17 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -143,11 +143,17 @@ if [ x"$res" = x"0" ]; then if $rebase_workaround; then echo "Rebase workaround: no regression between $baseline_rev and $bad_rev" sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name + cat > $artifacts/trigger-build-rebase < $artifacts/trigger-build-retry < $artifacts/jenkins/mail-recipients.txt trap "" EXIT -- cgit v1.2.3 From ecaa922692d868d9616ae53348d8b1160ef16de8 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 19 Feb 2019 11:40:04 +0000 Subject: round-robin.sh: Fix asserts Change-Id: I2e1887bc565c3b0423690c48c768b1eeaea33531 --- round-robin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/round-robin.sh b/round-robin.sh index caa6b47a..fe8a5974 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -433,7 +433,7 @@ $(cat ${rr[top_artifacts]}/results)" # We saw strange behavior with base-artifacts/results not being updated # or git-add'ed properly. Add a couple of asserts to catch such problems. - if [ x"$(diff -rup -x .git ${rr[top_artifacts]}/ base-artifacts/ | wc -l)" != x"0" ] \ + if [ x"$(diff -up ${rr[top_artifacts]}/results base-artifacts/results)" != x"" ] \ || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then cd base-artifacts git status -- cgit v1.2.3 From 7c7293dd644f77c788acea62fab77cd2637bb50e Mon Sep 17 00:00:00 2001 From: Diana Picus Date: Tue, 19 Feb 2019 11:44:11 +0100 Subject: convert_args_to_variables: Document "==" syntax Add a comment explaining "==arr[key] value" syntax for parameters. Change-Id: Id18bd5fc912297cd98e55161f8aa1fd30532e706 --- jenkins-helpers.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 03c3a51a..36b7cd2f 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -739,6 +739,8 @@ manifest_out () # variables and arrays. # "--var value" defines shell variable "$var" to "value". # "++arr elem" defines shell array "$arr[@]" and adds "elem" to it. +# "==arr[key] value" defines shell associative array "$arr[@]" and sets +# "${arr[key]}" to "value". # "@@ file" sources file. # "%% file" starts manifest in file. Also see "^^ true". # "^^ true/false %% manifest" whether to reproduce the build using manifest. -- cgit v1.2.3 From e05d0df704a0f4ccfddc040c7e451fdd2370f050 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 19 Feb 2019 12:01:36 +0000 Subject: jenkins-helpers.sh: Remove stale locks in git repos. We get locks in git repo when builds are interrupted during git remote update. Change-Id: I184f6e0f4e401237d85f0e0af328a67be654faec --- jenkins-helpers.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 36b7cd2f..adcbe352 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -405,6 +405,9 @@ clone_or_update_repo_no_checkout () # Also, prune all loose objects to avoid "git gc --auto" failing # and creating .git/gc.log, which will stop future "git gc --auto" runs. git -C "$dir" gc --auto --force --prune=all + # Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock + # These occur when builds are aborted during "git remote update" or similar. + find "$dir/.git" -name "*.lock" -delete fi ( -- cgit v1.2.3 From 76e0bb578316168c2de9e9d20a2259fd02f2b4f2 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 19 Feb 2019 16:29:24 +0000 Subject: round-robin.sh: Fix rsync in update_baseline. See inline comments for details. Change-Id: I22029c29185daf61e81309dbb266f233e06fafba --- round-robin.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index fe8a5974..8227d38e 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -392,10 +392,13 @@ update_baseline () git -C base-artifacts reset --hard $prev_head fi - # Rsync current artifacts. + # Rsync current artifacts. Make sure to use -I rsync option since + # quite often size and timestamp on artifacts/results will be the same + # as on base-artifacts/results due to "git reset --hard HEAD^" below. + # This caused rsync's "quick check" heuristic to skip "results" file. # !!! From this point on, logs and other artifacts won't be included # in base-artifacts.git repo (though they will be uploaded to jenkins). - rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ + rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ local rev_count if [ x"$amend" = x"" ]; then @@ -432,7 +435,8 @@ update_baseline () $(cat ${rr[top_artifacts]}/results)" # We saw strange behavior with base-artifacts/results not being updated - # or git-add'ed properly. Add a couple of asserts to catch such problems. + # in the rsync above. This should be fixed by "-I" rsync option, but + # keep below asserts just in case. if [ x"$(diff -up ${rr[top_artifacts]}/results base-artifacts/results)" != x"" ] \ || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then cd base-artifacts -- cgit v1.2.3 From 1a60129203bf7933a360230e62cca84d9e8a00e7 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Wed, 20 Feb 2019 12:27:49 +0000 Subject: round-robin.sh: Fix base-artifacts grooming in update_baseline To have a history of pending regressions we mark baseline resets with reset-baseline file/marker (remember that resets correspond to "first-bad" builds triggered by bisect job). We have mistakenly tried to look for reset-baseline markers in the current revision of base-artifacts, while we should have looked in $prev_head revision. This patch fixes this and simplifies overall grooming of regression history. Change-Id: I3a819e6dfcdc2e8f02370216db001c00c067912b --- round-robin.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index 8227d38e..1512ca0e 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -372,23 +372,24 @@ update_baseline () # the current one, but keep entries for results that are better # (so that we have a record of pending regressions). while no_regression_p; do + prev_head="" if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then - prev_head=$(git -C base-artifacts rev-parse HEAD) + # For every regression we want to keep artifacts for the first-bad + # build, so reset to the most relevant regression (marked by reset-baseline). + if [ -f base-artifacts/reset-baseline ] && ! ${rr[reset_baseline]}; then + prev_head=$(git -C base-artifacts rev-parse HEAD) + fi git -C base-artifacts reset --hard HEAD^ else # We got to the beginning of git history, so amend the current # commit. The initial state of baseline is "empty" branch, # which we treat as worst possible in no_regression_p(). amend="--amend" - prev_head="" break fi done - # For every regression we want to keep artifacts for the first-bad - # build, so discard one less than needed. - if [ x"$prev_head" != x"" -a -f base-artifacts/reset-baseline ] \ - && ! ${rr[reset_baseline]}; then + if [ x"$prev_head" != x"" ]; then git -C base-artifacts reset --hard $prev_head fi -- cgit v1.2.3 From b553f7818103b3fb2c774020db885b20b584276f Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Wed, 20 Feb 2019 12:42:58 +0000 Subject: jenkins-helpers.sh: Simplify run_step* functions Remove copying of entries from base-artifacts/STEP/ to artifacts/STEP/ for skipped steps. We used to rely on $run_step_prev_artifacts/build-ok markers to detect pass/fail conditions of the previous step -- something that we now explicitly handle with skip_/reset_/stop_on_on_fail modes of run_step(). So remove unused feature. Change-Id: I06e4298aa06ffb2aaf677d2c51e89da61ae9b119 --- jenkins-helpers.sh | 29 ++++------------------------- tcwg_kernel-build.sh | 13 ++----------- 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index adcbe352..58ec2a29 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -1030,18 +1030,13 @@ git_push () # $1: Step to start execution at (or "" to start at the very first step) # $2: Step to finish execution at (or "" to run till the very end) # $3: Top artifact directory -# $4: Top baseline artifact directory (see Note 1) -# $5: Whether to enable "set -x" verbosity for execution steps. -# -# Note 1: Artifacts for steps before $run_step_start_at/$1 will be copied over -# from $base_artifacts/$4 +# $4: Whether to enable "set -x" verbosity for execution steps. run_step_init () { run_step_start_at="$1" run_step_finish_at="$2" run_step_top_artifacts="$3" - run_step_base_artifacts="$4" - run_step_verbose="$5" + run_step_verbose="$4" run_step_count="0" run_step_prev_step="" @@ -1049,12 +1044,6 @@ run_step_init () run_step_status=0 run_step_artifacts="" - if [ x"$run_step_base_artifacts" != x"" ]; then - run_step_use_baseline=true - else - run_step_use_baseline=false - fi - # We need absolute paths for $run_step_artifacts, which is constructed from # $run_step_top_artifacts. mkdir -p "$run_step_top_artifacts" @@ -1071,9 +1060,8 @@ run_step_init () # 2. artifact handling -- create/clean artifact directories per step. # Also, copy baseline artifacts for steps before START_AT to simulate # skipped steps. -# Step commands have $run_step_artifacts and $run_step_prev_artifacts -# pointing to artifact directories for current and previous step -# respectively. +# Step commands have $run_step_artifacts pointing to artifact directory +# for current step. # 3. logging -- dump stdout and and stderr output of step commands # into per-step console.log files, and, also, into the top-level # console.log file. @@ -1110,10 +1098,8 @@ run_step () if [ x"$pretty_step" = x"$run_step_start_at" \ -o x"$run_step_start_at" = x"" -a x"$run_step_prev_step" = x"" ]; then run_step_active=true - run_step_use_baseline=false fi - run_step_prev_artifacts=$run_step_artifacts run_step_artifacts=$run_step_top_artifacts/$run_step_count-$pretty_step rm -rf "$run_step_artifacts" @@ -1157,13 +1143,6 @@ run_step () assert false esac fi - elif $run_step_use_baseline; then - echo "COPYING BASE-ARTIFACTS for ${step[@]}" - local base_artifacts - base_artifacts="$run_step_base_artifacts/$(basename $run_step_artifacts)" - if [ -d "$base_artifacts" ]; then - rsync -a --del "$base_artifacts/" "$run_step_artifacts/" - fi else echo "SKIPPING ${step[@]}" fi diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index dabc2995..a5737837 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -10,7 +10,7 @@ convert_args_to_variables "$@" obligatory_variables rr[ci_config] -# Execution mode: baseline, bisect, continue, jenkins-full +# Execution mode: baseline, bisect, jenkins-full rr[mode]="${rr[mode]-baseline}" # Set custom revision for one of the projects, and use baseline revisions @@ -52,7 +52,6 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT # Set start and finish steps for different modes. default_start_at="" default_finish_at="" -base_artifacts_opt="base-artifacts" case "${rr[mode]}" in "baseline") default_finish_at="update_baseline" @@ -68,14 +67,6 @@ case "${rr[mode]}" in esac default_finish_at="check_regression" ;; - "continue") - # Developer mode. - case "${rr[toolchain]}" in - "gnu") default_start_at="prepare_abe" ;; - "llvm") default_start_at="build_llvm" ;; - esac - base_artifacts_opt="" - ;; "jenkins-full") ;; esac if [ x"$start_at" = x"default" ]; then @@ -85,7 +76,7 @@ if [ x"$finish_at" = x"default" ]; then finish_at="$default_finish_at" fi -run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose" +run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose" # Build Linux kernel build_linux () -- cgit v1.2.3 From f42643fadb4fd1b4112b49719bb3fa7107f52838 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Wed, 20 Feb 2019 13:50:17 +0000 Subject: Trivial cleanups The "last-good" result is, in reality, just "good". We no longer kernel.release file (to troubleshoot ccache issues). Remove "if true". Change-Id: If64798625e693c9abfaf085f6e5b23f77eadf73f --- round-robin.sh | 2 +- tcwg_kernel-build.sh | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index 1512ca0e..24b2d4e2 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -416,7 +416,7 @@ update_baseline () touch base-artifacts/reset-baseline msg_title="$msg_title: first-bad" else - msg_title="$msg_title: last-good" + msg_title="$msg_title: good" fi local single_component diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index a5737837..7cdb9d4f 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -140,7 +140,6 @@ EOF KBUILD_BUILD_TIMESTAMP=0 make $opts -j$(nproc --all) -s -k & local res=0 && wait $! || res=$? ccache -s - cp include/config/kernel.release $run_step_artifacts/ return $res ) } @@ -151,19 +150,17 @@ count_linux_objs () ( set -euf -o pipefail - if true; then - build_linux & - local res=0; wait $! || res=$? - - # Number of .o files created is the main success metric. - echo "linux_n_obj:" >> ${rr[top_artifacts]}/results - if [ $res != 0 ]; then - local linux_n_obj - linux_n_obj=$(find linux -name "*.o" | wc -l) - echo "$linux_n_obj" >> ${rr[top_artifacts]}/results - else - echo "all" >> ${rr[top_artifacts]}/results - fi + build_linux & + local res=0; wait $! || res=$? + + # Number of .o files created is the main success metric. + echo "linux_n_obj:" >> ${rr[top_artifacts]}/results + if [ $res != 0 ]; then + local linux_n_obj + linux_n_obj=$(find linux -name "*.o" | wc -l) + echo "$linux_n_obj" >> ${rr[top_artifacts]}/results + else + echo "all" >> ${rr[top_artifacts]}/results fi ) } -- cgit v1.2.3 From 2d68fd0a3cceb58f0a4299320a8e681023238eed Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Thu, 21 Feb 2019 11:17:46 +0000 Subject: tcwg_kernel-build.sh: Add boot-testing of kernel This implements https://projects.linaro.org/browse/TCWG-1503 . Count_linux_objs() was renamed to build_linux (with build_linux() renamed to build_linux_1()). New step boot_linux() was added to boot kernel with qemu. Defconfig configurations for aarch64 and arm seem to boot, allnoconfig doesn't boot, others -- will see. No_regression_p() was reworked to handle result hierarchy "boot" -> "all" -> "Number of objects". Change-Id: Ib3377c72de51745a537714d1c693303b3935188c --- tcwg_kernel-build.sh | 100 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 36 deletions(-) diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index 7cdb9d4f..130e3403 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -79,7 +79,7 @@ fi run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose" # Build Linux kernel -build_linux () +build_linux_1 () { ( set -euf -o pipefail @@ -144,27 +144,57 @@ EOF ) } -# Count number successfully built .o files in linux (and build linux) -count_linux_objs () +# Build linux and count number successfully built .o files in linux +build_linux () { ( set -euf -o pipefail - build_linux & - local res=0; wait $! || res=$? + build_linux_1 & + local res=0 && wait $! || res=$? # Number of .o files created is the main success metric. + local linux_n_obj + linux_n_obj=$(find linux -name "*.o" | wc -l) echo "linux_n_obj:" >> ${rr[top_artifacts]}/results - if [ $res != 0 ]; then - local linux_n_obj - linux_n_obj=$(find linux -name "*.o" | wc -l) - echo "$linux_n_obj" >> ${rr[top_artifacts]}/results - else + echo "$linux_n_obj" >> ${rr[top_artifacts]}/results + + if [ $res = 0 ]; then + echo "linux build successful:" >> ${rr[top_artifacts]}/results echo "all" >> ${rr[top_artifacts]}/results fi + + return $res ) } +# Boot linux kernel +boot_linux () +{ + ( + set -euf -o pipefail + + local image cpu + case ${rr[target]} in + aarch64) + image=linux/arch/arm64/boot/Image.gz + cpu="-cpu cortex-a53" + ;; + arm) + image=linux/arch/arm/boot/zImage + cpu="" + ;; + *) assert false ;; + esac + timeout 10s qemu-system-${rr[target]} \ + -kernel $image -machine virt $cpu -m 512 \ + -serial mon:stdio -display none \ + -append "console=ttyAMA0 panic=-1" -no-reboot + + echo "linux boot successful:" >> ${rr[top_artifacts]}/results + echo "boot" >> ${rr[top_artifacts]}/results + ) +} # Exit with code 0 if no regression compared to base-artifacts/results. no_regression_p () @@ -175,37 +205,34 @@ no_regression_p () local linux_n_obj linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results) - if [ x"$linux_n_obj" = x"all" ]; then - # Ideal result, no need to compare to baseline. - return 0 - fi - - if ! [ "$linux_n_obj" -ge "-10" ]; then - # Something is very wrong with result (e.g., it's not a number). - return 1 - fi - # Assume worst for non-existent baseline. local base_linux_n_obj="-10" - if [ -f base-artifacts/results ]; then base_linux_n_obj=$(tail -n1 base-artifacts/results) - if [ x"$base_linux_n_obj" = x"all" ]; then - # Current build is not ideal, so make sure baseline "all" is - # better than the current result. - base_linux_n_obj=$(($linux_n_obj+1)) - elif ! [ "$base_linux_n_obj" -ge "-10" ]; then - # Something is very wrong with baseline result - # (e.g., it's not a number). - return 0 - fi fi - if [ $linux_n_obj -ge $base_linux_n_obj ]; then - return 0 - else - return 1 - fi + case "$linux_n_obj:$base_linux_n_obj" in + boot:*) return 0 ;; + *:boot) return 1 ;; + all:*) return 0 ;; + *:all) return 1 ;; + *) + if ! [ "$linux_n_obj" -ge "-10" ]; then + # Something is very wrong with result (e.g., it's not a number). + return 1 + fi + if ! [ "$base_linux_n_obj" -ge "-10" ]; then + # Something is very wrong with result (e.g., it's not a number). + return 0 + fi + + if [ $linux_n_obj -ge $base_linux_n_obj ]; then + return 0 + else + return 1 + fi + ;; + esac ) } @@ -220,7 +247,8 @@ case "${rr[toolchain]}" in run_step skip_on_fail -1 build_llvm ;; esac -run_step skip_on_fail x count_linux_objs +run_step skip_on_fail x build_linux +run_step skip_on_fail x boot_linux run_step reset_on_fail x check_regression run_step stop_on_fail x update_baseline run_step stop_on_fail x push_baseline -- cgit v1.2.3 From 8fb56bc0a39beaa4232cbba4a346931251b0f0d4 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Fri, 22 Feb 2019 16:44:32 +0000 Subject: tcwg_kernel-build.sh: Update build step name I missed this change in the rename commit. Change-Id: I82e5e93fe52e8e53d0ca9cfc95b70dda8485e2fb --- tcwg_kernel-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index 130e3403..5f29c9bb 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -60,9 +60,9 @@ case "${rr[mode]}" in case "${rr[toolchain]}:$(print_single_updated_component)" in gnu:binutils) default_start_at="build_abe-binutils" ;; gnu:gcc) default_start_at="build_abe-stage1" ;; - gnu:linux) default_start_at="count_linux_objs" ;; + gnu:linux) default_start_at="build_linux" ;; llvm:llvm) default_start_at="build_llvm" ;; - llvm:linux) default_start_at="count_linux_objs" ;; + llvm:linux) default_start_at="build_linux" ;; *) assert false ;; esac default_finish_at="check_regression" -- cgit v1.2.3 From 569ae0fb042c80005f771c6052c9212f64df772d Mon Sep 17 00:00:00 2001 From: Peter Smith Date: Wed, 20 Feb 2019 17:03:39 +0000 Subject: round-robin.sh: Add LLD to the linux kernel round-robin build We use the same form as clang, copying lld from the monorepo into tools/lld. Change-Id: Ieff55d1aafe2a49c87f3c7e518933a5adebcf8ff --- round-robin.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/round-robin.sh b/round-robin.sh index 24b2d4e2..5c665887 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -276,8 +276,9 @@ build_llvm () clone_repo llvm # Copy only components from the monorepo that are required for kernel build - rsync -a --del --exclude /tools/clang llvm/llvm/ llvm-src/ + rsync -a --del --exclude /tools/clang --exclude /tools/lld llvm/llvm/ llvm-src/ rsync -a --del llvm/clang/ llvm-src/tools/clang/ + rsync -a --del llvm/lld/ llvm-src/tools/lld/ # Setup ccache and ninja wrappers. rm -rf $(pwd)/bin -- cgit v1.2.3 From 57ade8502c9139986159481bdf6b950ca47c75a9 Mon Sep 17 00:00:00 2001 From: Peter Smith Date: Fri, 22 Feb 2019 14:06:19 +0000 Subject: tcwg_kernel-build.sh: Use LLD for LLVM Linux kernel link jobs It is expected that developers using clang to build the linux kernel will also use LLD for the linker. We would like to test this in the LLVM builds of the linux kernel as much as possible. To use LLD in the kernel build we set LD=/path/to/linker when calling make. At the moment we use LLD for llvm-*-aarch64-*-* builds and for llvm-master-arm-*-* builds. Change-Id: Ib1d543480278abec193cde6aeba0f457bcd603aa --- tcwg_kernel-build.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index 5f29c9bb..9170117a 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -25,6 +25,7 @@ IFS=- read -a ci_config < Date: Tue, 26 Feb 2019 09:51:58 +0000 Subject: tcwg_kernel-build.sh: Restrict linking with LLD to defconfig Many configurations seems to fail. Below are some examples: - llvm-master-aarch64-lts-allmodconfig -- produces 2k less .o files - llvm-master-arm-next-allmodconfig -- produces 4k less .o files - llvm-master-aarch64-next-allmodconfig -- produces 5k less .o files - llvm-release-aarch64-lts-allnoconfig -- can't boot - llvm-release-aarch64-mainline-allnoconfig -- can't boot Change-Id: I5e88e83579e6acbf75aaf850b3defcf5d4ed850a --- tcwg_kernel-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index 9170117a..9485e0ba 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -103,9 +103,9 @@ build_linux_1 () ;; esac - case "${rr[toolchain]}-${rr[release]}-${rr[target]}" in - llvm-*-aarch64) ld_opt="LD=$bin/ld.lld" ;; - llvm-master-arm) ld_opt="LD=$bin/ld.lld" ;; + case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in + llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;; + llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;; *) ld_opt="" ;; esac -- cgit v1.2.3 From 90935acc02c6219a06613ae6a0c85a65f8566acc Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Tue, 26 Feb 2019 09:31:47 +0000 Subject: tcwg-cleanup-stale-containers.sh: Fix volume cleanup "Echo" concatenates all elements into a single string, so we need to break them up for "grep" to match its pattern. Change-Id: I954ca80fc6f0e289185780f76988688ce0f1ed78 --- tcwg-cleanup-stale-containers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh index 86fc04b8..0dd642f7 100755 --- a/tcwg-cleanup-stale-containers.sh +++ b/tcwg-cleanup-stale-containers.sh @@ -148,7 +148,7 @@ status=$(($status|(2*$res))) rm_volumes=($($DOCKER volume ls -q -f dangling=true)) # Filter-out named volumes like host-home and home-$USER. Leave only volumes # named like a sha1 hash. -rm_volumes=($(echo "${rm_volumes[@]}" | grep "^[a-f0-9]\{64\}\$" | cat)) +rm_volumes=($(echo "${rm_volumes[@]}" | tr " " "\n" | grep "^[a-f0-9]\{64\}\$" | cat)) if [ ${#rm_volumes[@]} != 0 ]; then echo "Removing dangling volumes" -- cgit v1.2.3 From 644c2eba2e682c14cf064de47c21f91cf5106fd9 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Mon, 18 Feb 2019 18:13:24 +0000 Subject: tcwg-dev-build: Enable builds using manifests If manifest.txt was uploaded in job's parameters, then pass --manifest option to MakeRelease.job. Fix MakeRelease.job to not pass --target when manifest is set, otherwise ABE complains. Change-Id: I2aa25d3840281159e2ed00cfaab3b9e47a661801 --- MakeRelease.job | 15 +++++++++------ jenkins-helpers.sh | 11 +++++++++++ tcwg-dev-build.sh | 7 +++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/MakeRelease.job b/MakeRelease.job index 83d80f20..6ca48bc3 100755 --- a/MakeRelease.job +++ b/MakeRelease.job @@ -99,10 +99,6 @@ else rsh="ssh $fileserver" fi -if test x"${target}" != x"native" -a x"${target}" != x; then - platform="--target ${target}" -fi - shared="/home/buildslave/workspace/shared/" user_snapshots="${user_workspace}/snapshots" @@ -156,6 +152,13 @@ gcc=${gcc_src:+gcc=${gcc_src}} srcs="${gcc} ${binutils} ${glibc} ${manifest}" logfile=${user_workspace}/MakeRelease-${buildnumber}-$(uname -m).log +# Set ABE's --target setting. No setting means native. +# If manifest file is set, then ABE must use its setting. +target_opt= +if [ x"${target}" != x"native" -a x"${target}" != x -a x"$manifest" = x"" ]; then + target_opt="--target ${target}" +fi + # Build a binary release tarball # Remove logfile if present (for some unknown reason) rm -f ${logfile} @@ -164,7 +167,7 @@ manifests=() tarballs=() # Canadian cross builds require a Linux hosted cross compiler first if test x"${canadian}" = x"true"; then - $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} ${platform} --build all ${libc} ${extra} --tarbin >> ${logfile} + $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} $target_opt --build all ${libc} ${extra} --tarbin >> ${logfile} abe_ret=$? host="--host i686-w64-mingw32" manifests+=($(read_var ${user_workspace}/artifacts1.txt manifest)) @@ -177,7 +180,7 @@ fi # build the mingw32 compiler only if the previous cross-compiler build was # successful. if test ${abe_ret} -eq 0; then - $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} ${platform} ${host} --build all ${libc} ${extra} >> ${logfile} + $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} $target_opt ${host} --build all ${libc} ${extra} >> ${logfile} abe_ret=$? manifests+=($(read_var ${user_workspace}/artifacts2.txt manifest)) fi diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 58ec2a29..33218625 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -1,5 +1,16 @@ #!/bin/bash +# Print absolute path to a file or directory +# $1: Path (must exist) +abs_path () +{ + ( + set -euf -o pipefail + + echo "$(cd $(dirname "$1") && pwd)/$(basename "$1")" + ) +} + # Assert that $@ returns success. assert () { diff --git a/tcwg-dev-build.sh b/tcwg-dev-build.sh index cf24ab60..24b7b930 100755 --- a/tcwg-dev-build.sh +++ b/tcwg-dev-build.sh @@ -8,6 +8,7 @@ convert_args_to_variables "$@" abe_branch="${abe_branch-tested}" dry_run="${dry_run-false}" +manifest="" release_name="${release_name-default}" target="${target-aarch64-linux-gnu}" version="${version-default}" @@ -23,6 +24,11 @@ if [ x"$release_name" = x"default" ]; then release_name="$(date +%Y%m%d-%H_%M_%S)" fi +manifest_opt= +if [ -f "$manifest" ]; then + manifest_opt="--manifest $(abs_path "$manifest")" +fi + manifest_validation_opt= if [ x"$target" = x"native" ]; then manifest_validation_opt="--manifest_validation false" @@ -35,6 +41,7 @@ $scripts/MakeRelease.job \ --target $target \ --toolchainconfig $version \ --workspace `pwd` \ + $manifest_opt \ $manifest_validation_opt \ ${binutils:+--binutils "$binutils"} \ ${gcc:+--gcc "$gcc"} -- cgit v1.2.3 From b74e544b3a9a53cf28be75becca0c57298e76fd3 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Fri, 1 Mar 2019 06:22:46 +0000 Subject: tcwg-dev-build.sh: Fix thinko Use --manifest value when it's provided. Change-Id: I7c4667c6c607e14ee31eafe2f0fee958abb962cb --- tcwg-dev-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcwg-dev-build.sh b/tcwg-dev-build.sh index 24b7b930..ff4afacc 100755 --- a/tcwg-dev-build.sh +++ b/tcwg-dev-build.sh @@ -8,7 +8,7 @@ convert_args_to_variables "$@" abe_branch="${abe_branch-tested}" dry_run="${dry_run-false}" -manifest="" +manifest="${manifest-}" release_name="${release_name-default}" target="${target-aarch64-linux-gnu}" version="${version-default}" -- cgit v1.2.3 From 56317f8c7c31f67194c0f56cb537e53bfe000e8d Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Wed, 27 Feb 2019 14:58:56 +0000 Subject: tcwg_kernel-bisect.sh: Handle bisecting between branches/tags Rename bad_rev and baseline_rev to bad_branch and baseline_branch, and resolve them to SHA1 revisions. The motivation for this change is ability to gather historical data, e.g., code size regressions from GCC/LLVM 5 to 6, to 7, to 8, and so on. The bisects are intended to be triggered manually, which is why branch handling is desired. Change-Id: I9c46270e0092e72836986a4bed08799ec6d84a00 --- round-robin.sh | 4 ++-- tcwg_kernel-bisect.sh | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/round-robin.sh b/round-robin.sh index 5c665887..81b04e95 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -349,8 +349,8 @@ check_regression () cat > ${rr[top_artifacts]}/trigger-bisect < $artifacts/trigger-bisect < $artifacts/jenkins/mail-recipients.txt -- cgit v1.2.3 From f64d30e3f08e7d731f6b37ccade481b9db2a1138 Mon Sep 17 00:00:00 2001 From: Maxim Kuvyrkov Date: Wed, 27 Feb 2019 15:06:33 +0000 Subject: tcwg_kernel-bisect.sh: Fix linux-next rebase workaround for gnu builds Specify binutils_branch and gcc_branch for gnu builds. Change-Id: I52cd3d0d2837f1fe3c5c492dd657cc6c71408100 --- tcwg_kernel-bisect.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index bfc8e6db..136dc24c 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -152,8 +152,21 @@ if [ x"$res" = x"0" ]; then if $rebase_workaround; then echo "Rebase workaround: no regression between $baseline_rev and $bad_rev" sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name - cat > $artifacts/trigger-build-rebase < $artifacts/trigger-build-rebase < $artifacts/trigger-build-rebase <> $artifacts/trigger-build-rebase < Date: Mon, 4 Mar 2019 10:28:48 +0000 Subject: tcwg-cleanup-stale-containers.sh: Report failure to kill stale ssh-agent processes. Without this patch, failure to kill such processes is not reported as part of the script exit status. This happens for instance when the killall command is not installed on the machine. Change-Id: Ic0fa10ce6b69f12fb2382cfa3680fcc1c7d1b4bd --- tcwg-cleanup-stale-containers.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh index 0dd642f7..cc748c5c 100755 --- a/tcwg-cleanup-stale-containers.sh +++ b/tcwg-cleanup-stale-containers.sh @@ -200,6 +200,10 @@ fi if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then res=0; killall --older-than ${cleanup_ssh_agent_hours}h -u $USER ssh-agent & wait $! || res=$? + if [ $res != 0 ]; then + echo "WARNING: could not kill stale ssh-agent processes" + status=$(($status|16)) + fi fi # Check if we have more containers than max_containers @@ -207,7 +211,7 @@ nb_containers=$($DOCKER ps -a | wc -l) if [ ${max_containers} -gt 0 -a ${nb_containers} -gt ${max_containers} ]; then echo "ERROR: Too many containers left after cleanup: ${nb_containers} (max: ${max_containers})" - status=$(($status|16)) + status=$(($status|32)) fi exit $status -- cgit v1.2.3 From 532a858fab12ed1e011c4a9e19461f19bf26f478 Mon Sep 17 00:00:00 2001 From: Christophe Lyon Date: Tue, 5 Mar 2019 16:48:42 +0000 Subject: Use APM boards for cross-testing. In print_tester_label_for_target(), do not use tk1/tx1 anymore, use APMs instead. However, this means we no longer use armv7 hardware to test arm-* configurations (as opposed to armv8l*). Change-Id: Iff08678c39ad15f59628e5a91f62d0812d0bb0b3 --- jenkins-helpers.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 33218625..c9c90560 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -227,9 +227,12 @@ print_tester_label_for_target () # doesn't support KVM. Test on APM builders for now. echo "tcwg-apm_64-build" ;; - aarch64-linux*) echo "tcwg-tx1_64-test" ;; - armv8l-linux*) echo "tcwg-tx1_32-test" ;; - arm-linux*) echo "tcwg-tk1_32-test" ;; + # We allocate all TK1/TX1 boards to benchmarking, so use APMs + # for cross-testing. This means we no longer test on armv7 + # hardware. + aarch64-linux*) echo "tcwg-apm_64-test" ;; + armv8l-linux*) echo "tcwg-apm_32-test" ;; + arm-linux*) echo "tcwg-apm_32-test" ;; esac ) } -- cgit v1.2.3