summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xMakeRelease.job15
-rw-r--r--jenkins-helpers.sh54
-rw-r--r--round-robin.sh184
-rwxr-xr-xtcwg-cleanup-stale-containers.sh8
-rwxr-xr-xtcwg-dev-build.sh7
-rwxr-xr-xtcwg_kernel-bisect.sh120
-rwxr-xr-xtcwg_kernel-build.sh168
7 files changed, 348 insertions, 208 deletions
diff --git a/MakeRelease.job b/MakeRelease.job
index 83d80f20..6ca48bc3 100755
--- a/MakeRelease.job
+++ b/MakeRelease.job
@@ -99,10 +99,6 @@ else
rsh="ssh $fileserver"
fi
-if test x"${target}" != x"native" -a x"${target}" != x; then
- platform="--target ${target}"
-fi
-
shared="/home/buildslave/workspace/shared/"
user_snapshots="${user_workspace}/snapshots"
@@ -156,6 +152,13 @@ gcc=${gcc_src:+gcc=${gcc_src}}
srcs="${gcc} ${binutils} ${glibc} ${manifest}"
logfile=${user_workspace}/MakeRelease-${buildnumber}-$(uname -m).log
+# Set ABE's --target setting. No setting means native.
+# If manifest file is set, then ABE must use its setting.
+target_opt=
+if [ x"${target}" != x"native" -a x"${target}" != x -a x"$manifest" = x"" ]; then
+ target_opt="--target ${target}"
+fi
+
# Build a binary release tarball
# Remove logfile if present (for some unknown reason)
rm -f ${logfile}
@@ -164,7 +167,7 @@ manifests=()
tarballs=()
# Canadian cross builds require a Linux hosted cross compiler first
if test x"${canadian}" = x"true"; then
- $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} ${platform} --build all ${libc} ${extra} --tarbin >> ${logfile}
+ $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts1.txt ${update} --release ${release} ${srcs} $target_opt --build all ${libc} ${extra} --tarbin >> ${logfile}
abe_ret=$?
host="--host i686-w64-mingw32"
manifests+=($(read_var ${user_workspace}/artifacts1.txt manifest))
@@ -177,7 +180,7 @@ fi
# build the mingw32 compiler only if the previous cross-compiler build was
# successful.
if test ${abe_ret} -eq 0; then
- $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} ${platform} ${host} --build all ${libc} ${extra} >> ${logfile}
+ $CONFIG_SHELL ${abe_dir}/abe.sh --list-artifacts ${user_workspace}/artifacts2.txt ${update} --release ${release} --tarbin ${srcs} $target_opt ${host} --build all ${libc} ${extra} >> ${logfile}
abe_ret=$?
manifests+=($(read_var ${user_workspace}/artifacts2.txt manifest))
fi
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index 8a66b2a9..9c0b7ff6 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -1,5 +1,16 @@
#!/bin/bash
+# Print absolute path to a file or directory
+# $1: Path (must exist)
+abs_path ()
+{
+ (
+ set -euf -o pipefail
+
+ echo "$(cd $(dirname "$1") && pwd)/$(basename "$1")"
+ )
+}
+
# Assert that $@ returns success.
assert ()
{
@@ -216,9 +227,12 @@ print_tester_label_for_target ()
# doesn't support KVM. Test on APM builders for now.
echo "tcwg-apm_64-build"
;;
- aarch64-linux*) echo "tcwg-tx1_64-test" ;;
- armv8l-linux*) echo "tcwg-tx1_32-test" ;;
- arm-linux*) echo "tcwg-tk1_32-test" ;;
+ # We allocate all TK1/TX1 boards to benchmarking, so use APMs
+ # for cross-testing. This means we no longer test on armv7
+ # hardware.
+ aarch64-linux*) echo "tcwg-apm_64-test" ;;
+ armv8l-linux*) echo "tcwg-apm_32-test" ;;
+ arm-linux*) echo "tcwg-apm_32-test" ;;
esac
)
}
@@ -405,6 +419,9 @@ clone_or_update_repo_no_checkout ()
# Also, prune all loose objects to avoid "git gc --auto" failing
# and creating .git/gc.log, which will stop future "git gc --auto" runs.
git -C "$dir" gc --auto --force --prune=all
+ # Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock
+ # These occur when builds are aborted during "git remote update" or similar.
+ find "$dir/.git" -name "*.lock" -delete
fi
(
@@ -739,6 +756,8 @@ manifest_out ()
# variables and arrays.
# "--var value" defines shell variable "$var" to "value".
# "++arr elem" defines shell array "$arr[@]" and adds "elem" to it.
+# "==arr[key] value" defines shell associative array "$arr[@]" and sets
+# "${arr[key]}" to "value".
# "@@ file" sources file.
# "%% file" starts manifest in file. Also see "^^ true".
# "^^ true/false %% manifest" whether to reproduce the build using manifest.
@@ -1025,18 +1044,13 @@ git_push ()
# $1: Step to start execution at (or "" to start at the very first step)
# $2: Step to finish execution at (or "" to run till the very end)
# $3: Top artifact directory
-# $4: Top baseline artifact directory (see Note 1)
-# $5: Whether to enable "set -x" verbosity for execution steps.
-#
-# Note 1: Artifacts for steps before $run_step_start_at/$1 will be copied over
-# from $base_artifacts/$4
+# $4: Whether to enable "set -x" verbosity for execution steps.
run_step_init ()
{
run_step_start_at="$1"
run_step_finish_at="$2"
run_step_top_artifacts="$3"
- run_step_base_artifacts="$4"
- run_step_verbose="$5"
+ run_step_verbose="$4"
run_step_count="0"
run_step_prev_step=""
@@ -1044,12 +1058,6 @@ run_step_init ()
run_step_status=0
run_step_artifacts=""
- if [ x"$run_step_base_artifacts" != x"" ]; then
- run_step_use_baseline=true
- else
- run_step_use_baseline=false
- fi
-
# We need absolute paths for $run_step_artifacts, which is constructed from
# $run_step_top_artifacts.
mkdir -p "$run_step_top_artifacts"
@@ -1066,9 +1074,8 @@ run_step_init ()
# 2. artifact handling -- create/clean artifact directories per step.
# Also, copy baseline artifacts for steps before START_AT to simulate
# skipped steps.
-# Step commands have $run_step_artifacts and $run_step_prev_artifacts
-# pointing to artifact directories for current and previous step
-# respectively.
+# Step commands have $run_step_artifacts pointing to artifact directory
+# for current step.
# 3. logging -- dump stdout and and stderr output of step commands
# into per-step console.log files, and, also, into the top-level
# console.log file.
@@ -1105,10 +1112,8 @@ run_step ()
if [ x"$pretty_step" = x"$run_step_start_at" \
-o x"$run_step_start_at" = x"" -a x"$run_step_prev_step" = x"" ]; then
run_step_active=true
- run_step_use_baseline=false
fi
- run_step_prev_artifacts=$run_step_artifacts
run_step_artifacts=$run_step_top_artifacts/$run_step_count-$pretty_step
rm -rf "$run_step_artifacts"
@@ -1152,13 +1157,6 @@ run_step ()
assert false
esac
fi
- elif $run_step_use_baseline; then
- echo "COPYING BASE-ARTIFACTS for ${step[@]}"
- local base_artifacts
- base_artifacts="$run_step_base_artifacts/$(basename $run_step_artifacts)"
- if [ -d "$base_artifacts" ]; then
- rsync -a --del "$base_artifacts/" "$run_step_artifacts/"
- fi
else
echo "SKIPPING ${step[@]}"
fi
diff --git a/round-robin.sh b/round-robin.sh
index cfbfe117..b9c12e99 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -7,6 +7,11 @@
#rr[PROJECT_url]
# PROJECT's git branch or SHA1 revision parsable by git rev-parse.
+# A special value "baseline" means that PROJECT is not being updated
+# in this build, and its baseline branch should be used.
+# In a successful build "update_baseline" step will update baseline
+# branches of all PROJECTs to the current values, thus setting
+# a baseline for the next build.
#rr[PROJECT_branch]
# PROJECT's git SHA1 revision. These are mostly used in manifests.
@@ -16,13 +21,6 @@
# be present in all above git repos (if ${rr[init_configuration]} is false).
#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}"
-# PROJECT that we are testing in this build. Use ${rr[current_branch]} for
-# this project, and ${rr[baseline_branch]} for all other projects.
-#rr[current_project]="$current_project"
-
-# Git branch or SHA1 revision of ${rr[current_project]} to test.
-#rr[current_branch]="$current_branch"
-
# Run mode: bisect or non-bisect. In bisect mode we do a couple things
# slightly differently (e.g., don't touch repo in clone_repo() ).
#rr[mode]="$mode"
@@ -40,6 +38,40 @@
# to initialize baseline branches in git repos.
#rr[init_configuration]=false
+# Print round-robin components that are being updated in this build
+# (the ones using non-baseline branches).
+print_updated_components ()
+{
+ (
+ set -euf -o pipefail
+
+ local delim=""
+ local c
+ for c in ${rr[components]}; do
+ if [ x"${rr[${c}_branch]}" != x"baseline" ]; then
+ printf "%s%s" "$delim" "$c"
+ delim=${1- }
+ fi
+ done
+ )
+}
+
+# Print the single round-robin component being updated in this build.
+# Print nothing if multiple components are being updated.
+print_single_updated_component ()
+{
+ (
+ set -euf -o pipefail
+
+ local -a updated_components
+ updated_components=($(print_updated_components))
+
+ if [ ${#updated_components[@]} = 1 ]; then
+ echo "${updated_components[0]}"
+ fi
+ )
+}
+
# Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
@@ -91,24 +123,15 @@ clone_repo ()
fi
local branch
- # Select the branch to build.
- # ${rr[current_branch]} specifies branch for ${rr[current_project]},
- # and everything else uses baseline branch.
- if [ x"$project" = x"${rr[current_project]}" ]; then
- # Use the tip of the branch tracked in the current configuration.
- if [ x"${rr[current_branch]}" = x"default" ]; then
- branch=${rr[${rr[current_project]}_branch]}
- elif [ x"${rr[current_branch]}" = x"baseline" ]; then
- branch="refs/remotes/baseline/${rr[baseline_branch]}"
- else
- branch=${rr[current_branch]}
- fi
- elif ${rr[init_configuration]-false}; then
- branch="${rr[${project}_branch]}"
- else
+
+ # Resolve "baseline" branch specifier.
+ if [ x"${rr[${project}_branch]}" = x"baseline" ]; then
branch="refs/remotes/baseline/${rr[baseline_branch]}"
+ else
+ branch="${rr[${project}_branch]}"
fi
+ # Allow manifest override
branch="${rr[${project}_rev]-$branch}"
# Decide on whether to use read-only or read-write mode for
@@ -116,7 +139,7 @@ clone_repo ()
# developers without ssh keys on Linaro git servers to reproduce builds
# in --mode "baseline".
local read_only="true"
- if [ x"$project" = x"${rr[current_project]}" -a \
+ if [ x"${rr[${project}_branch]}" != x"baseline" -a \
x"${rr[mode]}" = x"jenkins-full" ]; then
read_only=false
fi
@@ -134,20 +157,6 @@ clone_repo ()
cat <<EOF | manifest_out
rr[${project}_rev]=$cur_rev
EOF
-
- if [ x"$project" = x"${rr[current_project]}" ] \
- && ! ${rr[init_configuration]-false}; then
- local baseline_rev
- baseline_rev=$(git_rev_parse_long $project ${rr[baseline_branch]} baseline)
- # Prepare for failure. If build fails we will bisect sha1 for
- # ${rr[current_branch]} and sha1 for ${rr[baseline_branch]}.
- cat > ${rr[top_artifacts]}/trigger-bisect-on-failure <<EOF
-current_project=${rr[current_project]}
-baseline_rev=$baseline_rev
-bad_rev=$cur_rev
-EOF
- git -C $project rev-list --count HEAD ^$baseline_rev > ${rr[top_artifacts]}/distance-to-baseline
- fi
)
}
@@ -298,8 +307,9 @@ build_llvm ()
clone_repo llvm
# Copy only components from the monorepo that are required for kernel build
- rsync -a --del --exclude /tools/clang llvm/llvm/ llvm-src/
+ rsync -a --del --exclude /tools/clang --exclude /tools/lld llvm/llvm/ llvm-src/
rsync -a --del llvm/clang/ llvm-src/tools/clang/
+ rsync -a --del llvm/lld/ llvm-src/tools/lld/
# Setup ccache and ninja wrappers.
rm -rf $(pwd)/bin
@@ -349,6 +359,32 @@ check_regression ()
set -euf -o pipefail
if ! ${rr[reset_baseline]} && ! no_regression_p; then
+ local single_component=$(print_single_updated_component)
+
+ if [ x"$single_component" = x"" -o x"${rr[mode]}" = x"bisect" ]; then
+ local c
+ for c in $(print_updated_components); do
+ local c2
+ for c2 in ${rr[components]}; do
+ if [ x"$c" = x"$c2" ]; then
+ echo "${c2}_branch=${rr[${c}_branch]}"
+ else
+ echo "${c2}_branch=baseline"
+ fi
+ done >> ${rr[top_artifacts]}/trigger-build-$c
+ done
+ else
+ local baseline_rev cur_rev
+ baseline_rev=$(git_rev_parse_long $single_component ${rr[baseline_branch]} baseline)
+ cur_rev=$(git -C $single_component rev-parse HEAD)
+
+ cat > ${rr[top_artifacts]}/trigger-bisect <<EOF
+current_project=$single_component
+baseline_branch=$baseline_rev
+bad_branch=$cur_rev
+EOF
+ fi
+
# Fail.
false
fi
@@ -368,42 +404,84 @@ update_baseline ()
# the current one, but keep entries for results that are better
# (so that we have a record of pending regressions).
while no_regression_p; do
+ prev_head=""
if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then
- prev_head=$(git -C base-artifacts rev-parse HEAD)
+ # For every regression we want to keep artifacts for the first-bad
+ # build, so reset to the most relevant regression (marked by reset-baseline).
+ if [ -f base-artifacts/reset-baseline ] && ! ${rr[reset_baseline]}; then
+ prev_head=$(git -C base-artifacts rev-parse HEAD)
+ fi
git -C base-artifacts reset --hard HEAD^
else
# We got to the beginning of git history, so amend the current
# commit. The initial state of baseline is "empty" branch,
# which we treat as worst possible in no_regression_p().
amend="--amend"
- prev_head=""
break
fi
done
- # For every regression we want to keep artifacts for the first-bad
- # build, so discard one less than needed.
if [ x"$prev_head" != x"" ]; then
git -C base-artifacts reset --hard $prev_head
fi
- # Rsync current artifacts.
+ # Rsync current artifacts. Make sure to use -I rsync option since
+ # quite often size and timestamp on artifacts/results will be the same
+ # as on base-artifacts/results due to "git reset --hard HEAD^" below.
+ # This caused rsync's "quick check" heuristic to skip "results" file.
# !!! From this point on, logs and other artifacts won't be included
# in base-artifacts.git repo (though they will be uploaded to jenkins).
- rsync -a --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/
+ rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/
- local build_rev
- build_rev=$(git -C ${rr[current_project]} rev-parse HEAD)
+ local rev_count
+ if [ x"$amend" = x"" ]; then
+ rev_count=$(git -C base-artifacts rev-list --count HEAD)
+ else
+ rev_count="0"
+ fi
- cd base-artifacts
- git add .
- git commit $amend -m "${rr[current_project]}-$build_rev: $(tail -n1 ${rr[top_artifacts]}/results)
+ local msg_title="$rev_count"
+
+ if ${rr[reset_baseline]}; then
+ # Create a marker for builds that reset baselines (these are builds
+ # for bisected regressions).
+ touch base-artifacts/reset-baseline
+ msg_title="$msg_title: first-bad"
+ else
+ msg_title="$msg_title: good"
+ fi
+
+ local single_component
+ single_component=$(print_single_updated_component)
+ if [ x"$single_component" != x"" ]; then
+ local single_rev
+ single_rev=$(git -C $single_component rev-parse HEAD)
+ msg_title="$msg_title: $single_component-$single_rev"
+ else
+ msg_title="$msg_title: $(print_updated_components "-")"
+ fi
+ msg_title="$msg_title: $(tail -n1 ${rr[top_artifacts]}/results)"
+
+ git -C base-artifacts add .
+ git -C base-artifacts commit $amend -m "$msg_title
$(cat ${rr[top_artifacts]}/results)"
+
+ # We saw strange behavior with base-artifacts/results not being updated
+ # in the rsync above. This should be fixed by "-I" rsync option, but
+ # keep below asserts just in case.
+ if [ x"$(diff -up ${rr[top_artifacts]}/results base-artifacts/results)" != x"" ] \
+ || [ x"$amend" = x"" -a x"$(git -C base-artifacts diff HEAD HEAD^ -- results)" = x"" ]; then
+ cd base-artifacts
+ git status
+ cat results
+ ls -la
+ assert false
+ fi
)
}
-# Push to ${rr[current_project]} baseline branch and to base-artifacts repo.
+# Push to baseline branches and to base-artifacts repo.
push_baseline ()
{
(
@@ -411,6 +489,10 @@ push_baseline ()
git_init_linaro_local_remote base-artifacts baseline false
git_push base-artifacts baseline ${rr[baseline_branch]}
- git_push ${rr[current_project]} baseline ${rr[baseline_branch]}
+
+ local c
+ for c in $(print_updated_components); do
+ git_push $c baseline ${rr[baseline_branch]}
+ done
)
}
diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh
index 86fc04b8..cc748c5c 100755
--- a/tcwg-cleanup-stale-containers.sh
+++ b/tcwg-cleanup-stale-containers.sh
@@ -148,7 +148,7 @@ status=$(($status|(2*$res)))
rm_volumes=($($DOCKER volume ls -q -f dangling=true))
# Filter-out named volumes like host-home and home-$USER. Leave only volumes
# named like a sha1 hash.
-rm_volumes=($(echo "${rm_volumes[@]}" | grep "^[a-f0-9]\{64\}\$" | cat))
+rm_volumes=($(echo "${rm_volumes[@]}" | tr " " "\n" | grep "^[a-f0-9]\{64\}\$" | cat))
if [ ${#rm_volumes[@]} != 0 ]; then
echo "Removing dangling volumes"
@@ -200,6 +200,10 @@ fi
if [ "$cleanup_ssh_agent_hours" -gt "0" ]; then
res=0; killall --older-than ${cleanup_ssh_agent_hours}h -u $USER ssh-agent &
wait $! || res=$?
+ if [ $res != 0 ]; then
+ echo "WARNING: could not kill stale ssh-agent processes"
+ status=$(($status|16))
+ fi
fi
# Check if we have more containers than max_containers
@@ -207,7 +211,7 @@ nb_containers=$($DOCKER ps -a | wc -l)
if [ ${max_containers} -gt 0 -a ${nb_containers} -gt ${max_containers} ]; then
echo "ERROR: Too many containers left after cleanup: ${nb_containers} (max: ${max_containers})"
- status=$(($status|16))
+ status=$(($status|32))
fi
exit $status
diff --git a/tcwg-dev-build.sh b/tcwg-dev-build.sh
index cf24ab60..ff4afacc 100755
--- a/tcwg-dev-build.sh
+++ b/tcwg-dev-build.sh
@@ -8,6 +8,7 @@ convert_args_to_variables "$@"
abe_branch="${abe_branch-tested}"
dry_run="${dry_run-false}"
+manifest="${manifest-}"
release_name="${release_name-default}"
target="${target-aarch64-linux-gnu}"
version="${version-default}"
@@ -23,6 +24,11 @@ if [ x"$release_name" = x"default" ]; then
release_name="$(date +%Y%m%d-%H_%M_%S)"
fi
+manifest_opt=
+if [ -f "$manifest" ]; then
+ manifest_opt="--manifest $(abs_path "$manifest")"
+fi
+
manifest_validation_opt=
if [ x"$target" = x"native" ]; then
manifest_validation_opt="--manifest_validation false"
@@ -35,6 +41,7 @@ $scripts/MakeRelease.job \
--target $target \
--toolchainconfig $version \
--workspace `pwd` \
+ $manifest_opt \
$manifest_validation_opt \
${binutils:+--binutils "$binutils"} \
${gcc:+--gcc "$gcc"}
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 45a6aaf0..136dc24c 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -16,7 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"
convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"
-obligatory_variables bad_rev baseline_rev build_script rr[ci_project]
+obligatory_variables bad_branch baseline_branch build_script current_project
BUILD_URL="${BUILD_URL:-$(pwd)}"
reproduce_bisect="${reproduce_bisect:-false}"
@@ -25,13 +25,7 @@ reproduce_bisect="${reproduce_bisect:-false}"
convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@"
$reproduce_bisect || manifest_pop
-obligatory_variables rr[current_project] rr[ci_config]
-
-current_project=${rr[current_project]}
-
-if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then
- obligatory_variables toolchain
-fi
+obligatory_variables rr[ci_project] rr[ci_config]
verbose="${verbose:-true}"
@@ -43,38 +37,41 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/fai
rebase_workaround=false
-if [ x"${rr[ci_project]}" = x"tcwg_kernel" ] \
- && [ x"$current_project" = x"linux" -a x"${rr[linux_version]}" = x"next" ]; then
- # Workaround linux-next/master rebasing on top of linux-next/stable.
- # Search for regressions against linux-mainline:master (aka linux-next:stable).
- clone_or_update_repo $current_project stable ${rr[linux_url]}
- # Just in case linux-next:stable has advanced between the build and bisect jobs,
- # use merge base between linux-next:stable and $bad_rev.
- linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}"
- cat <<EOF | manifest_out
+case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
+ tcwg_kernel/*-next-*:linux)
+ # Workaround linux-next/master rebasing on top of linux-next/stable.
+ # Search for regressions against linux-mainline:master (aka linux-next:stable).
+ clone_or_update_repo $current_project stable ${rr[linux_url]}
+ # Just in case linux-next:stable has advanced between the build and bisect jobs,
+ # use merge base between linux-next:stable and $bad_branch.
+ bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}"
+ linux_next_stable="${linux_next_stable-$(git -C $current_project merge-base HEAD $bad_rev)}"
+ cat <<EOF | manifest_out
declare -g linux_next_stable=$linux_next_stable
EOF
- echo "Rebase workaround: forcing baseline_rev to $linux_next_stable"
- baseline_rev=$linux_next_stable
- rebase_workaround=true
-fi
+ echo "Rebase workaround: forcing baseline_rev to $linux_next_stable"
+ baseline_rev=$linux_next_stable
+ rebase_workaround=true
+ ;;
+esac
# Build baseline that we are going to re-use to speed-up bisection.
# (This also confirms that infrastructure is OK.)
-echo "Testing baseline_rev $baseline_rev (should be success)"
+echo "Testing baseline_branch $baseline_branch (should be success)"
$build_script \
^^ $reproduce_bisect \
%% $rel_artifacts/manifests/build-baseline.sh \
@@ $rel_artifacts/manifests/build-parameters.sh \
==rr[mode] "baseline" \
- ==rr[current_branch] "$baseline_rev" \
+ ==rr[${current_project}_branch] "$baseline_branch" \
==rr[reset_baseline] true \
==rr[top_artifacts] "$rel_artifacts/build-baseline" \
--verbose "$verbose"
-assert ! [ -f $artifacts/failures ]
-
-cd $current_project
+baseline_rev="${baseline_rev-$(git -C ${current_project} rev-parse HEAD)}"
+cat <<EOF | manifest_out
+declare -g baseline_rev=$baseline_rev
+EOF
ln -f -s "build-baseline" "$artifacts/build-$baseline_rev"
ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh"
@@ -124,6 +121,7 @@ $build_script \
%% $rel_artifacts/manifests/build-\$rev.sh \
@@ $rel_artifacts/manifests/build-parameters.sh \
==rr[mode] bisect \
+ ==rr[${current_project}_branch] \$rev \
==rr[top_artifacts] $rel_artifacts/build-\$rev \
--verbose "$verbose" &
res=0 && wait \$! || res=\$?
@@ -138,6 +136,13 @@ fi
EOF
chmod +x $artifacts/test.sh
+bad_rev="${bad_rev-$(git_rev_parse_long $current_project $bad_branch)}"
+cat <<EOF | manifest_out
+declare -g bad_rev=$bad_rev
+EOF
+
+cd $current_project
+
git checkout --detach $bad_rev
$artifacts/test.sh &
res=0 && wait $! || res=$?
@@ -147,14 +152,30 @@ if [ x"$res" = x"0" ]; then
if $rebase_workaround; then
echo "Rebase workaround: no regression between $baseline_rev and $bad_rev"
sed -i -e "s/\$/-bad_rev-good/" $artifacts/jenkins/build-name
+ case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
+ tcwg_kernel/llvm-*-next-*:linux)
+ cat > $artifacts/trigger-build-rebase <<EOF
+llvm_branch=baseline
+EOF
+ ;;
+ tcwg_kernel/gnu-*-next-*:linux)
+ cat > $artifacts/trigger-build-rebase <<EOF
+binutils_branch=baseline
+gcc_branch=baseline
+EOF
+ ;;
+ *) assert false ;;
+ esac
+ cat >> $artifacts/trigger-build-rebase <<EOF
+linux_branch=$baseline_rev
+reset_baseline=true
+EOF
else
echo "WARNING: build for bad_rev $bad_rev succeeded"
- # Build job had a spurious failure. Re-try.
- cat > $artifacts/trigger-2-build-master <<EOF
-current_project=$current_project
-current_branch=$bad_rev
-EOF
sed -i -e "s/\$/-spurious/" $artifacts/jenkins/build-name
+ # Retry build with default parameters
+ cat > $artifacts/trigger-build-retry <<EOF
+EOF
fi
echo > $artifacts/jenkins/mail-recipients.txt
trap "" EXIT
@@ -204,10 +225,10 @@ if [ x"$res" = x"0" ]; then
# We need to be careful to avoid re-trigger loops, so verify that
# last_good is an ancestor of bad_rev.
assert git merge-base --is-ancestor $last_good $bad_rev
- cat > $artifacts/trigger-0-bisect-again <<EOF
+ cat > $artifacts/trigger-bisect <<EOF
current_project=$current_project
-baseline_rev=$baseline_rev
-bad_rev=$last_good
+baseline_branch=$baseline_rev
+bad_branch=$last_good
EOF
# Don't send any emails.
echo > $artifacts/jenkins/mail-recipients.txt
@@ -271,6 +292,10 @@ Could not identify regression in $current_project for CI configuration ${rr[ci_c
EOF
fi
+# In log scan for errors below
+# - ": error:" detects compiler errors from GCC and Clang
+# - "^ERROR:" detects linker errors
+# - "] Error " detects GNU make errors
cat >> $artifacts/jenkins/mail-body.txt <<EOF
Configuration details:
$(cat $artifacts/manifests/build-baseline.sh | grep '_url]\|_branch]' | grep -v '="no_')
@@ -282,7 +307,7 @@ to (for $bad_name == $bad_sha1)
$(cat $artifacts/build-$bad_sha1/results)
First few errors in logs of $bad_name:
-$(grep ": error:" $artifacts/build-$bad_sha1/console.log | head)
+$(grep ": error:\|^ERROR:\|\] Error " $artifacts/build-$bad_sha1/console.log | head)
Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/
Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/
@@ -337,29 +362,22 @@ fi
# Set mail recipients last to preserve catch-error value from .yaml file.
# Email developers.
CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org"
-if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then
- case "$toolchain:$current_project" in
- gnu:linux) ;;
- gnu:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;;
- llvm:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
- llvm:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;;
- esac
-fi
+case "${rr[ci_project]}/${rr[ci_config]}:$current_project" in
+ tcwg_kernel/gnu-*:linux) ;;
+ tcwg_kernel/gnu-*:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;;
+ tcwg_kernel/llvm-*:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
+ tcwg_kernel/llvm-*:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;;
+esac
cat > $artifacts/jenkins/mail-recipients.txt <<EOF
$CI_MAIL_RECIPIENTS
EOF
# Reset baseline to the regressed commit so that we will catch subsequent
# regressions (worse than $bad_rev).
-cat > $artifacts/trigger-1-reset-baseline <<EOF
-current_project=$current_project
-current_branch=$first_bad
-reset_baseline=true
-EOF
+cp $artifacts/build-$first_bad/trigger-build-$current_project $artifacts/trigger-build-1-reset
+echo "reset_baseline=true" >> $artifacts/trigger-build-1-reset
# Trigger master build now instead of waiting for next timed SCM trigger.
-cat > $artifacts/trigger-2-build-master <<EOF
-current_project=$current_project
-EOF
+cp $artifacts/build-$bad_rev/trigger-build-$current_project $artifacts/trigger-build-2-default
trap "" EXIT
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index ab169e5b..9485e0ba 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -4,26 +4,19 @@ set -euf -o pipefail
scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh
+. $scripts/round-robin.sh
convert_args_to_variables "$@"
obligatory_variables rr[ci_config]
-# Execution mode: baseline, bisect, continue, jenkins-full
+# Execution mode: baseline, bisect, jenkins-full
rr[mode]="${rr[mode]-baseline}"
-case "${rr[mode]}" in
- "jenkins-full")
- obligatory_variables rr[current_project] rr[current_branch]
- ;;
-esac
-
# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[ci_project]="${rr[ci_project]-tcwg_kernel}"
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
-rr[current_branch]="${rr[current_branch]-default}"
-rr[current_project]="${rr[current_project]-none}"
rr[reset_baseline]="${rr[reset_baseline]-false}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
@@ -32,9 +25,22 @@ IFS=- read -a ci_config <<EOF
${rr[ci_config]}
EOF
rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
+rr[release]=${rr[release]-${ci_config[1]}}
rr[target]=${rr[target]-${ci_config[2]}}
rr[linux_config]=${rr[linux_config]-${ci_config[4]}}
+case "${rr[toolchain]}" in
+ llvm) rr[components]="llvm linux" ;;
+ gnu) rr[components]="binutils gcc linux" ;;
+ *) assert false ;;
+esac
+
+# Use baseline branches by default.
+for c in ${rr[components]}; do
+ rr[${c}_branch]=${rr[${c}_branch]-baseline}
+ obligatory_variables rr[${c}_url]
+done
+
start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
@@ -47,29 +53,21 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT
# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
-base_artifacts_opt="base-artifacts"
case "${rr[mode]}" in
"baseline")
default_finish_at="update_baseline"
;;
"bisect")
- case "${rr[toolchain]}:${rr[current_project]}" in
+ case "${rr[toolchain]}:$(print_single_updated_component)" in
gnu:binutils) default_start_at="build_abe-binutils" ;;
gnu:gcc) default_start_at="build_abe-stage1" ;;
- gnu:linux) default_start_at="count_linux_objs" ;;
+ gnu:linux) default_start_at="build_linux" ;;
llvm:llvm) default_start_at="build_llvm" ;;
- llvm:linux) default_start_at="count_linux_objs" ;;
+ llvm:linux) default_start_at="build_linux" ;;
+ *) assert false ;;
esac
default_finish_at="check_regression"
;;
- "continue")
- # Developer mode.
- case "${rr[toolchain]}" in
- "gnu") default_start_at="prepare_abe" ;;
- "llvm") default_start_at="build_llvm" ;;
- esac
- base_artifacts_opt=""
- ;;
"jenkins-full") ;;
esac
if [ x"$start_at" = x"default" ]; then
@@ -79,12 +77,10 @@ if [ x"$finish_at" = x"default" ]; then
finish_at="$default_finish_at"
fi
-run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose"
-
-. $scripts/round-robin.sh
+run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose"
# Build Linux kernel
-build_linux ()
+build_linux_1 ()
{
(
set -euf -o pipefail
@@ -94,7 +90,7 @@ build_linux ()
rm -rf $(pwd)/bin
mkdir $(pwd)/bin
- local bin cc
+ local bin cc ld_opt
case "${rr[toolchain]}" in
llvm)
@@ -107,6 +103,12 @@ build_linux ()
;;
esac
+ case "${rr[toolchain]}-${rr[release]}-${rr[target]}-${rr[linux_config]}" in
+ llvm-*-aarch64-defconfig) ld_opt="LD=$bin/ld.lld" ;;
+ llvm-master-arm-defconfig) ld_opt="LD=$bin/ld.lld" ;;
+ *) ld_opt="" ;;
+ esac
+
# Use binutils, etc from $bin
export PATH="$bin:$PATH"
@@ -114,7 +116,8 @@ build_linux ()
# Otherwise the compiler is new in every build and we would
# only clobber ccache volume.
local ccache=""
- if [ x"${rr[mode]}" != x"jenkins-full" -a x"${rr[current_project]}" = x"linux" ]; then
+ if [ x"${rr[mode]}" != x"jenkins-full" \
+ -a x"$(print_single_updated_component)" = x"linux" ]; then
ccache="ccache"
fi
cat > $(pwd)/bin/${rr[target]}-cc <<EOF
@@ -124,7 +127,7 @@ EOF
chmod +x $(pwd)/bin/${rr[target]}-cc
# Define make variables.
- local opts="CC=$(pwd)/bin/${rr[target]}-cc SUBLEVEL=0 EXTRAVERSION=-bisect"
+ local opts="CC=$(pwd)/bin/${rr[target]}-cc $ld_opt SUBLEVEL=0 EXTRAVERSION=-bisect"
if [ x"${rr[target]}" != x"$(uname -m)" ]; then
opts="$opts ARCH=$(print_kernel_target ${rr[target]})"
opts="$opts CROSS_COMPILE=$(print_gnu_target ${rr[target]})-"
@@ -144,34 +147,61 @@ EOF
KBUILD_BUILD_TIMESTAMP=0 make $opts -j$(nproc --all) -s -k &
local res=0 && wait $! || res=$?
ccache -s
- cp include/config/kernel.release $run_step_artifacts/
return $res
)
}
-# Count number successfully built .o files in linux (and build linux)
-count_linux_objs ()
+# Build linux and count number successfully built .o files in linux
+build_linux ()
{
(
set -euf -o pipefail
- if true; then
- build_linux &
- local res=0; wait $! || res=$?
-
- # Number of .o files created is the main success metric.
- echo "linux_n_obj:" >> ${rr[top_artifacts]}/results
- if [ $res != 0 ]; then
- local linux_n_obj
- linux_n_obj=$(find linux -name "*.o" | wc -l)
- echo "$linux_n_obj" >> ${rr[top_artifacts]}/results
- else
- echo "all" >> ${rr[top_artifacts]}/results
- fi
+ build_linux_1 &
+ local res=0 && wait $! || res=$?
+
+ # Number of .o files created is the main success metric.
+ local linux_n_obj
+ linux_n_obj=$(find linux -name "*.o" | wc -l)
+ echo "linux_n_obj:" >> ${rr[top_artifacts]}/results
+ echo "$linux_n_obj" >> ${rr[top_artifacts]}/results
+
+ if [ $res = 0 ]; then
+ echo "linux build successful:" >> ${rr[top_artifacts]}/results
+ echo "all" >> ${rr[top_artifacts]}/results
fi
+
+ return $res
)
}
+# Boot linux kernel
+boot_linux ()
+{
+ (
+ set -euf -o pipefail
+
+ local image cpu
+ case ${rr[target]} in
+ aarch64)
+ image=linux/arch/arm64/boot/Image.gz
+ cpu="-cpu cortex-a53"
+ ;;
+ arm)
+ image=linux/arch/arm/boot/zImage
+ cpu=""
+ ;;
+ *) assert false ;;
+ esac
+ timeout 10s qemu-system-${rr[target]} \
+ -kernel $image -machine virt $cpu -m 512 \
+ -serial mon:stdio -display none \
+ -append "console=ttyAMA0 panic=-1" -no-reboot
+
+ echo "linux boot successful:" >> ${rr[top_artifacts]}/results
+ echo "boot" >> ${rr[top_artifacts]}/results
+ )
+}
# Exit with code 0 if no regression compared to base-artifacts/results.
no_regression_p ()
@@ -182,37 +212,34 @@ no_regression_p ()
local linux_n_obj
linux_n_obj=$(tail -n1 ${rr[top_artifacts]}/results)
- if [ x"$linux_n_obj" = x"all" ]; then
- # Ideal result, no need to compare to baseline.
- return 0
- fi
-
- if ! [ "$linux_n_obj" -ge "-10" ]; then
- # Something is very wrong with result (e.g., it's not a number).
- return 1
- fi
-
# Assume worst for non-existent baseline.
local base_linux_n_obj="-10"
-
if [ -f base-artifacts/results ]; then
base_linux_n_obj=$(tail -n1 base-artifacts/results)
- if [ x"$base_linux_n_obj" = x"all" ]; then
- # Current build is not ideal, so make sure baseline "all" is
- # better than the current result.
- base_linux_n_obj=$(($linux_n_obj+1))
- elif ! [ "$base_linux_n_obj" -ge "-10" ]; then
- # Something is very wrong with baseline result
- # (e.g., it's not a number).
- return 0
- fi
fi
- if [ $linux_n_obj -ge $base_linux_n_obj ]; then
- return 0
- else
- return 1
- fi
+ case "$linux_n_obj:$base_linux_n_obj" in
+ boot:*) return 0 ;;
+ *:boot) return 1 ;;
+ all:*) return 0 ;;
+ *:all) return 1 ;;
+ *)
+ if ! [ "$linux_n_obj" -ge "-10" ]; then
+ # Something is very wrong with result (e.g., it's not a number).
+ return 1
+ fi
+ if ! [ "$base_linux_n_obj" -ge "-10" ]; then
+ # Something is very wrong with result (e.g., it's not a number).
+ return 0
+ fi
+
+ if [ $linux_n_obj -ge $base_linux_n_obj ]; then
+ return 0
+ else
+ return 1
+ fi
+ ;;
+ esac
)
}
@@ -227,7 +254,8 @@ case "${rr[toolchain]}" in
run_step skip_on_fail -1 build_llvm
;;
esac
-run_step skip_on_fail x count_linux_objs
+run_step skip_on_fail x build_linux
+run_step skip_on_fail x boot_linux
run_step reset_on_fail x check_regression
run_step stop_on_fail x update_baseline
run_step stop_on_fail x push_baseline