summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPrathamesh Kulkarni <prathamesh.kulkarni@linaro.org>2019-02-12 19:41:45 +0530
committerPrathamesh Kulkarni <prathamesh.kulkarni@linaro.org>2019-02-12 19:41:45 +0530
commit8028d1160dbcf1defe5e4853a9f0bcfa68c115cb (patch)
treed18b5aefb3d3a8bf25d9ff7c26c93a0d3adbfe8e
parenta797f90814f5b0b8f31d4e97c3cf72db84f04d31 (diff)
parent99300555a21d0ef961d60042730afe5b79d3311a (diff)
Merge branch 'master' into tcwg_gnu
Change-Id: Ifd517229bb3ee7642266e3e66a8682f177992076
-rw-r--r--jenkins-helpers.sh14
-rwxr-xr-xjenkins.sh11
-rw-r--r--round-robin.sh43
-rwxr-xr-xtcwg-benchmark-results.sh1
-rwxr-xr-xtcwg-benchmark.sh7
-rwxr-xr-xtcwg_kernel-bisect.sh38
-rwxr-xr-xtcwg_kernel-build.sh66
-rw-r--r--tcwg_kernel/gnu-master.sh9
-rw-r--r--tcwg_kernel/gnu-release.sh9
-rw-r--r--tcwg_kernel/linux-lts.sh7
-rw-r--r--tcwg_kernel/linux-mainline.sh7
-rw-r--r--tcwg_kernel/linux-next.sh7
-rw-r--r--tcwg_kernel/linux-stable.sh7
-rw-r--r--tcwg_kernel/llvm-master.sh7
-rw-r--r--tcwg_kernel/llvm-release.sh7
15 files changed, 92 insertions, 148 deletions
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index a2fb51d7..5a471800 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -152,6 +152,11 @@ print_host_for_node ()
# configured ssh
local host="$1.tcwglab"
+ # Map tcwg-x86_64-build-* and tcwg-x86_64-dev-* to build-* and dev-*
+ case "$host" in
+ tcwg-x86_64-*) host=${host#tcwg-x86_64-} ;;
+ esac
+
if ! host "$host" >& /dev/null; then
# No DNS entry, try to see if it is handled by our ssh config
if ! ssh "$1" true >& /dev/null; then
@@ -236,7 +241,10 @@ remote_exec ()
cmd=()
# Add quotes to every parameter
for i in "$@"; do cmd+=("$(printf '%q' "$i")"); done
- ssh $opts ${port:+-p$port} $host "${dir:+cd "$(printf '%q' "$dir")" &&} exec ${cmd[@]}"
+ # Be careful to prepend statements before ${cmd[@]} only if necessary.
+ # E.g., when triggering jobs via jenkins-cli, the command is not a binary,
+ # so we can't "exec" it.
+ ssh $opts ${port:+-p$port} $host "${dir:+cd "$(printf '%q' "$dir")" && exec }${cmd[@]}"
)
}
@@ -669,7 +677,9 @@ print_docker_path ()
if [ -f "/.dockerenv" ] && mount | grep -q "/run/docker.sock "; then
# If inside "host" container (with proxied docker and /home from host-home volume),
# convert paths to refer to volume's path on bare-metal.
- echo "$path" | sed -e "s#^/home/#/var/lib/docker/volumes/host-home/_data/#"
+ local docker_root
+ docker_root=$(docker info | grep "Docker Root Dir:" | cut -d: -f 2)
+ echo "$path" | sed -e "s#^/home/#$docker_root/volumes/host-home/_data/#"
else
echo "$path"
fi
diff --git a/jenkins.sh b/jenkins.sh
index 9847a565..b2fab679 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -390,17 +390,6 @@ if test x"${debug}" = x"true"; then
export CONFIG_SHELL="/bin/bash -x"
fi
-# Download QEMU provided by Peter Maydell.
-# The tarball has README with version information.
-if [ x"$(uname -m)" = x"x86_64" ]; then
- wget --progress=dot:giga http://people.linaro.org/~maxim.kuvyrkov/qemu-20160707.tgz
- tar xf qemu-20160707.tgz
- export PATH="$(pwd)/qemu-wip:$PATH"
- for i in aarch64 arm armeb; do
- qemu-$i --version
- done
-fi
-
# Print some information about the build machine
echo Running on $(hostname)
uname -a
diff --git a/round-robin.sh b/round-robin.sh
index 1bc1f865..26cfe534 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -12,9 +12,9 @@
# PROJECT's git SHA1 revision. These are mostly used in manifests.
#rr[PROJECT_rev]
-# Baseline branch name for current configuration. These branches are
-# present in all above git repos.
-#rr[baseline_branch]
+# Baseline branch name for current configuration. These branches should
+# be present in all above git repos (if ${rr[init_configuration]} is false).
+#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}"
# PROJECT that we are testing in this build. Use ${rr[current_branch]} for
# this project, and ${rr[baseline_branch]} for all other projects.
@@ -36,6 +36,10 @@
# Top-level artifacts directory.
#rr[top_artifacts]="$top_artifacts"
+# Set rr[init_configuration] to 'true' for new projects/configurations
+# to initialize baseline branches in git repos.
+#rr[init_configuration]=false
+
# Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
@@ -91,6 +95,7 @@ clone_repo ()
# ${rr[current_branch]} specifies branch for ${rr[current_project]},
# and everything else uses baseline branch.
if [ x"$project" = x"${rr[current_project]}" ]; then
+ # Use the tip of the branch tracked in the current configuration.
if [ x"${rr[current_branch]}" = x"default" ]; then
branch=${rr[${rr[current_project]}_branch]}
elif [ x"${rr[current_branch]}" = x"baseline" ]; then
@@ -217,16 +222,23 @@ build_abe ()
;;
esac
- clone_repo $project
-
- # Don't use ABE's repo clone functions and setup abe/snapshots/ directory
- # to have the right entries.
- local git_dir git_path
- git_dir="$(basename ${rr[${project}_url]})"
- git_path="abe/snapshots/$git_dir"
- rm -rf $git_path $git_path~master
- ln -s $(pwd)/$project $git_path
- ln -s $(pwd)/$project $git_path~master
+ # Use our custom sources for everything, but kernel headers.
+ local custom_abe_src_opt=""
+ local git_dir="$project"
+ if [ x"$component" != x"linux" ]; then
+ clone_repo $project
+
+ # Don't use ABE's repo clone functions and setup abe/snapshots/ directory
+ # to have the right entries.
+ local git_dir git_path
+ git_dir="$(basename ${rr[${project}_url]})"
+ git_path="abe/snapshots/$git_dir"
+ rm -rf $git_path $git_path~master
+ ln -s $(pwd)/$project $git_path
+ ln -s $(pwd)/$project $git_path~master
+
+ custom_abe_src_opt="$project=${rr[${project}_url]}~master --disable update"
+ fi
cd abe
@@ -235,7 +247,7 @@ build_abe ()
# Remove previous build directories and .stamp files.
# We rely on ccache for fast rebuilds.
- set +f; rm -rf builds/*/$gnu_target/$git_dir~master*; set -f
+ set +f; rm -rf builds/*/*/$git_dir*; set -f
export PATH=$(pwd)/bin:$PATH
@@ -259,8 +271,7 @@ build_abe ()
--$action $project \
$target_opt \
--extraconfigdir config/master \
- --disable update \
- $project=${rr[${project}_url]}~master \
+ $custom_abe_src_opt \
$stage
ccache -s
)
diff --git a/tcwg-benchmark-results.sh b/tcwg-benchmark-results.sh
index f7147f8c..90e6bcd2 100755
--- a/tcwg-benchmark-results.sh
+++ b/tcwg-benchmark-results.sh
@@ -28,6 +28,7 @@ for i in $results_ref $results_id; do
rsync -az --delete "$results_top-$i/" results-$num/
$scripts/../bmk-scripts/perfdatadir2csv.sh --buildid-dir local --format sample,size --sort-field sample --results-dir "results-$num/" > "results-$num.csv"
csvs="$csvs results-$num.csv"
+ num=$(($num+1))
done
ssh -O exit dev-01.tcwglab
diff --git a/tcwg-benchmark.sh b/tcwg-benchmark.sh
index fd9000bc..fbb49111 100755
--- a/tcwg-benchmark.sh
+++ b/tcwg-benchmark.sh
@@ -46,7 +46,9 @@ case "$toolchain_url" in
"ssh://"*)
ccprefix="${toolchain_url##ssh://}"
- # Extract host:port: specification from ccprefix.
+ # Extract host:port: specification from ccprefix, we don't
+ # need to care about :parallelize here, just pass it to run.sh
+ # if present.
case ${ccprefix} in
*:*:*)
build=${ccprefix%:*}
@@ -85,7 +87,8 @@ case "$toolchain_url" in
echo "ERROR: Unsupported sysroot $sysroot for toolchain_url $toolchain_url"
exit 1
fi
- toolchaindir="$(dirname $(echo $ccprefix | cut -s -d: -f 3))"
+ # Last component of ccprefix is the path, keep it
+ toolchaindir="$(dirname ${ccprefix##*:})"
;;
"http://"*".tar.xz"|"https://"*".tar.xz")
toolchaindir=$(untar_url "$toolchain_url" "$WORKSPACE" "--strip-components 1")
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index e4d11e2b..45a6aaf0 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -16,8 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"
convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"
-obligatory_variables bad_rev baseline_rev build_script \
- rr[ci_project] rr[ci_config]
+obligatory_variables bad_rev baseline_rev build_script rr[ci_project]
BUILD_URL="${BUILD_URL:-$(pwd)}"
reproduce_bisect="${reproduce_bisect:-false}"
@@ -26,7 +25,9 @@ reproduce_bisect="${reproduce_bisect:-false}"
convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@"
$reproduce_bisect || manifest_pop
-obligatory_variables current_project
+obligatory_variables rr[current_project] rr[ci_config]
+
+current_project=${rr[current_project]}
if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then
obligatory_variables toolchain
@@ -65,10 +66,10 @@ $build_script \
^^ $reproduce_bisect \
%% $rel_artifacts/manifests/build-baseline.sh \
@@ $rel_artifacts/manifests/build-parameters.sh \
- --mode "baseline" \
- --current_branch "$baseline_rev" \
- --reset_baseline true \
- --top_artifacts "$rel_artifacts/build-baseline" \
+ ==rr[mode] "baseline" \
+ ==rr[current_branch] "$baseline_rev" \
+ ==rr[reset_baseline] true \
+ ==rr[top_artifacts] "$rel_artifacts/build-baseline" \
--verbose "$verbose"
assert ! [ -f $artifacts/failures ]
@@ -122,8 +123,8 @@ $build_script \
^^ $reproduce_bisect \
%% $rel_artifacts/manifests/build-\$rev.sh \
@@ $rel_artifacts/manifests/build-parameters.sh \
- --mode bisect \
- --top_artifacts $rel_artifacts/build-\$rev \
+ ==rr[mode] bisect \
+ ==rr[top_artifacts] $rel_artifacts/build-\$rev \
--verbose "$verbose" &
res=0 && wait \$! || res=\$?
@@ -251,7 +252,7 @@ if [ -f $artifacts/first-bad ]; then
ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh"
cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Successfully identified regression in CI configuration ${rr[ci_config]} for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev .
+Successfully identified regression in $current_project for CI configuration ${rr[ci_config]}.
Culprit:
<cut>
@@ -265,23 +266,23 @@ else
bad_name="bad"
bad_sha1="$bad_rev"
cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Could not identify regression in CI configuration ${rr[ci_config]} for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev down to a single commit. See 'Bisect log' in the links below for bisection details.
+Could not identify regression in $current_project for CI configuration ${rr[ci_config]}. See 'Bisect log' in the links below for bisection details.
EOF
fi
cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Results regressed from (for $good_name == $good_sha1)
+Configuration details:
+$(cat $artifacts/manifests/build-baseline.sh | grep '_url]\|_branch]' | grep -v '="no_')
-<cut>
+Results regressed from (for $good_name == $good_sha1)
$(cat $artifacts/build-$good_sha1/results)
-</cut>
to (for $bad_name == $bad_sha1)
-
-<cut>
$(cat $artifacts/build-$bad_sha1/results)
-</cut>
+
+First few errors in logs of $bad_name:
+$(grep ": error:" $artifacts/build-$bad_sha1/console.log | head)
Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/
Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/
@@ -336,8 +337,9 @@ fi
# Set mail recipients last to preserve catch-error value from .yaml file.
# Email developers.
CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org"
-if [ x"$ci_project" = x"tcwg_kernel" ]; then
+if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then
case "$toolchain:$current_project" in
+ gnu:linux) ;;
gnu:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;;
llvm:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;;
llvm:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;;
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index 5f64b163..ab169e5b 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -1,51 +1,59 @@
#!/bin/bash
-set -ef -o pipefail
+set -euf -o pipefail
scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh
convert_args_to_variables "$@"
-# Execution mode: baseline, bisect, continue, jenkins-full
-mode="${mode-baseline}"
+obligatory_variables rr[ci_config]
-obligatory_variables target toolchain
+# Execution mode: baseline, bisect, continue, jenkins-full
+rr[mode]="${rr[mode]-baseline}"
-case "$mode" in
+case "${rr[mode]}" in
"jenkins-full")
- obligatory_variables current_project current_branch
+ obligatory_variables rr[current_project] rr[current_branch]
;;
esac
# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
-current_project="${current_project-none}"
-current_branch="${current_branch-default}"
+rr[ci_project]="${rr[ci_project]-tcwg_kernel}"
+rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
+rr[current_branch]="${rr[current_branch]-default}"
+rr[current_project]="${rr[current_project]-none}"
+rr[reset_baseline]="${rr[reset_baseline]-false}"
+rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
+
+# {toolchain_name}-{toolchain_ver}-{target}-{linux}-{linux_config}
+IFS=- read -a ci_config <<EOF
+${rr[ci_config]}
+EOF
+rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
+rr[target]=${rr[target]-${ci_config[2]}}
+rr[linux_config]=${rr[linux_config]-${ci_config[4]}}
-linux_config="${linux_config-defconfig}"
-reset_baseline="${reset_baseline-false}"
start_at="${start_at-default}"
finish_at="${finish_at-default}"
-top_artifacts="${top_artifacts-$(pwd)/artifacts}"
verbose="${verbose-true}"
verbose2="${verbose2-false}"
-set -u
-
if $verbose2; then set -x; fi
-trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $top_artifacts/failures" EXIT
+trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT
# Set start and finish steps for different modes.
default_start_at=""
default_finish_at=""
-case "$mode" in
+base_artifacts_opt="base-artifacts"
+case "${rr[mode]}" in
"baseline")
default_finish_at="update_baseline"
;;
"bisect")
- case "$toolchain:$current_project" in
+ case "${rr[toolchain]}:${rr[current_project]}" in
gnu:binutils) default_start_at="build_abe-binutils" ;;
gnu:gcc) default_start_at="build_abe-stage1" ;;
gnu:linux) default_start_at="count_linux_objs" ;;
@@ -56,14 +64,14 @@ case "$mode" in
;;
"continue")
# Developer mode.
- case "$toolchain" in
+ case "${rr[toolchain]}" in
"gnu") default_start_at="prepare_abe" ;;
"llvm") default_start_at="build_llvm" ;;
esac
+ base_artifacts_opt=""
;;
"jenkins-full") ;;
esac
-
if [ x"$start_at" = x"default" ]; then
start_at="$default_start_at"
fi
@@ -71,27 +79,7 @@ if [ x"$finish_at" = x"default" ]; then
finish_at="$default_finish_at"
fi
-if [ x"$mode" != x"continue" ]; then
- base_artifacts_opt="base-artifacts"
-else
- base_artifacts_opt=""
-fi
-
-run_step_init "$start_at" "$finish_at" "$top_artifacts" "$base_artifacts_opt" "$verbose"
-
-rr[baseline_branch]="linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}"
-rr[current_project]="$current_project"
-rr[current_branch]="$current_branch"
-rr[mode]="$mode"
-rr[reset_baseline]="$reset_baseline"
-rr[target]="$target"
-rr[top_artifacts]="$top_artifacts"
-
-# Linux config to build in build_linux.
-rr[linux_config]="$linux_config"
-
-# Toolchain to use: gnu or llvm.
-rr[toolchain]="$toolchain"
+run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose"
. $scripts/round-robin.sh
diff --git a/tcwg_kernel/gnu-master.sh b/tcwg_kernel/gnu-master.sh
deleted file mode 100644
index 1621a04c..00000000
--- a/tcwg_kernel/gnu-master.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[gnu_version]=master
-rr[binutils_url]=git://sourceware.org/git/binutils-gdb.git
-rr[binutils_rev]=master
-rr[gcc_url]=git://gcc.gnu.org/git/gcc.git
-rr[gcc_rev]=master
diff --git a/tcwg_kernel/gnu-release.sh b/tcwg_kernel/gnu-release.sh
deleted file mode 100644
index 3c8728fa..00000000
--- a/tcwg_kernel/gnu-release.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[gnu_version]=release
-rr[binutils_url]=git://sourceware.org/git/binutils-gdb.git
-rr[binutils_rev]=binutils-2_31-branch
-rr[gcc_url]=git://gcc.gnu.org/git/gcc.git
-rr[gcc_rev]=gcc-8-branch
diff --git a/tcwg_kernel/linux-lts.sh b/tcwg_kernel/linux-lts.sh
deleted file mode 100644
index b849f02c..00000000
--- a/tcwg_kernel/linux-lts.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[linux_version]=lts
-rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
-rr[linux_rev]=linux-4.14.y
diff --git a/tcwg_kernel/linux-mainline.sh b/tcwg_kernel/linux-mainline.sh
deleted file mode 100644
index 983d69e9..00000000
--- a/tcwg_kernel/linux-mainline.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[linux_version]=mainline
-rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
-rr[linux_rev]=master
diff --git a/tcwg_kernel/linux-next.sh b/tcwg_kernel/linux-next.sh
deleted file mode 100644
index 585e9c1b..00000000
--- a/tcwg_kernel/linux-next.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[linux_version]=next
-rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
-rr[linux_rev]=master
diff --git a/tcwg_kernel/linux-stable.sh b/tcwg_kernel/linux-stable.sh
deleted file mode 100644
index 48cc61f7..00000000
--- a/tcwg_kernel/linux-stable.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[linux_version]=stable
-rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
-rr[linux_rev]=linux-4.18.y
diff --git a/tcwg_kernel/llvm-master.sh b/tcwg_kernel/llvm-master.sh
deleted file mode 100644
index 70f1a0c4..00000000
--- a/tcwg_kernel/llvm-master.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[llvm_version]=master
-rr[llvm_url]=https://github.com/llvm-project/llvm-project-20170507.git
-rr[llvm_rev]=master
diff --git a/tcwg_kernel/llvm-release.sh b/tcwg_kernel/llvm-release.sh
deleted file mode 100644
index 8653e670..00000000
--- a/tcwg_kernel/llvm-release.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-declare -gA rr
-
-rr[llvm_version]=release
-rr[llvm_url]=https://github.com/llvm-project/llvm-project-20170507.git
-rr[llvm_rev]=release_70