summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2019-02-12 09:12:36 +0000
committerMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2019-02-12 09:12:36 +0000
commit2fa4f7225c39474d8e27f864d8c173d68699ed2d (patch)
tree104de8d186ea193b993a31b7cf6040cb659ae73f
parentc2fd5316323b08ecf5d51cec019ddd1001b50a5d (diff)
[rr-many 1/N]: Support testing multiple projects in the same build
This patch series is both a major cleanup and an optimization of round-robin jobs. We switch from testing each component (binutils, gcc, linux) in separate builds in round-robin fashion to testing all components in the same build, and, if there's a failure, re-triggering builds for individual components like we are going now. Then if individual build fails, it triggers bisect on that one component. The cleanup part is that jenkins triggers are greatly simplified and we can remove all trigger jobs (of which we had hundreds). The optimization part is that successful builds (which is the majority) now consume N times less resources, where N is the number of component -- 3 in case of binutils, gcc, linux, -- because all components are updated and tested in the same build. This patch is the first in the series, and it replaces uses of $current_project and $current_branch with "baseline" keyword of components' branches. I.e., for testing LLVM in LLVM+Kernel jobs we currently use ==rr[llvm_branch] master ==rr[linux_branch] master \ ==rr[current_project] llvm ==rr[current_branch] SHA1 and it will change to ==rr[llvm_branch] SHA1 ==rr[linux_branch] baseline . IMO, the latter is significantly more intuitive. Change-Id: Ib47dcac1571381f870150e4872ec37be5d900a07
-rw-r--r--round-robin.sh46
-rwxr-xr-xtcwg_kernel-bisect.sh8
-rwxr-xr-xtcwg_kernel-build.sh29
3 files changed, 59 insertions, 24 deletions
diff --git a/round-robin.sh b/round-robin.sh
index e4ff41a0..1fdd27de 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -7,6 +7,11 @@
#rr[PROJECT_url]
# PROJECT's git branch or SHA1 revision parsable by git rev-parse.
+# A special value "baseline" means that PROJECT is not being updated
+# in this build, and its baseline branch should be used.
+# In a successful build "update_baseline" step will update baseline
+# branches of all PROJECTs to the current values, thus setting
+# a baseline for the next build.
#rr[PROJECT_branch]
# PROJECT's git SHA1 revision. These are mostly used in manifests.
@@ -16,13 +21,6 @@
# be present in all above git repos (if ${rr[init_configuration]} is false).
#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}"
-# PROJECT that we are testing in this build. Use ${rr[current_branch]} for
-# this project, and ${rr[baseline_branch]} for all other projects.
-#rr[current_project]="$current_project"
-
-# Git branch or SHA1 revision of ${rr[current_project]} to test.
-#rr[current_branch]="$current_branch"
-
# Run mode: bisect or non-bisect. In bisect mode we do a couple things
# slightly differently (e.g., don't touch repo in clone_repo() ).
#rr[mode]="$mode"
@@ -40,6 +38,40 @@
# to initialize baseline branches in git repos.
#rr[init_configuration]=false
+# Print round-robin components that are being updated in this build
+# (the ones using non-baseline branches).
+print_updated_components ()
+{
+ (
+ set -euf -o pipefail
+
+ local delim=""
+ local c
+ for c in ${rr[components]}; do
+ if [ x"${rr[${c}_branch]}" != x"baseline" ]; then
+ printf "%s%s" "$delim" "$c"
+ delim=${1- }
+ fi
+ done
+ )
+}
+
+# Print the single round-robin component being updated in this build.
+# Print nothing if multiple components are being updated.
+print_single_updated_component ()
+{
+ (
+ set -euf -o pipefail
+
+ local -a updated_components
+ updated_components=($(print_updated_components))
+
+ if [ ${#updated_components[@]} = 1 ]; then
+ echo "${updated_components[0]}"
+ fi
+ )
+}
+
# Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most
# important artifact, since it records the metric of how successful the build
# is.
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index e9d84d98..01465b76 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -16,7 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"
convert_args_to_variables "$@"
shift "$SHIFT_CONVERTED_ARGS"
-obligatory_variables bad_rev baseline_rev build_script rr[ci_project]
+obligatory_variables bad_rev baseline_rev build_script current_project
BUILD_URL="${BUILD_URL:-$(pwd)}"
reproduce_bisect="${reproduce_bisect:-false}"
@@ -25,9 +25,7 @@ reproduce_bisect="${reproduce_bisect:-false}"
convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@"
$reproduce_bisect || manifest_pop
-obligatory_variables rr[current_project] rr[ci_config]
-
-current_project=${rr[current_project]}
+obligatory_variables rr[ci_project] rr[ci_config]
verbose="${verbose:-true}"
@@ -64,7 +62,7 @@ $build_script \
%% $rel_artifacts/manifests/build-baseline.sh \
@@ $rel_artifacts/manifests/build-parameters.sh \
==rr[mode] "baseline" \
- ==rr[current_branch] "$baseline_rev" \
+ ==rr[${current_project}_branch] "$baseline_rev" \
==rr[reset_baseline] true \
==rr[top_artifacts] "$rel_artifacts/build-baseline" \
--verbose "$verbose"
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index ab169e5b..dabc2995 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -4,6 +4,7 @@ set -euf -o pipefail
scripts=$(dirname $0)
. $scripts/jenkins-helpers.sh
+. $scripts/round-robin.sh
convert_args_to_variables "$@"
@@ -12,18 +13,10 @@ obligatory_variables rr[ci_config]
# Execution mode: baseline, bisect, continue, jenkins-full
rr[mode]="${rr[mode]-baseline}"
-case "${rr[mode]}" in
- "jenkins-full")
- obligatory_variables rr[current_project] rr[current_branch]
- ;;
-esac
-
# Set custom revision for one of the projects, and use baseline revisions
# for all other projects.
rr[ci_project]="${rr[ci_project]-tcwg_kernel}"
rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
-rr[current_branch]="${rr[current_branch]-default}"
-rr[current_project]="${rr[current_project]-none}"
rr[reset_baseline]="${rr[reset_baseline]-false}"
rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
@@ -35,6 +28,18 @@ rr[toolchain]=${rr[toolchain]-${ci_config[0]}}
rr[target]=${rr[target]-${ci_config[2]}}
rr[linux_config]=${rr[linux_config]-${ci_config[4]}}
+case "${rr[toolchain]}" in
+ llvm) rr[components]="llvm linux" ;;
+ gnu) rr[components]="binutils gcc linux" ;;
+ *) assert false ;;
+esac
+
+# Use baseline branches by default.
+for c in ${rr[components]}; do
+ rr[${c}_branch]=${rr[${c}_branch]-baseline}
+ obligatory_variables rr[${c}_url]
+done
+
start_at="${start_at-default}"
finish_at="${finish_at-default}"
verbose="${verbose-true}"
@@ -53,12 +58,13 @@ case "${rr[mode]}" in
default_finish_at="update_baseline"
;;
"bisect")
- case "${rr[toolchain]}:${rr[current_project]}" in
+ case "${rr[toolchain]}:$(print_single_updated_component)" in
gnu:binutils) default_start_at="build_abe-binutils" ;;
gnu:gcc) default_start_at="build_abe-stage1" ;;
gnu:linux) default_start_at="count_linux_objs" ;;
llvm:llvm) default_start_at="build_llvm" ;;
llvm:linux) default_start_at="count_linux_objs" ;;
+ *) assert false ;;
esac
default_finish_at="check_regression"
;;
@@ -81,8 +87,6 @@ fi
run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose"
-. $scripts/round-robin.sh
-
# Build Linux kernel
build_linux ()
{
@@ -114,7 +118,8 @@ build_linux ()
# Otherwise the compiler is new in every build and we would
# only clobber ccache volume.
local ccache=""
- if [ x"${rr[mode]}" != x"jenkins-full" -a x"${rr[current_project]}" = x"linux" ]; then
+ if [ x"${rr[mode]}" != x"jenkins-full" \
+ -a x"$(print_single_updated_component)" = x"linux" ]; then
ccache="ccache"
fi
cat > $(pwd)/bin/${rr[target]}-cc <<EOF