diff options
author | Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org> | 2019-02-12 19:41:45 +0530 |
---|---|---|
committer | Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org> | 2019-02-12 19:41:45 +0530 |
commit | 8028d1160dbcf1defe5e4853a9f0bcfa68c115cb (patch) | |
tree | d18b5aefb3d3a8bf25d9ff7c26c93a0d3adbfe8e | |
parent | a797f90814f5b0b8f31d4e97c3cf72db84f04d31 (diff) | |
parent | 99300555a21d0ef961d60042730afe5b79d3311a (diff) |
Merge branch 'master' into tcwg_gnu
Change-Id: Ifd517229bb3ee7642266e3e66a8682f177992076
-rw-r--r-- | jenkins-helpers.sh | 14 | ||||
-rwxr-xr-x | jenkins.sh | 11 | ||||
-rw-r--r-- | round-robin.sh | 43 | ||||
-rwxr-xr-x | tcwg-benchmark-results.sh | 1 | ||||
-rwxr-xr-x | tcwg-benchmark.sh | 7 | ||||
-rwxr-xr-x | tcwg_kernel-bisect.sh | 38 | ||||
-rwxr-xr-x | tcwg_kernel-build.sh | 66 | ||||
-rw-r--r-- | tcwg_kernel/gnu-master.sh | 9 | ||||
-rw-r--r-- | tcwg_kernel/gnu-release.sh | 9 | ||||
-rw-r--r-- | tcwg_kernel/linux-lts.sh | 7 | ||||
-rw-r--r-- | tcwg_kernel/linux-mainline.sh | 7 | ||||
-rw-r--r-- | tcwg_kernel/linux-next.sh | 7 | ||||
-rw-r--r-- | tcwg_kernel/linux-stable.sh | 7 | ||||
-rw-r--r-- | tcwg_kernel/llvm-master.sh | 7 | ||||
-rw-r--r-- | tcwg_kernel/llvm-release.sh | 7 |
15 files changed, 92 insertions, 148 deletions
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index a2fb51d7..5a471800 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -152,6 +152,11 @@ print_host_for_node () # configured ssh local host="$1.tcwglab" + # Map tcwg-x86_64-build-* and tcwg-x86_64-dev-* to build-* and dev-* + case "$host" in + tcwg-x86_64-*) host=${host#tcwg-x86_64-} ;; + esac + if ! host "$host" >& /dev/null; then # No DNS entry, try to see if it is handled by our ssh config if ! ssh "$1" true >& /dev/null; then @@ -236,7 +241,10 @@ remote_exec () cmd=() # Add quotes to every parameter for i in "$@"; do cmd+=("$(printf '%q' "$i")"); done - ssh $opts ${port:+-p$port} $host "${dir:+cd "$(printf '%q' "$dir")" &&} exec ${cmd[@]}" + # Be careful to prepend statements before ${cmd[@]} only if necessary. + # E.g., when triggering jobs via jenkins-cli, the command is not a binary, + # so we can't "exec" it. + ssh $opts ${port:+-p$port} $host "${dir:+cd "$(printf '%q' "$dir")" && exec }${cmd[@]}" ) } @@ -669,7 +677,9 @@ print_docker_path () if [ -f "/.dockerenv" ] && mount | grep -q "/run/docker.sock "; then # If inside "host" container (with proxied docker and /home from host-home volume), # convert paths to refer to volume's path on bare-metal. - echo "$path" | sed -e "s#^/home/#/var/lib/docker/volumes/host-home/_data/#" + local docker_root + docker_root=$(docker info | grep "Docker Root Dir:" | cut -d: -f 2) + echo "$path" | sed -e "s#^/home/#$docker_root/volumes/host-home/_data/#" else echo "$path" fi @@ -390,17 +390,6 @@ if test x"${debug}" = x"true"; then export CONFIG_SHELL="/bin/bash -x" fi -# Download QEMU provided by Peter Maydell. -# The tarball has README with version information. -if [ x"$(uname -m)" = x"x86_64" ]; then - wget --progress=dot:giga http://people.linaro.org/~maxim.kuvyrkov/qemu-20160707.tgz - tar xf qemu-20160707.tgz - export PATH="$(pwd)/qemu-wip:$PATH" - for i in aarch64 arm armeb; do - qemu-$i --version - done -fi - # Print some information about the build machine echo Running on $(hostname) uname -a diff --git a/round-robin.sh b/round-robin.sh index 1bc1f865..26cfe534 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -12,9 +12,9 @@ # PROJECT's git SHA1 revision. These are mostly used in manifests. #rr[PROJECT_rev] -# Baseline branch name for current configuration. These branches are -# present in all above git repos. -#rr[baseline_branch] +# Baseline branch name for current configuration. These branches should +# be present in all above git repos (if ${rr[init_configuration]} is false). +#rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}" # PROJECT that we are testing in this build. Use ${rr[current_branch]} for # this project, and ${rr[baseline_branch]} for all other projects. @@ -36,6 +36,10 @@ # Top-level artifacts directory. #rr[top_artifacts]="$top_artifacts" +# Set rr[init_configuration] to 'true' for new projects/configurations +# to initialize baseline branches in git repos. +#rr[init_configuration]=false + # Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most # important artifact, since it records the metric of how successful the build # is. @@ -91,6 +95,7 @@ clone_repo () # ${rr[current_branch]} specifies branch for ${rr[current_project]}, # and everything else uses baseline branch. if [ x"$project" = x"${rr[current_project]}" ]; then + # Use the tip of the branch tracked in the current configuration. if [ x"${rr[current_branch]}" = x"default" ]; then branch=${rr[${rr[current_project]}_branch]} elif [ x"${rr[current_branch]}" = x"baseline" ]; then @@ -217,16 +222,23 @@ build_abe () ;; esac - clone_repo $project - - # Don't use ABE's repo clone functions and setup abe/snapshots/ directory - # to have the right entries. - local git_dir git_path - git_dir="$(basename ${rr[${project}_url]})" - git_path="abe/snapshots/$git_dir" - rm -rf $git_path $git_path~master - ln -s $(pwd)/$project $git_path - ln -s $(pwd)/$project $git_path~master + # Use our custom sources for everything, but kernel headers. + local custom_abe_src_opt="" + local git_dir="$project" + if [ x"$component" != x"linux" ]; then + clone_repo $project + + # Don't use ABE's repo clone functions and setup abe/snapshots/ directory + # to have the right entries. + local git_dir git_path + git_dir="$(basename ${rr[${project}_url]})" + git_path="abe/snapshots/$git_dir" + rm -rf $git_path $git_path~master + ln -s $(pwd)/$project $git_path + ln -s $(pwd)/$project $git_path~master + + custom_abe_src_opt="$project=${rr[${project}_url]}~master --disable update" + fi cd abe @@ -235,7 +247,7 @@ build_abe () # Remove previous build directories and .stamp files. # We rely on ccache for fast rebuilds. - set +f; rm -rf builds/*/$gnu_target/$git_dir~master*; set -f + set +f; rm -rf builds/*/*/$git_dir*; set -f export PATH=$(pwd)/bin:$PATH @@ -259,8 +271,7 @@ build_abe () --$action $project \ $target_opt \ --extraconfigdir config/master \ - --disable update \ - $project=${rr[${project}_url]}~master \ + $custom_abe_src_opt \ $stage ccache -s ) diff --git a/tcwg-benchmark-results.sh b/tcwg-benchmark-results.sh index f7147f8c..90e6bcd2 100755 --- a/tcwg-benchmark-results.sh +++ b/tcwg-benchmark-results.sh @@ -28,6 +28,7 @@ for i in $results_ref $results_id; do rsync -az --delete "$results_top-$i/" results-$num/ $scripts/../bmk-scripts/perfdatadir2csv.sh --buildid-dir local --format sample,size --sort-field sample --results-dir "results-$num/" > "results-$num.csv" csvs="$csvs results-$num.csv" + num=$(($num+1)) done ssh -O exit dev-01.tcwglab diff --git a/tcwg-benchmark.sh b/tcwg-benchmark.sh index fd9000bc..fbb49111 100755 --- a/tcwg-benchmark.sh +++ b/tcwg-benchmark.sh @@ -46,7 +46,9 @@ case "$toolchain_url" in "ssh://"*) ccprefix="${toolchain_url##ssh://}" - # Extract host:port: specification from ccprefix. + # Extract host:port: specification from ccprefix, we don't + # need to care about :parallelize here, just pass it to run.sh + # if present. case ${ccprefix} in *:*:*) build=${ccprefix%:*} @@ -85,7 +87,8 @@ case "$toolchain_url" in echo "ERROR: Unsupported sysroot $sysroot for toolchain_url $toolchain_url" exit 1 fi - toolchaindir="$(dirname $(echo $ccprefix | cut -s -d: -f 3))" + # Last component of ccprefix is the path, keep it + toolchaindir="$(dirname ${ccprefix##*:})" ;; "http://"*".tar.xz"|"https://"*".tar.xz") toolchaindir=$(untar_url "$toolchain_url" "$WORKSPACE" "--strip-components 1") diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh index e4d11e2b..45a6aaf0 100755 --- a/tcwg_kernel-bisect.sh +++ b/tcwg_kernel-bisect.sh @@ -16,8 +16,7 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*" convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" -obligatory_variables bad_rev baseline_rev build_script \ - rr[ci_project] rr[ci_config] +obligatory_variables bad_rev baseline_rev build_script rr[ci_project] BUILD_URL="${BUILD_URL:-$(pwd)}" reproduce_bisect="${reproduce_bisect:-false}" @@ -26,7 +25,9 @@ reproduce_bisect="${reproduce_bisect:-false}" convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@" $reproduce_bisect || manifest_pop -obligatory_variables current_project +obligatory_variables rr[current_project] rr[ci_config] + +current_project=${rr[current_project]} if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then obligatory_variables toolchain @@ -65,10 +66,10 @@ $build_script \ ^^ $reproduce_bisect \ %% $rel_artifacts/manifests/build-baseline.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ - --mode "baseline" \ - --current_branch "$baseline_rev" \ - --reset_baseline true \ - --top_artifacts "$rel_artifacts/build-baseline" \ + ==rr[mode] "baseline" \ + ==rr[current_branch] "$baseline_rev" \ + ==rr[reset_baseline] true \ + ==rr[top_artifacts] "$rel_artifacts/build-baseline" \ --verbose "$verbose" assert ! [ -f $artifacts/failures ] @@ -122,8 +123,8 @@ $build_script \ ^^ $reproduce_bisect \ %% $rel_artifacts/manifests/build-\$rev.sh \ @@ $rel_artifacts/manifests/build-parameters.sh \ - --mode bisect \ - --top_artifacts $rel_artifacts/build-\$rev \ + ==rr[mode] bisect \ + ==rr[top_artifacts] $rel_artifacts/build-\$rev \ --verbose "$verbose" & res=0 && wait \$! || res=\$? @@ -251,7 +252,7 @@ if [ -f $artifacts/first-bad ]; then ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh" cat >> $artifacts/jenkins/mail-body.txt <<EOF -Successfully identified regression in CI configuration ${rr[ci_config]} for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev . +Successfully identified regression in $current_project for CI configuration ${rr[ci_config]}. Culprit: <cut> @@ -265,23 +266,23 @@ else bad_name="bad" bad_sha1="$bad_rev" cat >> $artifacts/jenkins/mail-body.txt <<EOF -Could not identify regression in CI configuration ${rr[ci_config]} for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev down to a single commit. See 'Bisect log' in the links below for bisection details. +Could not identify regression in $current_project for CI configuration ${rr[ci_config]}. See 'Bisect log' in the links below for bisection details. EOF fi cat >> $artifacts/jenkins/mail-body.txt <<EOF -Results regressed from (for $good_name == $good_sha1) +Configuration details: +$(cat $artifacts/manifests/build-baseline.sh | grep '_url]\|_branch]' | grep -v '="no_') -<cut> +Results regressed from (for $good_name == $good_sha1) $(cat $artifacts/build-$good_sha1/results) -</cut> to (for $bad_name == $bad_sha1) - -<cut> $(cat $artifacts/build-$bad_sha1/results) -</cut> + +First few errors in logs of $bad_name: +$(grep ": error:" $artifacts/build-$bad_sha1/console.log | head) Artifacts of $good_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$good_sha1/ Artifacts of $bad_name build: ${BUILD_URL}artifact/$rel_artifacts/build-$bad_sha1/ @@ -336,8 +337,9 @@ fi # Set mail recipients last to preserve catch-error value from .yaml file. # Email developers. CI_MAIL_RECIPIENTS="tcwg-validation@linaro.org" -if [ x"$ci_project" = x"tcwg_kernel" ]; then +if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then case "$toolchain:$current_project" in + gnu:linux) ;; gnu:*) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, christophe.lyon@linaro.org, maxim.kuvyrkov@linaro.org" ;; llvm:linux) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, arnd@linaro.org, mark.brown@linaro.org, ndesaulniers@google.com, trong@google.com" ;; llvm:llvm) CI_MAIL_RECIPIENTS="$CI_MAIL_RECIPIENTS, adhemerval.zanella@linaro.org, maxim.kuvyrkov@linaro.org, ndesaulniers@google.com, trong@google.com, yvan.roux@linaro.org" ;; diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh index 5f64b163..ab169e5b 100755 --- a/tcwg_kernel-build.sh +++ b/tcwg_kernel-build.sh @@ -1,51 +1,59 @@ #!/bin/bash -set -ef -o pipefail +set -euf -o pipefail scripts=$(dirname $0) . $scripts/jenkins-helpers.sh convert_args_to_variables "$@" -# Execution mode: baseline, bisect, continue, jenkins-full -mode="${mode-baseline}" +obligatory_variables rr[ci_config] -obligatory_variables target toolchain +# Execution mode: baseline, bisect, continue, jenkins-full +rr[mode]="${rr[mode]-baseline}" -case "$mode" in +case "${rr[mode]}" in "jenkins-full") - obligatory_variables current_project current_branch + obligatory_variables rr[current_project] rr[current_branch] ;; esac # Set custom revision for one of the projects, and use baseline revisions # for all other projects. -current_project="${current_project-none}" -current_branch="${current_branch-default}" +rr[ci_project]="${rr[ci_project]-tcwg_kernel}" +rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}" +rr[current_branch]="${rr[current_branch]-default}" +rr[current_project]="${rr[current_project]-none}" +rr[reset_baseline]="${rr[reset_baseline]-false}" +rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}" + +# {toolchain_name}-{toolchain_ver}-{target}-{linux}-{linux_config} +IFS=- read -a ci_config <<EOF +${rr[ci_config]} +EOF +rr[toolchain]=${rr[toolchain]-${ci_config[0]}} +rr[target]=${rr[target]-${ci_config[2]}} +rr[linux_config]=${rr[linux_config]-${ci_config[4]}} -linux_config="${linux_config-defconfig}" -reset_baseline="${reset_baseline-false}" start_at="${start_at-default}" finish_at="${finish_at-default}" -top_artifacts="${top_artifacts-$(pwd)/artifacts}" verbose="${verbose-true}" verbose2="${verbose2-false}" -set -u - if $verbose2; then set -x; fi -trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $top_artifacts/failures" EXIT +trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT # Set start and finish steps for different modes. default_start_at="" default_finish_at="" -case "$mode" in +base_artifacts_opt="base-artifacts" +case "${rr[mode]}" in "baseline") default_finish_at="update_baseline" ;; "bisect") - case "$toolchain:$current_project" in + case "${rr[toolchain]}:${rr[current_project]}" in gnu:binutils) default_start_at="build_abe-binutils" ;; gnu:gcc) default_start_at="build_abe-stage1" ;; gnu:linux) default_start_at="count_linux_objs" ;; @@ -56,14 +64,14 @@ case "$mode" in ;; "continue") # Developer mode. - case "$toolchain" in + case "${rr[toolchain]}" in "gnu") default_start_at="prepare_abe" ;; "llvm") default_start_at="build_llvm" ;; esac + base_artifacts_opt="" ;; "jenkins-full") ;; esac - if [ x"$start_at" = x"default" ]; then start_at="$default_start_at" fi @@ -71,27 +79,7 @@ if [ x"$finish_at" = x"default" ]; then finish_at="$default_finish_at" fi -if [ x"$mode" != x"continue" ]; then - base_artifacts_opt="base-artifacts" -else - base_artifacts_opt="" -fi - -run_step_init "$start_at" "$finish_at" "$top_artifacts" "$base_artifacts_opt" "$verbose" - -rr[baseline_branch]="linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}" -rr[current_project]="$current_project" -rr[current_branch]="$current_branch" -rr[mode]="$mode" -rr[reset_baseline]="$reset_baseline" -rr[target]="$target" -rr[top_artifacts]="$top_artifacts" - -# Linux config to build in build_linux. -rr[linux_config]="$linux_config" - -# Toolchain to use: gnu or llvm. -rr[toolchain]="$toolchain" +run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose" . $scripts/round-robin.sh diff --git a/tcwg_kernel/gnu-master.sh b/tcwg_kernel/gnu-master.sh deleted file mode 100644 index 1621a04c..00000000 --- a/tcwg_kernel/gnu-master.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[gnu_version]=master -rr[binutils_url]=git://sourceware.org/git/binutils-gdb.git -rr[binutils_rev]=master -rr[gcc_url]=git://gcc.gnu.org/git/gcc.git -rr[gcc_rev]=master diff --git a/tcwg_kernel/gnu-release.sh b/tcwg_kernel/gnu-release.sh deleted file mode 100644 index 3c8728fa..00000000 --- a/tcwg_kernel/gnu-release.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[gnu_version]=release -rr[binutils_url]=git://sourceware.org/git/binutils-gdb.git -rr[binutils_rev]=binutils-2_31-branch -rr[gcc_url]=git://gcc.gnu.org/git/gcc.git -rr[gcc_rev]=gcc-8-branch diff --git a/tcwg_kernel/linux-lts.sh b/tcwg_kernel/linux-lts.sh deleted file mode 100644 index b849f02c..00000000 --- a/tcwg_kernel/linux-lts.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[linux_version]=lts -rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git -rr[linux_rev]=linux-4.14.y diff --git a/tcwg_kernel/linux-mainline.sh b/tcwg_kernel/linux-mainline.sh deleted file mode 100644 index 983d69e9..00000000 --- a/tcwg_kernel/linux-mainline.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[linux_version]=mainline -rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git -rr[linux_rev]=master diff --git a/tcwg_kernel/linux-next.sh b/tcwg_kernel/linux-next.sh deleted file mode 100644 index 585e9c1b..00000000 --- a/tcwg_kernel/linux-next.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[linux_version]=next -rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git -rr[linux_rev]=master diff --git a/tcwg_kernel/linux-stable.sh b/tcwg_kernel/linux-stable.sh deleted file mode 100644 index 48cc61f7..00000000 --- a/tcwg_kernel/linux-stable.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[linux_version]=stable -rr[linux_url]=git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git -rr[linux_rev]=linux-4.18.y diff --git a/tcwg_kernel/llvm-master.sh b/tcwg_kernel/llvm-master.sh deleted file mode 100644 index 70f1a0c4..00000000 --- a/tcwg_kernel/llvm-master.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[llvm_version]=master -rr[llvm_url]=https://github.com/llvm-project/llvm-project-20170507.git -rr[llvm_rev]=master diff --git a/tcwg_kernel/llvm-release.sh b/tcwg_kernel/llvm-release.sh deleted file mode 100644 index 8653e670..00000000 --- a/tcwg_kernel/llvm-release.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -declare -gA rr - -rr[llvm_version]=release -rr[llvm_url]=https://github.com/llvm-project/llvm-project-20170507.git -rr[llvm_rev]=release_70 |