diff options
| -rwxr-xr-x | docker-run.sh | 7 | ||||
| -rw-r--r-- | jenkins-helpers.sh | 26 | ||||
| -rw-r--r-- | round-robin.sh | 13 | ||||
| -rwxr-xr-x | start-container-docker.sh | 21 | ||||
| -rwxr-xr-x | tcwg-benchmark-results-compare.sh | 2 | ||||
| -rwxr-xr-x | tcwg-benchmark-results.sh | 2 | ||||
| -rwxr-xr-x | tcwg-benchmark.sh | 8 | ||||
| -rwxr-xr-x | tcwg_bmk-build.sh | 206 |
8 files changed, 255 insertions, 30 deletions
diff --git a/docker-run.sh b/docker-run.sh index b1914333..de7f340a 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -1,14 +1,13 @@ #!/bin/bash -set -ef -o pipefail - -qemu="${qemu-}" -set -u +set -euf -o pipefail . $(dirname $0)/jenkins-helpers.sh convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" +qemu="${qemu-}" + trap "container_cleanup" EXIT case "$qemu" in diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 5291873e..65428d9a 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -246,18 +246,22 @@ print_tester_label_for_target () } # Run command on remote machine in given directory via ssh on a given port -# "$1" -- <host>[:<port>[:<dir>[:<ssh_opts>]]] +# "$1" -- <host>[:<port>[:<dir>[:<ssh_opts>[:<env>]]]] # "$2, $3, etc" -- command and its arguments # E.g., remote_exec dev-01.tcwglab::/tmp find -name "my file.bak" +# NOTE: The environment variables are not escaped, so pass only simple things. +# This is because we want ability to pass multiple variables "a=b c=d", +# and escaping will make that into a single a="b c=d" variable. remote_exec () { ( set -euf -o pipefail - local host="$(echo $1 | cut -d: -f 1)" - local port="$(echo $1 | cut -s -d: -f 2)" - local dir="$(echo $1 | cut -s -d: -f 3)" - local opts="$(echo $1 | cut -s -d: -f 4)" + local host="$(echo "$1" | cut -d: -f 1)" + local port="$(echo "$1" | cut -s -d: -f 2)" + local dir="$(echo "$1" | cut -s -d: -f 3)" + local opts="$(echo "$1" | cut -s -d: -f 4)" + local env_vars="$(echo "$1" | cut -s -d: -f 5)" shift local -a cmd cmd=() @@ -266,7 +270,7 @@ remote_exec () # Be careful to prepend statements before ${cmd[@]} only if necessary. # E.g., when triggering jobs via jenkins-cli, the command is not a binary, # so we can't "exec" it. - ssh $opts ${port:+-p$port} $host "${dir:+cd "$(printf '%q' "$dir")" && exec }${cmd[@]}" + ssh $opts ${port:+-p$port} $host "${env_vars:+export $env_vars && }${dir:+cd "$(printf '%q' "$dir")" && exec }${cmd[@]}" ) } @@ -1122,11 +1126,6 @@ run_step () run_step_active=true fi - run_step_artifacts=$run_step_top_artifacts/$run_step_count-$pretty_step - - rm -rf "$run_step_artifacts" - mkdir -p "$run_step_artifacts" - if $run_step_active; then local skip=false case "$run_step_status:$run_mode" in @@ -1147,6 +1146,11 @@ run_step () esac if ! $skip; then + run_step_artifacts=$run_step_top_artifacts/$(printf "%02d" $run_step_count)-$pretty_step + + rm -rf "$run_step_artifacts" + mkdir -p "$run_step_artifacts" + echo "RUNNING ${step[@]}; see tail -f $run_step_artifacts/console.log" run_step_status=0 eval "if $run_step_verbose; then set -x; else set +x; fi; ${step[@]}" 2>&1 | tee -a $run_step_top_artifacts/console.log > $run_step_artifacts/console.log & diff --git a/round-robin.sh b/round-robin.sh index 967c70de..abe27dfe 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -322,6 +322,8 @@ build_llvm () ( set -euf -o pipefail + local use_abe=${1-false} + clone_repo llvm # Copy only components from the monorepo that are required for kernel build @@ -356,12 +358,17 @@ EOF export PATH=$(pwd)/bin:$PATH - # Remove stale build directories. We rely on ccache for fast rebuilds. - rm -rf llvm-build llvm-install + # Freshen up build and install directories. We rely on ccache for fast rebuilds. + if $use_abe; then + rsync -a --del abe/builds/destdir/x86_64-unknown-linux-gnu/ llvm-install/ + else + rm -rf llvm-install + fi + rm -rf llvm-build mkdir -p llvm-build cd llvm-build - cmake -G Ninja ../llvm-src -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install -DLLVM_TARGETS_TO_BUILD=$(print_llvm_target ${rr[target]}) + cmake -G Ninja ../llvm-src -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install -DLLVM_TARGETS_TO_BUILD=$(print_llvm_target ${rr[target]}) -DLLVM_BINUTILS_INCDIR=/usr/include ccache -z ninja ninja install diff --git a/start-container-docker.sh b/start-container-docker.sh index a0f09aa7..dfcff4ee 100755 --- a/start-container-docker.sh +++ b/start-container-docker.sh @@ -15,7 +15,7 @@ set -e # - definition of ${session_host} and ${session_port}, can be used for # a remote connexion to the container usage() { - echo "Usage: $0 [--arch container-arch] --distro flavour [--docker_opts opts] [--dryrun true/false] [--label label] [--newuser username:[uid]] [--node node] [--prefix prefix] [--session-host host] [--session-name name] [--task {build|test}] [--user user] [--weight weight] [--verbose true/false]" + echo "Usage: $0 [--arch container-arch] --distro flavour [--docker_opts opts] [--dryrun true/false] [--label label] [--newuser username:[uid]] [--node node] [--prefix prefix] [--session-host host] [--session-name name] [--ssh_info true/false] [--task {build|test}] [--user user] [--weight weight] [--verbose true/false]" echo echo " container-arch: architecture (eg: amd64, i386, arm64, armhf)" echo " distro: distribution (eg: trusty)" @@ -27,6 +27,7 @@ usage() { echo " session-host: hostname where the container will run, defaults to localhost" echo " useful if the name resolution does not work correctly" echo " session-name: session, in case the default '$BUILD_NUMBER-$JOB_NAME' is not suitable" + echo " ssh_info: set $ssh_host and $ssh_port env variables in the container" echo " task: type of container (build or test, default=build)" echo " user: remote user to use in the container." echo " weight: container weight, reserves resources. Default=1" @@ -50,6 +51,7 @@ newuser= prefix= session_host= session_name= +ssh_info=false task="build" weight=1 user= @@ -109,6 +111,11 @@ do [ x${session_name} = x ] && usage shift 2 ;; + --ssh_info) + ssh_info=$2 + [ x$ssh_info = x ] && usage + shift 2 + ;; --task) task=$2 [ x${task} = x ] && usage @@ -196,11 +203,6 @@ fi image=linaro/ci-${container_arch}-tcwg-${task}-ubuntu:${distro} -case "$session_host" in - "localhost"*) docker_host="" ;; - *) docker_host="$session_host" ;; -esac - DOCKER="ssh $session_host docker" $DOCKER pull $image @@ -284,6 +286,11 @@ fi # Do not remove the container upon exit: it is now ready trap EXIT +ssh_info_opt="" +if $ssh_info; then + ssh_info_opt="ssh_host=${user}$session_host ssh_port=$session_port" +fi + # Restore stdout/stderr exec 1>&3 2>&4 @@ -314,7 +321,7 @@ ${prefix}container_stop () } ${prefix}container_exec () { - $dryruncmd remote_exec "${user}${session_host}:${session_port}:\$(pwd)" "\$@" + $dryruncmd remote_exec "${user}${session_host}:${session_port}:\$(pwd)::$ssh_info_opt" "\$@" } ${prefix}container_host=${session_host} diff --git a/tcwg-benchmark-results-compare.sh b/tcwg-benchmark-results-compare.sh index c9650dd8..f6450254 100755 --- a/tcwg-benchmark-results-compare.sh +++ b/tcwg-benchmark-results-compare.sh @@ -13,7 +13,7 @@ peak="$peak" peak_opt="" $peak && peak_opt=--peak -results_top="dev-01.tcwglab:/home/tcwg-benchmark/results" +results_top="bkp-01.tcwglab:/home/tcwg-benchmark/results" rm -rf results-eval results-ref artifacts diff --git a/tcwg-benchmark-results.sh b/tcwg-benchmark-results.sh index 6cdd2810..3f3e2617 100755 --- a/tcwg-benchmark-results.sh +++ b/tcwg-benchmark-results.sh @@ -20,7 +20,7 @@ set -u if $verbose; then set -x; fi -results_top="dev-01.tcwglab:/home/tcwg-benchmark/results" +results_top="bkp-01.tcwglab:/home/tcwg-benchmark/results" num=0 csvs="" diff --git a/tcwg-benchmark.sh b/tcwg-benchmark.sh index 6fa63c22..1791369b 100755 --- a/tcwg-benchmark.sh +++ b/tcwg-benchmark.sh @@ -217,8 +217,10 @@ boardname=$(echo "${NODE_NAME}" | sed 's/-bmk//').tcwglab hw_type=$(print_hw_id_for_node "$NODE_NAME") results_id=$(echo "$results_id" \ - | sed -e "s/<hw_type>/$hw_type/g" \ - -e "s/<build_num>/$BUILD_NUMBER/g") + | sed -e "s/<hw_type>/$hw_type/g" \ + -e "s/@hw_type@/$hw_type/g" \ + -e "s/<build_num>/$BUILD_NUMBER/g" \ + -e "s/@build_num@/$BUILD_NUMBER/g") if echo "$results_id" | grep -q "\.\."; then echo "ERROR: results_id should not escape /home/tcwg-benchmark/results* hierarchy; do not use \"..\"" exit 1 @@ -252,7 +254,7 @@ ssh -t -Snone "$boardname" bmk-scripts/run.sh \ --run_profile "$run_profile" \ "${sysroot:+--sysroot "$sysroot"}" \ --toolchain "$toolchain_type" \ - --resultsdest "dev-01.tcwglab:/home/tcwg-benchmark/results-${results_id}/${NODE_NAME}" \ + --resultsdest "bkp-01.tcwglab:/home/tcwg-benchmark/results-${results_id}/${NODE_NAME}" \ --nodename "${NODE_NAME}" \ --forceinstall "${forceinstall}" \ "${clean_older_than:+--clean_older_than "$clean_older_than"}" \ diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh new file mode 100755 index 00000000..bd4eb565 --- /dev/null +++ b/tcwg_bmk-build.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +set -euf -o pipefail + +scripts=$(dirname $0) +. $scripts/jenkins-helpers.sh +. $scripts/round-robin.sh + +convert_args_to_variables "$@" + +obligatory_variables rr[ci_config] ssh_host ssh_port + +# Execution mode: baseline, bisect, jenkins-full +rr[mode]="${rr[mode]-baseline}" + +# Set custom revision for one of the projects, and use baseline revisions +# for all other projects. +rr[ci_project]="${rr[ci_project]-tcwg_bmk}" +rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}" +rr[reset_baseline]="${rr[reset_baseline]-false}" +rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}" + +# {toolchain_name}-{toolchain_ver}-{target}-{bmk}-{cflags} +IFS=- read -a ci_config <<EOF +${rr[ci_config]} +EOF +rr[toolchain]=${rr[toolchain]-${ci_config[0]}} +rr[target]=${rr[target]-${ci_config[2]}} +cflags=${cflags-${ci_config[4]}} + +case "${rr[toolchain]}" in + llvm) rr[components]="binutils gcc glibc llvm" ;; + gnu) rr[components]="binutils gcc glibc" ;; + *) assert false ;; +esac + +# Use baseline branches by default. +for c in ${rr[components]}; do + rr[${c}_branch]=${rr[${c}_branch]-baseline} + obligatory_variables rr[${c}_url] +done + +start_at="${start_at-default}" +finish_at="${finish_at-default}" +verbose="${verbose-true}" +verbose2="${verbose2-false}" + +if $verbose2; then set -x; fi + +trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT + +# Set start and finish steps for different modes. +default_start_at="" +default_finish_at="" +case "${rr[mode]}" in + "baseline") + default_finish_at="update_baseline" + ;; + "bisect") + case "$(print_single_updated_component)" in + binutils) default_start_at="build_abe-binutils" ;; + gcc) default_start_at="build_abe-stage1" ;; + glibc) default_start_at="build_abe-glibc" ;; + llvm) default_start_at="build_llvm-true" ;; + *) assert false ;; + esac + default_finish_at="check_regression" + ;; + "jenkins-full") ;; +esac +if [ x"$start_at" = x"default" ]; then + start_at="$default_start_at" +fi +if [ x"$finish_at" = x"default" ]; then + finish_at="$default_finish_at" +fi + +run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$verbose" + +benchmark () +{ + ( + set -euf -o pipefail + + rm -rf $(pwd)/bin + mkdir $(pwd)/bin + + local bmk_flags + bmk_flags="$(echo -$cflags | sed -e "s/_/ -/g" -e "s/LTO/flto/g")" + case "$cflags" in + "Os*"|"Oz*") + # Use static linking for size benchmarking to pickup library + # code size increases in the final executable. + bmk_flags="$bmk_flags -static" + ;; + esac + + local bench_list bin cc gnu_target sysroot + gnu_target=$(print_gnu_target ${rr[target]}) + sysroot="$(pwd)/abe/sysroots/$gnu_target" + case "${rr[toolchain]}" in + llvm) + bench_list="c_and_cxx" + bmk_flags="$bmk_flags --target=$gnu_target --sysroot=$sysroot" + bin="$(pwd)/llvm-install/bin" + cc="$bin/" + ;; + gnu) + bench_list="all" + bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin" + cc="$bin/$gnu_target-" + ;; + esac + sysroot="ssh://$ssh_host:$ssh_port:$sysroot" + + local hw results_id + case "${rr[target]}" in + arm*) hw=tk1 ;; + *) hw=tx1 ;; + esac + local results_id="@hw_type@/${rr[ci_project]}/${rr[ci_config]}-@build_num@" + + remote_exec "ci.linaro.org:2222::-l $USER@linaro.org" \ + build tcwg-benchmark -s -v \ + -p bench_list="$bench_list" \ + -p cflags="$bmk_flags" \ + -p testmode=benchmark \ + -p displaytag="${rr[mode]}-${rr[ci_config]}" \ + -p toolchain_url=ssh://$ssh_host:$ssh_port:$cc \ + -p toolchain_type=${rr[toolchain]} \ + -p sysroot="$sysroot" \ + -p results_id="$results_id" \ + -p target_list="$hw" \ + ${scripts_branch+-p scripts_branch="$scripts_branch"} \ + ${bmk_branch+-p bmk_branch="$bmk_branch"} \ + | tee $run_step_artifacts/benchmark.log + local build_num + build_num=$(tail -n1 $run_step_artifacts/benchmark.log \ + | grep SUCCESS \ + | sed -e "s/.*#\([0-9]\+\)-.*/\1/") + + echo "$results_id" | sed -e "s/@build_num@/$build_num/g" \ + > "${rr[top_artifacts]}/results_id" + ) +} + +# Exit with code 0 if no regression compared to base-artifacts/results. +no_regression_p () +{ + ( + set -euf -o pipefail + + if ! [ -f base-artifacts/results ]; then + return 0 + fi + + local build_result_ref build_result_new + build_result_ref=$(tail -n1 base-artifacts/results) + build_result_new=$(tail -n1 ${rr[top_artifacts]}/results) + + if [ $build_result_new -lt $build_result_ref ]; then + return 1 + fi + + if ! [ -f base-artifacts/results_id ]; then + return 0 + elif ! [ -f ${rr[top_artifacts]}/results_id ]; then + return 1 + fi + + local results_ref results_new + results_ref=$(cat base-artifacts/results_id) + results_new=$(cat ${rr[top_artifacts]}/results_id) + + $scripts/tcwg-benchmark-results.sh --results_ref $results_ref --results_id $results_new --top_artifacts "${rr[top_artifacts]}" --verbose $verbose + + # Skip header line + tail -n +2 ${rr[top_artifacts]}/results.csv | \ + while IFS=, read -r bmk symbol speed size; do + # Skip case where we have no info ("n/a") + if [ "$size" != "n/a" ]; then + if [ "$size" -gt 100 ]; then + echo "Regression in bench $bmk, symbol $symbol size increased ($size vs 100)" + return 1 + fi + fi + done + ) +} + +run_step stop_on_fail -10 reset_artifacts +run_step stop_on_fail x prepare_abe +run_step skip_on_fail -9 build_abe binutils +run_step skip_on_fail -8 build_abe stage1 +run_step skip_on_fail -7 build_abe linux +run_step skip_on_fail -6 build_abe glibc +run_step skip_on_fail -5 build_abe stage2 +case "${rr[toolchain]}" in + llvm) run_step skip_on_fail -1 build_llvm true ;; +esac +run_step skip_on_fail 0 benchmark +run_step reset_on_fail x check_regression +run_step stop_on_fail x update_baseline +run_step stop_on_fail x push_baseline + +trap "" EXIT |