[WIP] tcwg_bmk project

Change-Id: Ie0faa9b60e094f7f0ecd607d56d259dcff649382

9 files changed, 257 insertions, 80 deletions

diff --git a/docker-run.sh b/docker-run.sh
index b1914333..2af4c8e3 100755
--- a/docker-run.sh
+++ b/docker-run.sh
@@ -1,14 +1,14 @@
 #!/bin/bash
 
-set -ef -o pipefail
-
-qemu="${qemu-}"
-set -u
+set -euf -o pipefail
 
 . $(dirname $0)/jenkins-helpers.sh
 convert_args_to_variables "$@"
 shift "$SHIFT_CONVERTED_ARGS"
 
+qemu="${qemu-}"
+ssh_info="${ssh_info-false}"
+
 trap "container_cleanup" EXIT
 
 case "$qemu" in
@@ -17,4 +17,10 @@ case "$qemu" in
 esac
 . ./container.sh
 
-container_exec "$@"
+if $ssh_info; then
+    ssh_opts="--ssh_host $session_host --ssh_port $session_port"
+else
+    ssh_opts=""
+fi
+
+container_exec "$@" $ssh_opts
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh
index 10b7081a..a2fb51d7 100644
--- a/jenkins-helpers.sh
+++ b/jenkins-helpers.sh
@@ -646,11 +646,11 @@ print_volume_mounts ()
     local volume_id
 
     case "$job" in
-	tcwg_kernel-*)
+	tcwg_kernel-*|tcwg_bmk-*)
 	    # Add ccache volume for tcwg_kernel jobs.  These jobs
 	    # depend on ccache for fast rebuilds of LLVM and GCC with
 	    # the host compiler.
-	    job="tcwg_kernel"
+	    job="$(echo $job | cut -d- -f 1)"
 	    volume_id=$(print_docker_name "$job$suffix")
 	    mounts+=(ccache-$volume_id:$HOME/.ccache)
 	    ;;
diff --git a/round-robin.sh b/round-robin.sh
index a09ef49c..d1f201a3 100644
--- a/round-robin.sh
+++ b/round-robin.sh
@@ -14,7 +14,7 @@
 
 # Baseline branch name for current configuration.  These branches are
 # present in all above git repos.
-#rr[baseline_branch]="linaro-local/ci/CONFIGURATION"
+#rr[baseline_branch]
 
 # PROJECT that we are testing in this build.  Use ${rr[current_branch]} for
 # this project, and ${rr[baseline_branch]} for all other projects.
@@ -60,6 +60,11 @@ reset_artifacts ()
     # base-artifacts repo is big and changes all the time, so we
     # fetch only the $baseline_branch, instead of all branches.
     rr[base-artifacts_rev]="${rr[base-artifacts_rev]-${rr[baseline_branch]}}"
+    if ${rr[init_configuration]-false}; then
+	rr[base-artifacts_rev]=empty
+	single_branch=empty
+    fi
+
     clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch
     cat <<EOF | manifest_out
 rr[base-artifacts_rev]=$(git -C base-artifacts rev-parse HEAD)
@@ -87,12 +92,14 @@ clone_repo ()
     # and everything else uses baseline branch.
     if [ x"$project" = x"${rr[current_project]}" ]; then
 	if [ x"${rr[current_branch]}" = x"default" ]; then
-	    branch=${rr[${current_project}_branch]}
+	    branch=${rr[${rr[current_project]}_branch]}
 	elif [ x"${rr[current_branch]}" = x"baseline" ]; then
 	    branch="refs/remotes/baseline/${rr[baseline_branch]}"
 	else
 	    branch=${rr[current_branch]}
 	fi
+    elif ${rr[init_configuration]-false}; then
+	branch="${rr[${project}_branch]}"
     else
 	branch="refs/remotes/baseline/${rr[baseline_branch]}"
     fi
@@ -123,7 +130,8 @@ clone_repo ()
 rr[${project}_rev]=$cur_rev
 EOF
 
-    if [ x"$project" = x"${rr[current_project]}" ]; then
+    if [ x"$project" = x"${rr[current_project]}" ] \
+	   && ! ${rr[init_configuration]-false}; then
 	local baseline_rev
 	baseline_rev=$(git_rev_parse_long $project ${rr[baseline_branch]} baseline)
 	# Prepare for failure.  If build fails we will bisect sha1 for
diff --git a/start-container-docker.sh b/start-container-docker.sh
index a0f09aa7..ac6d1536 100755
--- a/start-container-docker.sh
+++ b/start-container-docker.sh
@@ -109,6 +109,9 @@ do
 	    [ x${session_name} = x ] && usage
 	    shift 2
 	    ;;
+	--ssh_info)
+	    shift 2
+	    ;;
 	--task)
 	    task=$2
 	    [ x${task} = x ] && usage
@@ -196,11 +199,6 @@ fi
 
 image=linaro/ci-${container_arch}-tcwg-${task}-ubuntu:${distro}
 
-case "$session_host" in
-    "localhost"*) docker_host="" ;;
-    *) docker_host="$session_host" ;;
-esac
-
 DOCKER="ssh $session_host docker"
 $DOCKER pull $image
 
diff --git a/tcwg-benchmark-results.sh b/tcwg-benchmark-results.sh
index 245e492a..f7147f8c 100755
--- a/tcwg-benchmark-results.sh
+++ b/tcwg-benchmark-results.sh
@@ -2,31 +2,35 @@
 
 # Clean: shellcheck -e 2001 ./tcwg-benchmark-results.sh
 
-set -ex
+set -ef -o pipefail
 
-# Make shellcheck happy and workaround Jenkins not defining variables
-# for empty arguments.
-results_id="$results_id"
-results_ref="$results_ref"
+scripts=$(dirname $0)
+. $scripts/jenkins-helpers.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables results_id results_ref
+
+top_artifacts="${top_artifacts-artifacts}"
+verbose="${verbose-true}"
 
 # Jenkins doesn't define variables when parameter value is empty (like cflags),
 # so enable "set -u" only after above binding of variables.
 set -u
 
-results_top="dev-01.tcwglab:/home/tcwg-benchmark/results"
+if $verbose; then set -x; fi
 
-rsync -az --delete "$results_top-$results_id/" results/
-rsync -az --delete "$results_top-$results_ref/" baseline/
+results_top="dev-01.tcwglab:/home/tcwg-benchmark/results"
 
-for i in baseline results; do
-    bmk-scripts/perfdatadir2csv.sh --buildid-dir local -t --num 0 --format sample --sort-field sample --results-dir "$i/" > "time-$i.csv"
-    bmk-scripts/perfdatadir2csv.sh --buildid-dir local --format sample --sort-field sample --results-dir "$i/" > "sample-$i.csv"
+num=0
+csvs=""
+for i in $results_ref $results_id; do
+    rsync -az --delete "$results_top-$i/" results-$num/
+    $scripts/../bmk-scripts/perfdatadir2csv.sh --buildid-dir local --format sample,size --sort-field sample --results-dir "results-$num/" > "results-$num.csv"
+    csvs="$csvs results-$num.csv"
 done
+ssh -O exit dev-01.tcwglab
 
-mkdir -p artifacts
-
-bmk-scripts/csvs2table.py --relative time-baseline.csv time-results.csv > artifacts/time.csv
-
-bmk-scripts/csvs2table.py --relative sample-baseline.csv sample-results.csv > artifacts/sample.csv
+mkdir -p $top_artifacts
 
-exit 0
+$scripts/../bmk-scripts/csvs2table.py --relative $csvs > $top_artifacts/results.csv
diff --git a/tcwg-benchmark.sh b/tcwg-benchmark.sh
index 0b1e125e..fd9000bc 100755
--- a/tcwg-benchmark.sh
+++ b/tcwg-benchmark.sh
@@ -69,13 +69,6 @@ case "$toolchain_url" in
 	    echo "ERROR: ssh:// toolchain_url lacks a port: $toolchain_url."
 	    exit 1
 	fi
-
-	# Create a helper function, like we have when we create a
-	# container.
-	build_container_exec ()
-	{
-	    remote_exec "${build}:$(pwd)" "$@"
-	}
 	;;
     *)
 	# Make sure to cleanup build container if something goes
@@ -243,21 +236,19 @@ case "$testmode" in
     benchmark) input_size="ref" ;;
 esac
 
-build_container_exec ssh -t "$boardname" bmk-scripts/run.sh \
-                     --bench "$(printf '%q' "$bench_list")" \
-                     --config "${BUILD_NUMBER}-$run_profile" \
-                     --cflags "$(printf '%q' "$cflags")" \
-                     --ccprefix "$ccprefix" \
-		     --extension "$extension" \
-		     --ignore_errors "$ignore_errors" \
-		     --input_size "$input_size" \
-		     --iterations "$iterations" \
-		     --run_profile "$run_profile" \
-                     "${sysroot:+--sysroot "$sysroot"}" \
-                     --toolchain "$toolchain_type" \
-                     --resultsdest "dev-01.tcwglab:/home/tcwg-benchmark/results-${results_id}/${NODE_NAME}" \
-                     --nodename "${NODE_NAME}" \
-                     --forceinstall "${forceinstall}" \
-                     --verbose true
-
-exit 0
+ssh -t -Snone "$boardname" bmk-scripts/run.sh \
+    --bench "$(printf '%q' "$bench_list")" \
+    --config "${BUILD_NUMBER}-$run_profile" \
+    --cflags "$(printf '%q' "$cflags")" \
+    --ccprefix "$ccprefix" \
+    --extension "$extension" \
+    --ignore_errors "$ignore_errors" \
+    --input_size "$input_size" \
+    --iterations "$iterations" \
+    --run_profile "$run_profile" \
+    "${sysroot:+--sysroot "$sysroot"}" \
+    --toolchain "$toolchain_type" \
+    --resultsdest "dev-01.tcwglab:/home/tcwg-benchmark/results-${results_id}/${NODE_NAME}" \
+    --nodename "${NODE_NAME}" \
+    --forceinstall "${forceinstall}" \
+    --verbose true
diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh
new file mode 100755
index 00000000..809c097c
--- /dev/null
+++ b/tcwg_bmk-build.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+scripts=$(dirname $0)
+. $scripts/jenkins-helpers.sh
+
+convert_args_to_variables "$@"
+
+obligatory_variables rr[ci_config] \
+		     BUILD_NUMBER ssh_host ssh_port
+
+# Execution mode: baseline, bisect, continue, jenkins-full
+rr[mode]="${rr[mode]-baseline}"
+
+case "${rr[mode]}" in
+    "jenkins-full")
+	obligatory_variables rr[current_project] rr[current_branch]
+	;;
+esac
+
+# Set custom revision for one of the projects, and use baseline revisions
+# for all other projects.
+rr[ci_project]="${rr[ci_project]-tcwg_bmk}"
+rr[baseline_branch]="${rr[baseline_branch]-linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}}"
+rr[current_branch]="${rr[current_branch]-default}"
+rr[current_project]="${rr[current_project]-none}"
+rr[reset_baseline]="${rr[reset_baseline]-false}"
+rr[top_artifacts]="${rr[top_artifacts]-$(pwd)/artifacts}"
+
+# {toolchain_name}-{toolchain_ver}-{target}-{bmk}-{cflags}
+IFS=- read -a ci_config <<EOF
+${rr[ci_config]}
+EOF
+rr[toolchain_name]=${rr[toolchain_name]-${ci_config[0]}}
+rr[target]=${rr[target]-${ci_config[2]}}
+cflags=${cflags-${ci_config[4]}}
+
+start_at="${start_at-default}"
+finish_at="${finish_at-default}"
+verbose="${verbose-true}"
+verbose2="${verbose2-false}"
+
+results_id="tx1/${rr[ci_project]}/${rr[ci_config]}-$BUILD_NUMBER"
+
+if $verbose2; then set -x; fi
+
+trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\"" EXIT
+
+# Set start and finish steps for different modes.
+default_start_at=""
+default_finish_at=""
+base_artifacts_opt="base-artifacts"
+case "${rr[mode]}" in
+    "baseline")
+	default_finish_at="update_baseline"
+	;;
+    "bisect")
+	case "${rr[toolchain]}:${rr[current_project]}" in
+	    gnu:binutils) default_start_at="build_abe-binutils" ;;
+	    gnu:gcc) default_start_at="build_abe-stage1" ;;
+	    llvm:llvm) default_start_at="build_llvm" ;;
+	esac
+	default_finish_at="check_regression"
+	;;
+    "continue")
+	# Developer mode.
+	case "${rr[toolchain]}" in
+	    "gnu") default_start_at="prepare_abe" ;;
+	    "llvm") default_start_at="build_llvm" ;;
+	esac
+	base_artifacts_opt=""
+	;;
+    "jenkins-full") ;;
+esac
+if [ x"$start_at" = x"default" ]; then
+    start_at="$default_start_at"
+fi
+if [ x"$finish_at" = x"default" ]; then
+    finish_at="$default_finish_at"
+fi
+
+run_step_init "$start_at" "$finish_at" "${rr[top_artifacts]}" "$base_artifacts_opt" "$verbose"
+
+. $scripts/round-robin.sh
+
+benchmark ()
+{
+    (
+    set -euf -o pipefail
+
+    rm -rf $(pwd)/bin
+    mkdir $(pwd)/bin
+
+    local bin cc
+
+    bin="$(pwd)/abe/builds/destdir/x86_64-unknown-linux-gnu/bin"
+    cc="$bin/$(print_gnu_target ${rr[target]})-"
+
+    local hw
+    case "${rr[target]}" in
+	arm*) hw=tk1 ;;
+	*) hw=tx1 ;;
+    esac
+
+    local bmk_flags
+    bmk_flags="$(echo _$cflags | sed -e "s/_/ -/g")"
+    case "$cflags" in
+	"Os*") bmk_flags="$bmk_flags -static" ;;
+    esac
+    
+    ssh -p2222 -l $USER@linaro.org ci.linaro.org build tcwg-benchmark -s -v \
+	-p bench_list=all \
+	-p cflags="$bmk_flags" \
+	-p displaytag=${rr[ci_config]} \
+	-p toolchain_url=ssh://$ssh_host:$ssh_port:$cc \
+	-p sysroot=ssh://$ssh_host:$ssh_port:$(pwd)/abe/sysroots/$(print_gnu_target ${rr[target]}) \
+	-p results_id="$results_id" \
+	-p scripts_branch="$scripts_branch" \
+	-p bmk_branch="$bmk_branch"
+
+    echo "$results_id" > "${rr[top_artifacts]}/results_id"
+    cat >> ${rr[top_artifacts]}/results <<EOF
+Succesful benchmark run: $results_id
+0
+EOF
+    )
+}
+
+# Exit with code 0 if no regression compared to base-artifacts/results.
+no_regression_p ()
+{
+    (
+    set -euf -o pipefail
+
+    if ! [ -f base-artifacts/results ]; then
+	return 0
+    fi
+    
+    local build_result_ref build_result_new
+    build_result_ref=$(tail -n1 base-artifacts/results)
+    build_result_new=$(tail -n1 ${rr[top_artifacts]}/results)
+
+    if [ $build_result_new -lt $build_result_ref ]; then
+	return 1
+    fi
+
+    if ! [ -f base-artifacts/results_id ]; then
+	return 0
+    fi
+
+    local results_ref
+    results_ref=$(cat base-artifacts/results_id)
+
+    $scripts/tcwg-benchmark-results.sh --results_ref $results_ref --results_id $results_id --top_artifacts "${rr[top_artifacts]}" --verbose $verbose
+
+    while IFS=, read -r bmk symbol speed size; do
+	if [ "$size" -gt 100 ]; then
+	    return 1
+	fi
+    done < ${rr[top_artifacts]}/results.csv
+    )
+}
+
+run_step stop_on_fail -10 reset_artifacts
+case "${rr[toolchain]}" in
+    gnu)
+	run_step stop_on_fail x prepare_abe
+	run_step skip_on_fail -9 build_abe binutils
+	run_step skip_on_fail -8 build_abe stage1
+	run_step skip_on_fail -7 build_abe linux
+	run_step skip_on_fail -6 build_abe glibc
+	run_step skip_on_fail -5 build_abe stage2
+	;;
+    llvm)
+	run_step skip_on_fail -1 build_llvm
+	;;
+esac
+run_step skip_on_fail x benchmark
+run_step reset_on_fail x check_regression
+run_step stop_on_fail x update_baseline
+run_step stop_on_fail x push_baseline
+
+trap "" EXIT
diff --git a/tcwg_kernel-bisect.sh b/tcwg_kernel-bisect.sh
index 196633a4..e4d11e2b 100755
--- a/tcwg_kernel-bisect.sh
+++ b/tcwg_kernel-bisect.sh
@@ -16,7 +16,8 @@ fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*"
 convert_args_to_variables "$@"
 shift "$SHIFT_CONVERTED_ARGS"
 
-obligatory_variables bad_rev baseline_rev build_script ci_project ci_config
+obligatory_variables bad_rev baseline_rev build_script \
+		     rr[ci_project] rr[ci_config]
 
 BUILD_URL="${BUILD_URL:-$(pwd)}"
 reproduce_bisect="${reproduce_bisect:-false}"
@@ -27,7 +28,7 @@ $reproduce_bisect || manifest_pop
 
 obligatory_variables current_project
 
-if [ x"$ci_project" = x"tcwg_kernel" ]; then
+if [ x"${rr[ci_project]}" = x"tcwg_kernel" ]; then
     obligatory_variables toolchain
 fi
 
@@ -41,7 +42,7 @@ trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $artifacts/fai
 
 rebase_workaround=false
 
-if [ x"$ci_project" = x"tcwg_kernel" ] \
+if [ x"${rr[ci_project]}" = x"tcwg_kernel" ] \
        && [ x"$current_project" = x"linux" -a x"${rr[linux_version]}" = x"next" ]; then
     # Workaround linux-next/master rebasing on top of linux-next/stable.
     # Search for regressions against linux-mainline:master (aka linux-next:stable).
@@ -250,7 +251,7 @@ if [ -f $artifacts/first-bad ]; then
     ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh"
 
     cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Successfully identified regression in CI configuration $ci_config for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev .
+Successfully identified regression in CI configuration ${rr[ci_config]} for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev .
 
 Culprit:
 <cut>
@@ -264,7 +265,7 @@ else
     bad_name="bad"
     bad_sha1="$bad_rev"
     cat >> $artifacts/jenkins/mail-body.txt <<EOF
-Could not identify regression in CI configuration $ci_config for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev down to a single commit.  See 'Bisect log' in the links below for bisection details.
+Could not identify regression in CI configuration ${rr[ci_config]} for $current_project between bad_rev $bad_rev and baseline_rev $baseline_rev down to a single commit.  See 'Bisect log' in the links below for bisection details.
 
 EOF
 fi
@@ -314,7 +315,7 @@ git checkout --detach $good_sha1
 cd ..
 </cut>
 
-History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/$ci_project/$ci_config
+History of pending regressions and results: https://git.linaro.org/toolchain/ci/base-artifacts.git/log/?h=linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}
 
 Bisect log: ${BUILD_URL}artifact/$rel_artifacts/bisect.log/*view*/
 Artifacts: ${BUILD_URL}artifact/$rel_artifacts/
diff --git a/tcwg_kernel-build.sh b/tcwg_kernel-build.sh
index 6eda2248..5f64b163 100755
--- a/tcwg_kernel-build.sh
+++ b/tcwg_kernel-build.sh
@@ -10,17 +10,7 @@ convert_args_to_variables "$@"
 # Execution mode: baseline, bisect, continue, jenkins-full
 mode="${mode-baseline}"
 
-obligatory_variables toolchain \
-		     rr[linux_version] rr[linux_url] rr[linux_branch]
-
-# Toolchain to use: gnu or llvm
-case "$toolchain" in
-    "gnu") obligatory_variables rr[gnu_version] \
-				rr[binutils_url] rr[binutils_branch] \
-				rr[gcc_url] rr[gcc_branch] ;;
-    "llvm") obligatory_variables rr[llvm_version] \
-				 rr[llvm_url] rr[llvm_branch] ;;
-esac
+obligatory_variables target toolchain
 
 case "$mode" in
     "jenkins-full")
@@ -37,7 +27,6 @@ linux_config="${linux_config-defconfig}"
 reset_baseline="${reset_baseline-false}"
 start_at="${start_at-default}"
 finish_at="${finish_at-default}"
-target="${target-native}"
 top_artifacts="${top_artifacts-$(pwd)/artifacts}"
 verbose="${verbose-true}"
 verbose2="${verbose2-false}"
@@ -48,10 +37,6 @@ if $verbose2; then set -x; fi
 
 trap "eval \"echo ERROR at \${FUNCNAME[0]}:\${BASH_LINENO[0]}\" > $top_artifacts/failures" EXIT
 
-if [ x"$target" = x"native" ]; then
-    target=$(uname -m)
-fi
-
 # Set start and finish steps for different modes.
 default_start_at=""
 default_finish_at=""
@@ -94,7 +79,7 @@ fi
 
 run_step_init "$start_at" "$finish_at" "$top_artifacts" "$base_artifacts_opt" "$verbose"
 
-rr[baseline_branch]="linaro-local/ci/tcwg_kernel/$toolchain-${rr[${toolchain}_version]}-$target-${rr[linux_version]}-$linux_config"
+rr[baseline_branch]="linaro-local/ci/${rr[ci_project]}/${rr[ci_config]}"
 rr[current_project]="$current_project"
 rr[current_branch]="$current_branch"
 rr[mode]="$mode"