round-robin.sh,tcwg_bmk-build.sh,tcwg-benchmark-results.sh: Store bare bmk results in artifacts

round-robin.sh initialize git-annex as pointing on
bkp-01:/home/tcwg-benchmark/base-artifacts, and consider the "annex/" directory
as annex datas to be saved with git-annex. Right now, only tcwg_bmk is using it.

Taking bare results to the artifacts & baseline simplifies a lot the
compare_results routine.

tcwg-benchmark-results.sh no more takes results_id string as inputs,
but directly takes the directories paths of each bmk results.
It takes an additional parameter hw_tag, to avoid precedent results_id
parsing. This means this script currently can only treat inputs
belonging to the same hw_tag.

bmk results are stored in :
 artifacts/annex/bmk-data

We are still keeping results_id file because it is useful to find the
benchmark job id that were used to run the benchmarks.

Change-Id: Ibac5ed3508962f1a32ad3361e429f20c66f7eae3

1 files changed, 36 insertions, 28 deletions

diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh
index d2214994..291abea2 100755
--- a/tcwg_bmk-build.sh
+++ b/tcwg_bmk-build.sh
@@ -304,7 +304,7 @@ benchmark ()
 		-p toolchain_url=$toolchain_proto://$ssh_host:$ssh_port:$cc \
 		-p toolchain_type=$toolchain \
 		-p sysroot="$sysroot" \
-		-p results_id="$results_id" \
+		-p results_dest="ssh://$ssh_host:$ssh_port:${rr[top_artifacts]}/annex/bmk-data" \
 		-p reboot="$reboot" \
 		-p run_profile="$run_profile" \
 		-p image_arch="$image_arch" \
@@ -371,14 +371,7 @@ compare_results ()
     set -euf -o pipefail
 
     local metric_id="$1"
-    local ref_results_id="$2"
-    local new_results_id="$3"
-    local cmp_options="$4"
-
-    assert_with_msg "ERROR: No $new_results_id file" [ -f $new_results_id ]
-
-    local results_ref results_new
-    results_new=$(cat $new_results_id)
+    local cmp_options="$2"
 
     case "${rr[target]}" in
         "arm_eabi")
@@ -394,13 +387,37 @@ compare_results ()
 	sudo /usr/lib/linux-tools/install-armhf-perf-workaround.sh
     fi
 
-    if ! [ -f $ref_results_id ]; then
-	# There's no ref metrics, only a new metrics. We don't try to
-	# generate results.csv, and we consider it as a non-regression.
-	return 0
+    local results_ref results_new
+    results_new="${rr[top_artifacts]}/annex/bmk-data"
+
+    local ref_results_id=${rr[base_artifacts]}/results_id
+    if ! [ -f $ref_results_id ] && ! [ -d "${rr[base_artifacts]}/annex/bmk-data" ]; then
+	# base-artifacts has no reference results, neither in results_id nor in annex
+	# This can happen on initialization of base-artifacts (update_baseline=init)
+	# or in corner-cases like forced build failing during toolchain build.
+	# In such cases we compare results to themselves just as an exercise.
+	results_ref=$results_new
+    elif [ -d "${rr[base_artifacts]}/annex/bmk-data" ]; then
+	# Results are part of the annex of the base-artifacts
+	results_ref="${rr[base_artifacts]}/annex/bmk-data"
+	# Import this annex locally
+	git -C base-artifacts annex init
+	git_set_remote base-artifacts "bkp-01" \
+	   "ssh://bkp-01.tcwglab/home/tcwg-benchmark/base-artifacts"
+	git -C base-artifacts annex get annex
+    else
+	# FIXME :
+	# Otherwise, getting it from bkp-01:/home/tcwg-benchmark/results-.
+	# Probably a builds not using bmk results in annex dir.
+	# This may be useless at some point, once all results are part of base-artifacts annex.
+	results_ref="${rr[base_artifacts]}/annex/bmk-data"
+	mkdir -p $results_ref
+	rsync -az --delete "bkp-01.tcwglab:/home/tcwg-benchmark/results-$(cat $ref_results_id)/" \
+	   $results_ref/
     fi
 
-    results_ref=$(cat $ref_results_id)
+    assert_with_msg "ERROR: No result dir $results_new" [ -d $results_new ]
+    assert_with_msg "ERROR: No reference dir $results_ref" [ -d $results_ref ]
 
     # Compare vs previous run
     mkdir -p ${rr[top_artifacts]}/results-vs-prev
@@ -408,7 +425,8 @@ compare_results ()
     $scripts/tcwg-benchmark-results.sh \
         --results_ref $results_ref ++results $results_new \
         --top_artifacts "${rr[top_artifacts]}/results-vs-prev" \
-	--verbose $verbose $cmp_options \
+	--verbose $verbose --hw_tag "$(tcwg_bmk_hw)" \
+	$cmp_options \
         > ${rr[top_artifacts]}/results-vs-prev/tcwg-benchmark-results.log 2>&1 &
 
     local res
@@ -470,6 +488,9 @@ compare_results ()
 	   cp $resfile ${rr[top_artifacts]}/notify/
 	fi
     done
+
+    assert_with_msg "Found a regression while comparing the build against itself" \
+			[ "$results_new" != "$results_ref" ]
     )
 }
 
@@ -495,16 +516,6 @@ no_regression_p ()
     assert_with_msg "Benchmarking succeeded, but results_id is missing" \
 		    [ -f $run_step_top_artifacts/results_id ]
 
-    local ref_results_id=base-artifacts/results_id
-    if ! [ -f $ref_results_id ]; then
-	# Absent reference results can happen on initialization of
-	# base-artifacts state or in corner-cases like forced build
-	# failing during toolchain build.
-	# In such cases we compare results to themselves just as
-	# an exercise.
-	ref_results_id=$run_step_top_artifacts/results_id
-    fi
-
     # Make sure there is no stray results.regression file, which we use
     # as failure marker.
     assert ! [ -f $run_step_artifacts/results.regressions ]
@@ -519,12 +530,9 @@ no_regression_p ()
     # - ref/results_id might not exist, (ex: baseline score<0)
     # we call compare_results to generate the csv, and check metric regressions.
     compare_results "$metric_id" \
-		    "$ref_results_id" "$run_step_top_artifacts/results_id" \
 		    "$compare_opts"
 
     if [ -f $run_step_artifacts/results.regressions ]; then
-	assert_with_msg "Found a regression while comparing the build against itself" \
-			[ "$ref_results_id" != "$run_step_top_artifacts/results_id" ]
 	return 1
     fi
     return 0