tcwg_bmk-build.sh: Create mail everytime

Change-Id: I0ee4ceb8c58bfc17d1cafea73b361c7b35a4356d

1 files changed, 99 insertions, 60 deletions

diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh
index 99b2d2ea..975abd40 100755
--- a/tcwg_bmk-build.sh
+++ b/tcwg_bmk-build.sh
@@ -407,7 +407,7 @@ compare_results ()
 	return $EXTERNAL_FAIL
     fi
 
-    local regressed_by exe_threshold symbol_threshold
+    local regressed_by improved_by changed_by exe_threshold symbol_threshold
 
     case $metric_id in
 	perf)
@@ -419,6 +419,7 @@ compare_results ()
 		    symbol_threshold=10
 		    metric_id="size"
 		    regressed_by="grew in size by"
+		    improved_by="reduced in size by"
 		    ;;
 		*)
 		    # We use 3% tolerance for binary speed
@@ -436,6 +437,7 @@ compare_results ()
 
 		    metric_id="time"
 		    regressed_by="slowed down by"
+		    improved_by="speeds up by"
 		    ;;
 	    esac
 	    ;;
@@ -443,6 +445,7 @@ compare_results ()
 	    exe_threshold=0
 	    symbol_threshold=0
 	    regressed_by="reduced by"
+	    improved_by="increased up by"
 	    ;;
 	*)
 	    assert false
@@ -451,7 +454,7 @@ compare_results ()
 
     local -a arr
     local metric bmk symbol rtime rsize rvect time1 time2 size1 size2 vect1 vect2
-    local regression short_symbol short_regression
+    local long_diag short_symbol short_diag
     local result prev_bmk
     echo "bmk,symbol,result" > $run_step_artifacts/results-compare.csv
     printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
@@ -518,49 +521,67 @@ compare_results ()
 		*) bmk_exe=false ;;
 	    esac
 
-	    local threshold
+	    local threshold neg_threshold
 	    if $bmk_exe; then
 		threshold=$exe_threshold
+		neg_threshold=$((-exe_threshold))
 	    else
 		threshold=$symbol_threshold
+		neg_threshold=$((-symbol_threshold))
 	    fi
 
-	    if ! [ "$metric" -le "$threshold" ]; then
+	    if [ "$metric" -gt "$threshold" ] || [ "$metric" -lt "$neg_threshold" ]; then
+	        if [ "$metric" -gt "$threshold" ]; then
+		   change_kind="regression"
+		   changed_by=$regressed_by
+		else
+		   change_kind="improvement"
+		   changed_by=$improved_by
+		fi
+
+		# There's a regression
 		result=100
 
 		case $metric_id in
 		    size)
-			short_regression="$regressed_by ${metric}%"
-			regression="$short_regression from $size1 to $size2 bytes"
+			short_diag="$changed_by ${metric}%"
+			long_diag="$short_diag from $size1 to $size2 bytes"
 			;;
 		    time)
-			short_regression="$regressed_by ${metric}%"
-			regression="$short_regression from $time1 to $time2 perf samples" ;;
+			short_diag="$changed_by ${metric}%"
+			long_diag="$short_diag from $time1 to $time2 perf samples" ;;
 		    vect)
-			short_regression="$regressed_by ${metric}%"
-			regression="$short_regression from $vect1 to $vect2" ;;
+			short_diag="$changed_by ${metric}%"
+			long_diag="$short_diag from $vect1 to $vect2" ;;
 		    *) assert false ;;
 		esac
+
 		if $bmk_exe; then
-		    short_regression="$bmk $short_regression"
-		    regression="$bmk $regression"
+		    short_diag="$bmk $short_diag"
+		    long_diag="$bmk $long_diag"
 		    # Detect magic sample counts that indicate failure to build
 		    # and failure to run
-		    case "$time2" in
-			888888888)
-			    short_regression="$bmk failed to run correctly"
-			    regression="$short_regression"
+		    case "$time1:$time2" in
+			# 888888888:888888888) ;; Will never happen here as we have either an improvement or a regression.
+			# 999999999:999999999) ;; Will never happen here as we have either an improvement or a regression.
+			888888888:999999999|*:999999999)
+			    short_diag="$bmk failed to build"
+			    long_diag="$short_diag"
+			    ;;
+			999999999:888888888)
+			    short_diag="$bmk now well builds, but not run"
+			    long_diag="$short_diag"
 			    ;;
-			999999999)
-			    short_regression="$bmk failed to build"
-			    regression="$short_regression"
+			*:888888888)
+			    short_diag="$bmk failed to run"
+			    long_diag="$short_diag"
 			    ;;
 		    esac
-		    echo "$metric,$bmk,$symbol,$short_regression,$regression" >> $run_step_artifacts/exe.regressions
+		    echo "$metric,$bmk,$symbol,$short_diag,$long_diag" >> $run_step_artifacts/exe.$change_kind
 		else
-		    short_regression="$bmk:$short_symbol $short_regression"
-		    regression="$bmk:$short_symbol $regression"
-		    echo "$metric,$bmk,$symbol,$short_regression,$regression" >> $run_step_artifacts/$bmk.regression
+		    short_diag="$bmk:$short_symbol $short_diag"
+		    long_diag="$bmk:$short_symbol $long_diag"
+		    echo "$metric,$bmk,$symbol,$short_diag,$long_diag" >> $run_step_artifacts/$bmk.symbols-$change_kind
 		fi
 		if [ x"$bmk" != x"$prev_bmk" ]; then
 		    printf "++benchmarks %s " $bmk >> $run_step_artifacts/extra-bisect-params
@@ -575,58 +596,76 @@ compare_results ()
     # the latter may contain commas in demangled symbol names, which will
     # interfere with parsing.
     done < <(tail -n +2 $run_step_artifacts/results-internal.csv)
+
+
     printf "\n" >> $run_step_artifacts/extra-bisect-params
 
     # Comparison is done.  Below we generate regression report.
-    cat > $run_step_artifacts/jira-body.txt <<EOF
+    cat > $run_step_artifacts/mail-body.txt <<EOF
 After \$COMMIT_COMPONENT \$COMMIT_LOG
 EOF
-    if [ -f $run_step_artifacts/exe.regressions ]; then
-	sort -gr -o $run_step_artifacts/exe.regressions \
-	     $run_step_artifacts/exe.regressions
 
-	cat >> $run_step_artifacts/jira-body.txt <<EOF
+    # aggregate all *.symbol-regression into a file symbol.regression
+    find $run_step_artifacts/ -name "*.symbols-regression" -print0 | xargs -0 cat \
+	| sort -gr -o $run_step_artifacts/symbol.regression
+    if [ x"$(cat $run_step_artifacts/symbol.regression)" = x"" ]; then
+	# Delete empty file
+	rm $run_step_artifacts/symbol.regression
+    fi
+
+
+    # If there's one regression. Don't bother about improvements.
+    if [ -f $run_step_artifacts/exe.regression ] || [ -f $run_step_artifacts/symbol.regression ]; then
+        change_kind=regression
+    else
+        change_kind=improvement
+    fi
+
+
+    if [ -f $run_step_artifacts/exe.$change_kind ]; then
+	sort -gr -o $run_step_artifacts/exe.$change_kind \
+	     $run_step_artifacts/exe.$change_kind
+
+	cat >> $run_step_artifacts/mail-body.txt <<EOF
 
 the following benchmarks $regressed_by more than ${exe_threshold}%:
 EOF
 	local exe
-	while IFS=, read metric exe symbol short_regression regression; do
-	    cat >> $run_step_artifacts/jira-body.txt <<EOF
-- $regression
+	while IFS=, read metric exe symbol short_diag long_diag; do
+	    cat >> $run_step_artifacts/mail-body.txt <<EOF
+- $long_diag
 EOF
-	    if [ -f $run_step_artifacts/$exe.regression ]; then
-		while IFS=, read metric bmk symbol short_regression regression; do
-		    cat >> $run_step_artifacts/jira-body.txt <<EOF
-  - $regression
+	    if [ -f $run_step_artifacts/$exe.symbols-$change_kind ]; then
+		while IFS=, read metric bmk symbol short_diag long_diag; do
+		    cat >> $run_step_artifacts/mail-body.txt <<EOF
+  - $long_diag
 EOF
-		done < $run_step_artifacts/$exe.regression
+		done < $run_step_artifacts/$exe.symbols-$change_kind
 		# Delete $bmk.regressions so that it doesn't show up
 		# in symbol-regression loop below.
-		rm $run_step_artifacts/$exe.regression
+		rm $run_step_artifacts/$exe.symbols-$change_kind
 	    fi
-	done < $run_step_artifacts/exe.regressions
+	done < $run_step_artifacts/exe.$change_kind
     fi
 
-    find $run_step_artifacts/ -name "*.regression" -print0 | xargs -0 cat \
-	| sort -gr -o $run_step_artifacts/symbol.regressions
-    if [ x"$(cat $run_step_artifacts/symbol.regressions)" = x"" ]; then
-	# Delete empty file
-	rm $run_step_artifacts/symbol.regressions
-    fi
 
-    if [ -f $run_step_artifacts/symbol.regressions ]; then
-	cat >> $run_step_artifacts/jira-body.txt <<EOF
+    if [ -f $run_step_artifacts/symbol.$change_kind ]; then
+	cat >> $run_step_artifacts/mail-body.txt <<EOF
 
 the following hot functions $regressed_by more than ${symbol_threshold}% (but their benchmarks $regressed_by less than ${exe_threshold}%):
 EOF
-	while IFS=, read metric bmk symbol short_regression regression; do
-	    cat >> $run_step_artifacts/jira-body.txt <<EOF
-- $regression
+	while IFS=, read metric bmk symbol short_diag long_diag; do
+	    cat >> $run_step_artifacts/mail-body.txt <<EOF
+- $long_diag
 EOF
-	done < $run_step_artifacts/symbol.regressions
+	done < $run_step_artifacts/symbol.$change_kind
     fi
 
-    cp $run_step_artifacts/jira-body.txt $run_step_artifacts/mail-body.txt
+
+    # generate jira-body only if regression
+    if [ "$change_kind" == "regression" ];then
+      cp $run_step_artifacts/mail-body.txt $run_step_artifacts/jira-body.txt
+    fi
 
     local bmk_suite="" publish_save_temps=false
     case "${benchmarks[*]}" in
@@ -643,7 +682,7 @@ EOF
 
     cat >> $run_step_artifacts/mail-body.txt <<EOF
 
-Below reproducer instructions can be used to re-build both "first_bad" and "last_good" cross-toolchains used in this bisection.  Naturally, the scripts will fail when triggerring benchmarking jobs if you don't have access to Linaro TCWG CI.
+Below reproducer instructions can be used to re-build both "first_bad" and "last_good" cross-toolchains used in this bisection.  Naturally, the scripts will fail when triggerring benchmarking jobs if you don\'t have access to Linaro TCWG CI.
 EOF
 
     # Copy save-temps tarballs to artifacts, so that they are accessible.
@@ -715,23 +754,23 @@ This benchmarking CI is work-in-progress, and we welcome feedback and suggestion
 EOF
 
     # Generate mail subject
-    if [ -f $run_step_artifacts/exe.regressions ]; then
-	IFS=, read metric bmk symbol short_regression regression \
-	   < <(head -n1 $run_step_artifacts/exe.regressions)
-    elif [ -f $run_step_artifacts/symbol.regressions ]; then
-	IFS=, read metric bmk symbol short_regression regression \
-	   < <(head -n1 $run_step_artifacts/symbol.regressions)
+    if [ -f $run_step_artifacts/exe.$change_kind ]; then
+	IFS=, read metric bmk symbol short_diag long_diag \
+	   < <(head -n1 $run_step_artifacts/exe.$change_kind)
+    elif [ -f $run_step_artifacts/symbol.$change_kind ]; then
+	IFS=, read metric bmk symbol short_diag long_diag \
+	   < <(head -n1 $run_step_artifacts/symbol.$change_kind)
     else
 	# Exit with no regressions
 	return 0
     fi
 
     cat > $run_step_artifacts/mail-subject.txt <<EOF
-[TCWG CI] $short_regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
+[TCWG CI] $short_diag after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
 EOF
     cat $run_step_artifacts/jira-body.txt \
 	| sed -e "s/^/# /" > $run_step_artifacts/results.regressions
-    echo "# $short_regression" >> $run_step_artifacts/results.regressions
+    echo "# $short_diag" >> $run_step_artifacts/results.regressions
     )
 }