From 692947065e5fae528af249b14e77de58571d9b73 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Tue, 14 Sep 2021 14:34:05 +0000
Subject: tcwg_bmk-build.sh: Rework compare_results_perf to generate regression
 mail

... reports.

Change-Id: I05dcba4b9ad39c2dbca03b9a89ce832f08f66d5a
---
 tcwg_bmk-build.sh | 182 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 165 insertions(+), 17 deletions(-)

(limited to 'tcwg_bmk-build.sh')

diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh
index d8fbde6d..9d93aa16 100755
--- a/tcwg_bmk-build.sh
+++ b/tcwg_bmk-build.sh
@@ -346,55 +346,86 @@ compare_results_perf ()
 	"-Os"*|"-Oz"*)
 	    # We use 1% tolerance for binary size
 	    # and 10% tolerance for symbol size.
-	    exe_threshold=101
-	    symbol_threshold=110
+	    exe_threshold=1
+	    symbol_threshold=10
 	    ;;
 	*)
 	    # We use 3% tolerance for binary speed
 	    # and 15% tolerance for symbol speed.
-	    exe_threshold=103
-	    symbol_threshold=115
+	    exe_threshold=3
+	    symbol_threshold=15
 	    # Reduce thresholds when bisecting to avoid considering borderline
 	    # regressions as spurious.  This should break cycles of build and
 	    # bisect jobs triggering each other on borderline regressions.
 	    if [ x"${rr[mode]}" = x"bisect" ]; then
-		exe_threshold=102
-		symbol_threshold=110
+		exe_threshold=2
+		symbol_threshold=10
 	    fi
 	    ;;
     esac
 
     local -a arr
-    local bmk symbol time size result prev_bmk
+    local bmk symbol metric time2 regression result prev_bmk
     echo "bmk,symbol,result" > $run_step_artifacts/results-compare.csv
     printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
 
+    assert_with_msg "Found stale regression files" \
+		    [ x"$(find $run_step_artifacts/ -name "*.regression" | wc -l)" = x"0" ]
+
+    local regressed_by metric_field
+    case "${cflags[0]}" in
+	"-Os"*|"-Oz"*)
+	    regressed_by="grew in size by"
+	    metric_field=3
+	    ;;
+	*)
+	    regressed_by="slowed down by"
+	    metric_field=2
+	    ;;
+    esac
+
     # Read result lines from <(tail -n +2 ...) below.
     # "-n +2" is to skip the header line.
     prev_bmk=""
     while IFS=, read -a arr; do
 	bmk=${arr[0]}
 	symbol=${arr[1]}
-	time=${arr[2]}
-	size=${arr[3]}
-	case "${cflags[0]}" in
-	    "-Os"*|"-Oz"*) metric="$size" ;;
-	    *) metric="$time" ;;
-	esac
+	metric=${arr[$metric_field]}
+	time2=${arr[5]}
 
 	# Skip case where we have no info ("n/a")
 	if [ "$metric" != "n/a" ]; then
+	    metric=$(($metric - 100))
+	    regression="$regressed_by ${metric}%"
 	    # Remove padding from the tail of $symbol (padding is added by
 	    # csvs2table.py for better formatting).
 	    local short_symbol="${symbol%%[ ]*}"
+
+	    local bmk_exe
 	    case "$short_symbol" in
-		"["*) threshold=$symbol_threshold ;;
-		*"_base.default") threshold=$exe_threshold ;;
-		*) threshold=$symbol_threshold ;;
+		"["*) bmk_exe=false ;;
+		*"_base.default") bmk_exe=true ;;
+		*) bmk_exe=false ;;
 	    esac
+
+	    local threshold
+	    if $bmk_exe; then
+		threshold=$exe_threshold
+	    else
+		threshold=$symbol_threshold
+	    fi
+
 	    if ! [ "$metric" -le "$threshold" ]; then
 		result=100
-		echo "# $bmk,$symbol regressed by $metric" >> $run_step_artifacts/results.regressions
+		if $bmk_exe; then
+		    case "$time2" in
+			888888888) regression="failed to run correctly" ;;
+			999999999) regression="failed to build" ;;
+		    esac
+		    echo "$metric,$bmk,$symbol,$regression" >> $run_step_artifacts/exe.regressions
+		else
+		    echo "$metric,$bmk,$symbol,$regression" >> $run_step_artifacts/$bmk.regression
+		fi
 		if [ x"$bmk" != x"$prev_bmk" ]; then
 		    printf "++benchmarks %s " $bmk >> $run_step_artifacts/extra-bisect-params
 		    prev_bmk="$bmk"
@@ -406,6 +437,123 @@ compare_results_perf ()
 	fi
     done < <(tail -n +2 $run_step_artifacts/results.csv)
     printf "\n" >> $run_step_artifacts/extra-bisect-params
+
+    # Comparison is done.  Below we generate regression report.
+    cat > $run_step_artifacts/jira-body.txt <<EOF
+After \$COMMIT_COMPONENT \$COMMIT_LOG
+EOF
+    if [ -f $run_step_artifacts/exe.regressions ]; then
+	sort -gr -o $run_step_artifacts/exe.regressions \
+	     $run_step_artifacts/exe.regressions
+
+	cat >> $run_step_artifacts/jira-body.txt <<EOF
+
+the following benchmarks $regressed_by more than ${exe_threshold}%:
+EOF
+	while IFS=, read metric bmk symbol regression; do
+	    cat >> $run_step_artifacts/jira-body.txt <<EOF
+- $bmk $regression
+EOF
+	    if [ -f $run_step_artifacts/$bmk.regression ]; then
+		while IFS=, read metric bmk symbol regression; do
+		    cat >> $run_step_artifacts/jira-body.txt <<EOF
+  - $bmk,$symbol $regression
+EOF
+		done < $run_step_artifacts/$bmk.regression
+		# Delete $bmk.regressions so that it doesn't show up
+		# in symbol-regression loop below.
+		rm $run_step_artifacts/$bmk.regression
+	    fi
+	done < $run_step_artifacts/exe.regressions
+    fi
+
+    find $run_step_artifacts/ -name "*.regression" -print0 | xargs -0 cat \
+	| sort -gr -o $run_step_artifacts/symbol.regressions
+    if [ x"$(cat $run_step_artifacts/symbol.regressions)" = x"" ]; then
+	# Delete empty file
+	rm $run_step_artifacts/symbol.regressions
+    fi
+
+    if [ -f $run_step_artifacts/symbol.regressions ]; then
+	cat >> $run_step_artifacts/jira-body.txt <<EOF
+
+the following hot functions $regressed_by more than ${symbol_threshold}% (but their benchmarks $regressed_by less than ${exe_threshold}%):
+EOF
+	while IFS=, read metric bmk symbol regression; do
+	    cat >> $run_step_artifacts/jira-body.txt <<EOF
+- $bmk,$symbol $regression
+EOF
+	done < $run_step_artifacts/symbol.regressions
+    fi
+
+    cp $run_step_artifacts/jira-body.txt $run_step_artifacts/mail-body.txt
+
+    local bmk_suite=""
+    local compiler="" libc="" linker="" version="" target="" bmk_flags="" hw=""
+    case "${benchmarks[*]}" in
+	coremark) bmk_suite="EEMBC CoreMark" ;;
+	spec2k6) bmk_suite="SPEC CPU2006" ;;
+	spec2017) bmk_suite="SPEC CPU2017" ;;
+    esac
+    case "${rr[toolchain]}" in
+	gnu)
+	    compiler="GCC"
+	    libc="Glibc"
+	    linker="GNU Linker"
+	    ;;
+	gnu_eabi)
+	    compiler="GCC"
+	    libc="Newlib"
+	    linker="GNU LD"
+	    ;;
+	llvm)
+	    compiler="Clang"
+	    libc="Glibc"
+	    linker="LLVM Linker"
+	    ;;
+    esac
+    case "${rr[ci_config]}" in
+	*-master-*) version="tip of trunk" ;;
+	*-release-*) version="latest release branch" ;;
+    esac
+    target=$(print_gnu_target ${rr[target]})
+    bmk_flags=$(echo "${cflags[0]}" | sed -e "s/_/ -/g" -e "s/LTO/flto/g" \
+					  -e "s/VECT/fmetric-vect/g")
+    case "${rr[ci_project]}" in
+	*_apm*) hw="APM Mustang 8x X-Gene1" ;;
+	*_tk1*) hw="NVidia TK1 4x Cortex-A15" ;;
+	*_tx1*) hw="NVidia TX1 4x Cortex-A57" ;;
+	*_stm32*) hw="STMicroelectronics STM32L476RGTx 1x Cortex-M4" ;;
+    esac
+
+    cat >> $run_step_artifacts/mail-body.txt <<EOF
+
+Benchmark: $bmk_suite
+Toolchain: $compiler + $libc + $linker
+Version: all components were built from their $version
+Target: $target
+Compiler flags: $bmk_flags
+Hardware: $hw
+EOF
+
+    if [ -f $run_step_artifacts/exe.regressions ]; then
+	IFS=, read metric bmk symbol regression < <(head -n1 $run_step_artifacts/exe.regressions)
+	cat > $run_step_artifacts/mail-subject.txt <<EOF
+[TCWG CI] $bmk $regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
+EOF
+    elif [ -f $run_step_artifacts/symbol.regressions ]; then
+	IFS=, read metric bmk symbol regression < <(head -n1 $run_step_artifacts/symbol.regressions)
+	symbol="${symbol%%[ ]*}"
+	cat > $run_step_artifacts/mail-subject.txt <<EOF
+[TCWG CI] $bmk,$symbol $regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
+EOF
+    else
+	# Exit with no regressions
+	return 0
+    fi
+
+    cat $run_step_artifacts/jira-body.txt \
+	| sed -e "s/^/# /" > $run_step_artifacts/results.regressions
     )
 }
 
-- 
cgit v1.2.3