summaryrefslogtreecommitdiff
path: root/tcwg_bmk-build.sh
diff options
context:
space:
mode:
authorMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-09-14 14:34:05 +0000
committerMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-09-14 14:34:05 +0000
commit692947065e5fae528af249b14e77de58571d9b73 (patch)
treedf6c2817baefe5d6617318c632a4c175e5f8cce2 /tcwg_bmk-build.sh
parent6cc316eb9769603b54253e4063a8292c714e1ab5 (diff)
tcwg_bmk-build.sh: Rework compare_results_perf to generate regression mail
... reports. Change-Id: I05dcba4b9ad39c2dbca03b9a89ce832f08f66d5a
Diffstat (limited to 'tcwg_bmk-build.sh')
-rwxr-xr-xtcwg_bmk-build.sh182
1 files changed, 165 insertions, 17 deletions
diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh
index d8fbde6d..9d93aa16 100755
--- a/tcwg_bmk-build.sh
+++ b/tcwg_bmk-build.sh
@@ -346,55 +346,86 @@ compare_results_perf ()
"-Os"*|"-Oz"*)
# We use 1% tolerance for binary size
# and 10% tolerance for symbol size.
- exe_threshold=101
- symbol_threshold=110
+ exe_threshold=1
+ symbol_threshold=10
;;
*)
# We use 3% tolerance for binary speed
# and 15% tolerance for symbol speed.
- exe_threshold=103
- symbol_threshold=115
+ exe_threshold=3
+ symbol_threshold=15
# Reduce thresholds when bisecting to avoid considering borderline
# regressions as spurious. This should break cycles of build and
# bisect jobs triggering each other on borderline regressions.
if [ x"${rr[mode]}" = x"bisect" ]; then
- exe_threshold=102
- symbol_threshold=110
+ exe_threshold=2
+ symbol_threshold=10
fi
;;
esac
local -a arr
- local bmk symbol time size result prev_bmk
+ local bmk symbol metric time2 regression result prev_bmk
echo "bmk,symbol,result" > $run_step_artifacts/results-compare.csv
printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params
+ assert_with_msg "Found stale regression files" \
+ [ x"$(find $run_step_artifacts/ -name "*.regression" | wc -l)" = x"0" ]
+
+ local regressed_by metric_field
+ case "${cflags[0]}" in
+ "-Os"*|"-Oz"*)
+ regressed_by="grew in size by"
+ metric_field=3
+ ;;
+ *)
+ regressed_by="slowed down by"
+ metric_field=2
+ ;;
+ esac
+
# Read result lines from <(tail -n +2 ...) below.
# "-n +2" is to skip the header line.
prev_bmk=""
while IFS=, read -a arr; do
bmk=${arr[0]}
symbol=${arr[1]}
- time=${arr[2]}
- size=${arr[3]}
- case "${cflags[0]}" in
- "-Os"*|"-Oz"*) metric="$size" ;;
- *) metric="$time" ;;
- esac
+ metric=${arr[$metric_field]}
+ time2=${arr[5]}
# Skip case where we have no info ("n/a")
if [ "$metric" != "n/a" ]; then
+ metric=$(($metric - 100))
+ regression="$regressed_by ${metric}%"
# Remove padding from the tail of $symbol (padding is added by
# csvs2table.py for better formatting).
local short_symbol="${symbol%%[ ]*}"
+
+ local bmk_exe
case "$short_symbol" in
- "["*) threshold=$symbol_threshold ;;
- *"_base.default") threshold=$exe_threshold ;;
- *) threshold=$symbol_threshold ;;
+ "["*) bmk_exe=false ;;
+ *"_base.default") bmk_exe=true ;;
+ *) bmk_exe=false ;;
esac
+
+ local threshold
+ if $bmk_exe; then
+ threshold=$exe_threshold
+ else
+ threshold=$symbol_threshold
+ fi
+
if ! [ "$metric" -le "$threshold" ]; then
result=100
- echo "# $bmk,$symbol regressed by $metric" >> $run_step_artifacts/results.regressions
+ if $bmk_exe; then
+ case "$time2" in
+ 888888888) regression="failed to run correctly" ;;
+ 999999999) regression="failed to build" ;;
+ esac
+ echo "$metric,$bmk,$symbol,$regression" >> $run_step_artifacts/exe.regressions
+ else
+ echo "$metric,$bmk,$symbol,$regression" >> $run_step_artifacts/$bmk.regression
+ fi
if [ x"$bmk" != x"$prev_bmk" ]; then
printf "++benchmarks %s " $bmk >> $run_step_artifacts/extra-bisect-params
prev_bmk="$bmk"
@@ -406,6 +437,123 @@ compare_results_perf ()
fi
done < <(tail -n +2 $run_step_artifacts/results.csv)
printf "\n" >> $run_step_artifacts/extra-bisect-params
+
+ # Comparison is done. Below we generate regression report.
+ cat > $run_step_artifacts/jira-body.txt <<EOF
+After \$COMMIT_COMPONENT \$COMMIT_LOG
+EOF
+ if [ -f $run_step_artifacts/exe.regressions ]; then
+ sort -gr -o $run_step_artifacts/exe.regressions \
+ $run_step_artifacts/exe.regressions
+
+ cat >> $run_step_artifacts/jira-body.txt <<EOF
+
+the following benchmarks $regressed_by more than ${exe_threshold}%:
+EOF
+ while IFS=, read metric bmk symbol regression; do
+ cat >> $run_step_artifacts/jira-body.txt <<EOF
+- $bmk $regression
+EOF
+ if [ -f $run_step_artifacts/$bmk.regression ]; then
+ while IFS=, read metric bmk symbol regression; do
+ cat >> $run_step_artifacts/jira-body.txt <<EOF
+ - $bmk,$symbol $regression
+EOF
+ done < $run_step_artifacts/$bmk.regression
+ # Delete $bmk.regressions so that it doesn't show up
+ # in symbol-regression loop below.
+ rm $run_step_artifacts/$bmk.regression
+ fi
+ done < $run_step_artifacts/exe.regressions
+ fi
+
+ find $run_step_artifacts/ -name "*.regression" -print0 | xargs -0 cat \
+ | sort -gr -o $run_step_artifacts/symbol.regressions
+ if [ x"$(cat $run_step_artifacts/symbol.regressions)" = x"" ]; then
+ # Delete empty file
+ rm $run_step_artifacts/symbol.regressions
+ fi
+
+ if [ -f $run_step_artifacts/symbol.regressions ]; then
+ cat >> $run_step_artifacts/jira-body.txt <<EOF
+
+the following hot functions $regressed_by more than ${symbol_threshold}% (but their benchmarks $regressed_by less than ${exe_threshold}%):
+EOF
+ while IFS=, read metric bmk symbol regression; do
+ cat >> $run_step_artifacts/jira-body.txt <<EOF
+- $bmk,$symbol $regression
+EOF
+ done < $run_step_artifacts/symbol.regressions
+ fi
+
+ cp $run_step_artifacts/jira-body.txt $run_step_artifacts/mail-body.txt
+
+ local bmk_suite=""
+ local compiler="" libc="" linker="" version="" target="" bmk_flags="" hw=""
+ case "${benchmarks[*]}" in
+ coremark) bmk_suite="EEMBC CoreMark" ;;
+ spec2k6) bmk_suite="SPEC CPU2006" ;;
+ spec2017) bmk_suite="SPEC CPU2017" ;;
+ esac
+ case "${rr[toolchain]}" in
+ gnu)
+ compiler="GCC"
+ libc="Glibc"
+ linker="GNU Linker"
+ ;;
+ gnu_eabi)
+ compiler="GCC"
+ libc="Newlib"
+ linker="GNU LD"
+ ;;
+ llvm)
+ compiler="Clang"
+ libc="Glibc"
+ linker="LLVM Linker"
+ ;;
+ esac
+ case "${rr[ci_config]}" in
+ *-master-*) version="tip of trunk" ;;
+ *-release-*) version="latest release branch" ;;
+ esac
+ target=$(print_gnu_target ${rr[target]})
+ bmk_flags=$(echo "${cflags[0]}" | sed -e "s/_/ -/g" -e "s/LTO/flto/g" \
+ -e "s/VECT/fmetric-vect/g")
+ case "${rr[ci_project]}" in
+ *_apm*) hw="APM Mustang 8x X-Gene1" ;;
+ *_tk1*) hw="NVidia TK1 4x Cortex-A15" ;;
+ *_tx1*) hw="NVidia TX1 4x Cortex-A57" ;;
+ *_stm32*) hw="STMicroelectronics STM32L476RGTx 1x Cortex-M4" ;;
+ esac
+
+ cat >> $run_step_artifacts/mail-body.txt <<EOF
+
+Benchmark: $bmk_suite
+Toolchain: $compiler + $libc + $linker
+Version: all components were built from their $version
+Target: $target
+Compiler flags: $bmk_flags
+Hardware: $hw
+EOF
+
+ if [ -f $run_step_artifacts/exe.regressions ]; then
+ IFS=, read metric bmk symbol regression < <(head -n1 $run_step_artifacts/exe.regressions)
+ cat > $run_step_artifacts/mail-subject.txt <<EOF
+[TCWG CI] $bmk $regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
+EOF
+ elif [ -f $run_step_artifacts/symbol.regressions ]; then
+ IFS=, read metric bmk symbol regression < <(head -n1 $run_step_artifacts/symbol.regressions)
+ symbol="${symbol%%[ ]*}"
+ cat > $run_step_artifacts/mail-subject.txt <<EOF
+[TCWG CI] $bmk,$symbol $regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT
+EOF
+ else
+ # Exit with no regressions
+ return 0
+ fi
+
+ cat $run_step_artifacts/jira-body.txt \
+ | sed -e "s/^/# /" > $run_step_artifacts/results.regressions
)
}