diff options
author | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2021-09-14 14:34:05 +0000 |
---|---|---|
committer | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2021-09-14 14:34:05 +0000 |
commit | 692947065e5fae528af249b14e77de58571d9b73 (patch) | |
tree | df6c2817baefe5d6617318c632a4c175e5f8cce2 | |
parent | 6cc316eb9769603b54253e4063a8292c714e1ab5 (diff) |
tcwg_bmk-build.sh: Rework compare_results_perf to generate regression mail
... reports.
Change-Id: I05dcba4b9ad39c2dbca03b9a89ce832f08f66d5a
-rwxr-xr-x | tcwg_bmk-build.sh | 182 |
1 files changed, 165 insertions, 17 deletions
diff --git a/tcwg_bmk-build.sh b/tcwg_bmk-build.sh index d8fbde6d..9d93aa16 100755 --- a/tcwg_bmk-build.sh +++ b/tcwg_bmk-build.sh @@ -346,55 +346,86 @@ compare_results_perf () "-Os"*|"-Oz"*) # We use 1% tolerance for binary size # and 10% tolerance for symbol size. - exe_threshold=101 - symbol_threshold=110 + exe_threshold=1 + symbol_threshold=10 ;; *) # We use 3% tolerance for binary speed # and 15% tolerance for symbol speed. - exe_threshold=103 - symbol_threshold=115 + exe_threshold=3 + symbol_threshold=15 # Reduce thresholds when bisecting to avoid considering borderline # regressions as spurious. This should break cycles of build and # bisect jobs triggering each other on borderline regressions. if [ x"${rr[mode]}" = x"bisect" ]; then - exe_threshold=102 - symbol_threshold=110 + exe_threshold=2 + symbol_threshold=10 fi ;; esac local -a arr - local bmk symbol time size result prev_bmk + local bmk symbol metric time2 regression result prev_bmk echo "bmk,symbol,result" > $run_step_artifacts/results-compare.csv printf "extra_build_params=" > $run_step_artifacts/extra-bisect-params + assert_with_msg "Found stale regression files" \ + [ x"$(find $run_step_artifacts/ -name "*.regression" | wc -l)" = x"0" ] + + local regressed_by metric_field + case "${cflags[0]}" in + "-Os"*|"-Oz"*) + regressed_by="grew in size by" + metric_field=3 + ;; + *) + regressed_by="slowed down by" + metric_field=2 + ;; + esac + # Read result lines from <(tail -n +2 ...) below. # "-n +2" is to skip the header line. prev_bmk="" while IFS=, read -a arr; do bmk=${arr[0]} symbol=${arr[1]} - time=${arr[2]} - size=${arr[3]} - case "${cflags[0]}" in - "-Os"*|"-Oz"*) metric="$size" ;; - *) metric="$time" ;; - esac + metric=${arr[$metric_field]} + time2=${arr[5]} # Skip case where we have no info ("n/a") if [ "$metric" != "n/a" ]; then + metric=$(($metric - 100)) + regression="$regressed_by ${metric}%" # Remove padding from the tail of $symbol (padding is added by # csvs2table.py for better formatting). local short_symbol="${symbol%%[ ]*}" + + local bmk_exe case "$short_symbol" in - "["*) threshold=$symbol_threshold ;; - *"_base.default") threshold=$exe_threshold ;; - *) threshold=$symbol_threshold ;; + "["*) bmk_exe=false ;; + *"_base.default") bmk_exe=true ;; + *) bmk_exe=false ;; esac + + local threshold + if $bmk_exe; then + threshold=$exe_threshold + else + threshold=$symbol_threshold + fi + if ! [ "$metric" -le "$threshold" ]; then result=100 - echo "# $bmk,$symbol regressed by $metric" >> $run_step_artifacts/results.regressions + if $bmk_exe; then + case "$time2" in + 888888888) regression="failed to run correctly" ;; + 999999999) regression="failed to build" ;; + esac + echo "$metric,$bmk,$symbol,$regression" >> $run_step_artifacts/exe.regressions + else + echo "$metric,$bmk,$symbol,$regression" >> $run_step_artifacts/$bmk.regression + fi if [ x"$bmk" != x"$prev_bmk" ]; then printf "++benchmarks %s " $bmk >> $run_step_artifacts/extra-bisect-params prev_bmk="$bmk" @@ -406,6 +437,123 @@ compare_results_perf () fi done < <(tail -n +2 $run_step_artifacts/results.csv) printf "\n" >> $run_step_artifacts/extra-bisect-params + + # Comparison is done. Below we generate regression report. + cat > $run_step_artifacts/jira-body.txt <<EOF +After \$COMMIT_COMPONENT \$COMMIT_LOG +EOF + if [ -f $run_step_artifacts/exe.regressions ]; then + sort -gr -o $run_step_artifacts/exe.regressions \ + $run_step_artifacts/exe.regressions + + cat >> $run_step_artifacts/jira-body.txt <<EOF + +the following benchmarks $regressed_by more than ${exe_threshold}%: +EOF + while IFS=, read metric bmk symbol regression; do + cat >> $run_step_artifacts/jira-body.txt <<EOF +- $bmk $regression +EOF + if [ -f $run_step_artifacts/$bmk.regression ]; then + while IFS=, read metric bmk symbol regression; do + cat >> $run_step_artifacts/jira-body.txt <<EOF + - $bmk,$symbol $regression +EOF + done < $run_step_artifacts/$bmk.regression + # Delete $bmk.regressions so that it doesn't show up + # in symbol-regression loop below. + rm $run_step_artifacts/$bmk.regression + fi + done < $run_step_artifacts/exe.regressions + fi + + find $run_step_artifacts/ -name "*.regression" -print0 | xargs -0 cat \ + | sort -gr -o $run_step_artifacts/symbol.regressions + if [ x"$(cat $run_step_artifacts/symbol.regressions)" = x"" ]; then + # Delete empty file + rm $run_step_artifacts/symbol.regressions + fi + + if [ -f $run_step_artifacts/symbol.regressions ]; then + cat >> $run_step_artifacts/jira-body.txt <<EOF + +the following hot functions $regressed_by more than ${symbol_threshold}% (but their benchmarks $regressed_by less than ${exe_threshold}%): +EOF + while IFS=, read metric bmk symbol regression; do + cat >> $run_step_artifacts/jira-body.txt <<EOF +- $bmk,$symbol $regression +EOF + done < $run_step_artifacts/symbol.regressions + fi + + cp $run_step_artifacts/jira-body.txt $run_step_artifacts/mail-body.txt + + local bmk_suite="" + local compiler="" libc="" linker="" version="" target="" bmk_flags="" hw="" + case "${benchmarks[*]}" in + coremark) bmk_suite="EEMBC CoreMark" ;; + spec2k6) bmk_suite="SPEC CPU2006" ;; + spec2017) bmk_suite="SPEC CPU2017" ;; + esac + case "${rr[toolchain]}" in + gnu) + compiler="GCC" + libc="Glibc" + linker="GNU Linker" + ;; + gnu_eabi) + compiler="GCC" + libc="Newlib" + linker="GNU LD" + ;; + llvm) + compiler="Clang" + libc="Glibc" + linker="LLVM Linker" + ;; + esac + case "${rr[ci_config]}" in + *-master-*) version="tip of trunk" ;; + *-release-*) version="latest release branch" ;; + esac + target=$(print_gnu_target ${rr[target]}) + bmk_flags=$(echo "${cflags[0]}" | sed -e "s/_/ -/g" -e "s/LTO/flto/g" \ + -e "s/VECT/fmetric-vect/g") + case "${rr[ci_project]}" in + *_apm*) hw="APM Mustang 8x X-Gene1" ;; + *_tk1*) hw="NVidia TK1 4x Cortex-A15" ;; + *_tx1*) hw="NVidia TX1 4x Cortex-A57" ;; + *_stm32*) hw="STMicroelectronics STM32L476RGTx 1x Cortex-M4" ;; + esac + + cat >> $run_step_artifacts/mail-body.txt <<EOF + +Benchmark: $bmk_suite +Toolchain: $compiler + $libc + $linker +Version: all components were built from their $version +Target: $target +Compiler flags: $bmk_flags +Hardware: $hw +EOF + + if [ -f $run_step_artifacts/exe.regressions ]; then + IFS=, read metric bmk symbol regression < <(head -n1 $run_step_artifacts/exe.regressions) + cat > $run_step_artifacts/mail-subject.txt <<EOF +[TCWG CI] $bmk $regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT +EOF + elif [ -f $run_step_artifacts/symbol.regressions ]; then + IFS=, read metric bmk symbol regression < <(head -n1 $run_step_artifacts/symbol.regressions) + symbol="${symbol%%[ ]*}" + cat > $run_step_artifacts/mail-subject.txt <<EOF +[TCWG CI] $bmk,$symbol $regression after \$COMMIT_COMPONENT: \$COMMIT_SUBJECT +EOF + else + # Exit with no regressions + return 0 + fi + + cat $run_step_artifacts/jira-body.txt \ + | sed -e "s/^/# /" > $run_step_artifacts/results.regressions ) } |