blob: c788d04490359d70f16af90587b3a5bcea55da39 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
#!/bin/bash
# Clean: shellcheck -e 2001 ./tcwg-benchmark-results.sh
set -euf -o pipefail
scripts=$(dirname $0)
# shellcheck source=jenkins-helpers.sh
. $scripts/jenkins-helpers.sh
convert_args_to_variables "$@"
obligatory_variables results hw_tag
top_artifacts="${top_artifacts-artifacts}"
verbose="${verbose-false}"
# shellcheck disable=SC2154
num_entries_opt="${num_dsos+--num-dsos $num_dsos} ${num_symbols+--num-symbols $num_symbols}"
entry_threshold="${entry_threshold-5}"
has_perf_logs="${has_perf_logs-yes}"
hw_tag="${hw_tag-unknown}"
if $verbose; then
set -x
fi
relative_opt=""
# shellcheck disable=SC2154
if [ x"${results_ref+set}" = x"set" ]; then
relative_opt="--relative"
results=("$results_ref" "${results[@]}")
num=0
else
results=("${results[@]}")
num=1
fi
mkdir -p $top_artifacts
j=$num
for i in "${results[@]}"; do
# results can be either
# - an existing local dir, create a symlink results-N -> results_dir
# - or a remote bkp-01 dir, rsync it to result-N
rm -rf results-$j
if [ -d $i ]; then
ln -s $i results-$j
else
rsync -az --delete "$i/" results-$j/
fi
csv_results_dir="$top_artifacts/csv-results-$j"
mkdir -p $csv_results_dir
$scripts/../bmk-scripts/gather-metrics.sh --results_dir results-$j/ \
--csv_results_dir $csv_results_dir \
++metrics perf \
--hw_tag $hw_tag --has_perf_logs $has_perf_logs \
--verbose $verbose $num_entries_opt
j=$((j+1))
done
$scripts/../bmk-scripts/gen_interesting_symbols.py \
--perf_csvs="$top_artifacts/csv-results-1/perf.csv" \
--out_csv="$top_artifacts/interesting-symbols.csv" \
--threshold_interesting=$entry_threshold
csvs=""
for i in "${results[@]}"; do
csv_results_dir="$top_artifacts/csv-results-$num"
# FIXME: This will (needlessly) recompute perf metric. Revisit later
# to avoid that.
$scripts/../bmk-scripts/gather-metrics.sh --results_dir "results-$num/" \
--csv_results_dir $csv_results_dir \
--hw_tag $hw_tag --has_perf_logs $has_perf_logs \
--verbose $verbose $num_entries_opt \
--interesting_symbols "$top_artifacts/interesting-symbols.csv"
csvs="$csvs $csv_results_dir/results.csv"
num=$(($num+1))
done
# Demangling symbol names may introduce commas which will interfere with
# parsing CSV files. So keep a separate copy results-internal.csv, which
# contains mangled names.
$scripts/../bmk-scripts/csvs2table.py -p $entry_threshold $relative_opt $csvs \
> $top_artifacts/csvs2table-results-internal.csv
cat $top_artifacts/csvs2table-results-internal.csv | c++filt -p > $top_artifacts/csvs2table-results.csv
$scripts/../bmk-scripts/csvs2table.py -p 0 $relative_opt $csvs | c++filt -p > $top_artifacts/csvs2table-results-full.csv
$scripts/../bmk-scripts/csvs2table.py -p 99 $relative_opt $csvs | c++filt -p > $top_artifacts/csvs2table-results-brief.csv
# Currently we only support relative comparison for 2 csvs with compare-results.py.
# If we are using non relative mode, or more than 2 csv files for comparison, fall back
# to csvs2table.py.
if [ x"$relative_opt" == x"--relative" ] && [ $num == 2 ]; then
$scripts/../bmk-scripts/compare-results.py $csvs \
--interesting_symbols "$top_artifacts/interesting-symbols.csv" \
> $top_artifacts/compare-results-internal.csv
$scripts/../bmk-scripts/diff-bmk-results.py \
$top_artifacts/compare-results-internal.csv \
$top_artifacts/csvs2table-results-internal.csv \
true \
> $top_artifacts/cmp-results.diff || true
# Only keep cmp-results.diff if there were differences found between output of
# compare-results.py and csvs2table.py for same input csvs.
if grep -Fxq "Same results" $top_artifacts/cmp-results.diff
then
rm $top_artifacts/cmp-results.diff
fi
fi
# We currently have two workflows to produce final list of regressions:
# (a) csvs2table.py
# -> results with rel metrics
# -> tcwg_bmk-build.sh:compare_results
# -> list of exe and symbol regressions.
#
# (b) compare-results.py
# -> results with rel metrics
# -> output-bmk-results.py
# -> list of exe and symbol regressions
#
# (a) is used by default for all cases currently.
# In short term we want to replace, (a) with (b) for relative mode
# with two input csvs, which is the common case for bmk CI. To achieve that,
# (i) Replace csvs2table.py with compare-results.py in (a).
# (ii) If (i) succeeds, then replace tcwg_bmk-build.sh:compare_results
# with output-bmk-results.py.
# In longer term, we should replace (a) with (b) for all cases.
cp $top_artifacts/csvs2table-results-internal.csv $top_artifacts/results-internal.csv
|
