diff options
Diffstat (limited to 'tcwg-benchmark-results.sh')
| -rwxr-xr-x | tcwg-benchmark-results.sh | 154 |
1 files changed, 94 insertions, 60 deletions
diff --git a/tcwg-benchmark-results.sh b/tcwg-benchmark-results.sh index 6b45c971..c788d044 100755 --- a/tcwg-benchmark-results.sh +++ b/tcwg-benchmark-results.sh @@ -8,54 +8,20 @@ scripts=$(dirname $0) # shellcheck source=jenkins-helpers.sh . $scripts/jenkins-helpers.sh -function gather_perf_data () -{ - local has_perf_logs=$1 - local hw_tag=$2 - local num=$3 - - if [ x"$has_perf_logs" = xyes ]; then - $scripts/../bmk-scripts/perfdatadir2csv.sh \ - --buildid-dir local --format sample,size --sort-field sample \ - --perf-bin /usr/lib/linux-tools/$hw_tag/perf \ - $verbose_opt $num_entries_opt \ - --results-dir "results-$num/" > "$top_artifacts/results-$num.csv" - else - # No perf logs to parse, just copy the plain results.csv. - # Use 'find' because results.csv is located under - # results-$num/NODE_NAME/ and we don't want to hardcode - # NODE_NAME. Since the whole script runs under 'set -f', using - # '*' does not work. - mapfile -t this_csv < <(find results-$num -name results.csv) - if [ "${#this_csv[@]}" -eq 1 ]; then - cp -v "${this_csv[@]}" "$top_artifacts/results-$num.csv" - else - echo "ERROR: Found ${#this_csv[@]} CSV results files in results-$num, expecting a single one." - exit 1 - fi - fi -} - convert_args_to_variables "$@" -obligatory_variables results +obligatory_variables results hw_tag top_artifacts="${top_artifacts-artifacts}" verbose="${verbose-false}" # shellcheck disable=SC2154 num_entries_opt="${num_dsos+--num-dsos $num_dsos} ${num_symbols+--num-symbols $num_symbols}" entry_threshold="${entry_threshold-5}" -metric="${metric-perf}" -has_perf_logs="${has_perf_logs-no}" - -if [ x"$metric" = x"perf" ]; then - has_perf_logs="yes" -fi +has_perf_logs="${has_perf_logs-yes}" +hw_tag="${hw_tag-unknown}" -verbose_opt="" if $verbose; then set -x - verbose_opt="--verbose" fi relative_opt="" @@ -65,38 +31,106 @@ if [ x"${results_ref+set}" = x"set" ]; then results=("$results_ref" "${results[@]}") num=0 else + results=("${results[@]}") num=1 fi mkdir -p $top_artifacts -results_top="bkp-01.tcwglab:/home/tcwg-benchmark/results" +j=$num +for i in "${results[@]}"; do + # results can be either + # - an existing local dir, create a symlink results-N -> results_dir + # - or a remote bkp-01 dir, rsync it to result-N + rm -rf results-$j + if [ -d $i ]; then + ln -s $i results-$j + else + rsync -az --delete "$i/" results-$j/ + fi + csv_results_dir="$top_artifacts/csv-results-$j" + mkdir -p $csv_results_dir + + $scripts/../bmk-scripts/gather-metrics.sh --results_dir results-$j/ \ + --csv_results_dir $csv_results_dir \ + ++metrics perf \ + --hw_tag $hw_tag --has_perf_logs $has_perf_logs \ + --verbose $verbose $num_entries_opt + j=$((j+1)) +done + +$scripts/../bmk-scripts/gen_interesting_symbols.py \ + --perf_csvs="$top_artifacts/csv-results-1/perf.csv" \ + --out_csv="$top_artifacts/interesting-symbols.csv" \ + --threshold_interesting=$entry_threshold csvs="" for i in "${results[@]}"; do - rsync -az --delete "$results_top-$i/" results-$num/ - - case $metric in - "perf") - hw_tag="${i%%/*}" - gather_perf_data $has_perf_logs $hw_tag $num - ;; - "vect") - $scripts/../bmk-scripts/vect-data-to-csv.py "results-$num" \ - "$top_artifacts/results-$num.csv" - ;; - *) - echo "ERROR: invalid value for metric: ${metric}" - exit 1 - ;; - esac - - csvs="$csvs $top_artifacts/results-$num.csv" + csv_results_dir="$top_artifacts/csv-results-$num" + + # FIXME: This will (needlessly) recompute perf metric. Revisit later + # to avoid that. + $scripts/../bmk-scripts/gather-metrics.sh --results_dir "results-$num/" \ + --csv_results_dir $csv_results_dir \ + --hw_tag $hw_tag --has_perf_logs $has_perf_logs \ + --verbose $verbose $num_entries_opt \ + --interesting_symbols "$top_artifacts/interesting-symbols.csv" + + csvs="$csvs $csv_results_dir/results.csv" num=$(($num+1)) done -$scripts/../bmk-scripts/csvs2table.py -p $entry_threshold $relative_opt $csvs > $top_artifacts/results.csv - -$scripts/../bmk-scripts/csvs2table.py -p 0 $relative_opt $csvs > $top_artifacts/results-full.csv +# Demangling symbol names may introduce commas which will interfere with +# parsing CSV files. So keep a separate copy results-internal.csv, which +# contains mangled names. +$scripts/../bmk-scripts/csvs2table.py -p $entry_threshold $relative_opt $csvs \ + > $top_artifacts/csvs2table-results-internal.csv +cat $top_artifacts/csvs2table-results-internal.csv | c++filt -p > $top_artifacts/csvs2table-results.csv + +$scripts/../bmk-scripts/csvs2table.py -p 0 $relative_opt $csvs | c++filt -p > $top_artifacts/csvs2table-results-full.csv + +$scripts/../bmk-scripts/csvs2table.py -p 99 $relative_opt $csvs | c++filt -p > $top_artifacts/csvs2table-results-brief.csv + +# Currently we only support relative comparison for 2 csvs with compare-results.py. +# If we are using non relative mode, or more than 2 csv files for comparison, fall back +# to csvs2table.py. + +if [ x"$relative_opt" == x"--relative" ] && [ $num == 2 ]; then + $scripts/../bmk-scripts/compare-results.py $csvs \ + --interesting_symbols "$top_artifacts/interesting-symbols.csv" \ + > $top_artifacts/compare-results-internal.csv + + $scripts/../bmk-scripts/diff-bmk-results.py \ + $top_artifacts/compare-results-internal.csv \ + $top_artifacts/csvs2table-results-internal.csv \ + true \ + > $top_artifacts/cmp-results.diff || true + + # Only keep cmp-results.diff if there were differences found between output of + # compare-results.py and csvs2table.py for same input csvs. + if grep -Fxq "Same results" $top_artifacts/cmp-results.diff + then + rm $top_artifacts/cmp-results.diff + fi +fi -$scripts/../bmk-scripts/csvs2table.py -p 99 $relative_opt $csvs > $top_artifacts/results-brief.csv +# We currently have two workflows to produce final list of regressions: +# (a) csvs2table.py +# -> results with rel metrics +# -> tcwg_bmk-build.sh:compare_results +# -> list of exe and symbol regressions. +# +# (b) compare-results.py +# -> results with rel metrics +# -> output-bmk-results.py +# -> list of exe and symbol regressions +# +# (a) is used by default for all cases currently. +# In short term we want to replace, (a) with (b) for relative mode +# with two input csvs, which is the common case for bmk CI. To achieve that, +# (i) Replace csvs2table.py with compare-results.py in (a). +# (ii) If (i) succeeds, then replace tcwg_bmk-build.sh:compare_results +# with output-bmk-results.py. +# In longer term, we should replace (a) with (b) for all cases. + +cp $top_artifacts/csvs2table-results-internal.csv $top_artifacts/results-internal.csv |