diff options
Diffstat (limited to 'tcwg-wip/push-results-to-squad.sh')
-rwxr-xr-x | tcwg-wip/push-results-to-squad.sh | 364 |
1 files changed, 364 insertions, 0 deletions
diff --git a/tcwg-wip/push-results-to-squad.sh b/tcwg-wip/push-results-to-squad.sh new file mode 100755 index 00000000..d387db03 --- /dev/null +++ b/tcwg-wip/push-results-to-squad.sh @@ -0,0 +1,364 @@ +#!/bin/bash -f + +# Example to push a base-artifacts branch : +# tcwg-wip/push-results-to-squad.sh --branch linaro-local/ci/tcwg_bmk_gnu_apm/gnu-master-aarch64-spec2k6-Os_LTO --push true + + +set -e + +scripts=$(dirname "$0")/.. +# shellcheck source=jenkins-helpers.sh +. "$scripts/jenkins-helpers.sh" + +convert_args_to_variables "$@" +obligatory_variables branch + +branch=${branch-} +push=${push-false} +dryrun=${dryrun-false} +verbose=${verbose-false} + +if [ "$dryrun" == "false" ]; then + curl="curl" +else + echo "DRYRUN : Will show but not push result" + curl="echo curl" +fi + + +# --------------------------- UTILS -------------------------------- +verbose() +{ + if $verbose; then + echo "$*" + fi +} + +error() +{ + echo "" + echo "ERROR : $*" + echo "" + exit 1 +} + +first_time=true +clean_squad_project_if_first_time() +{ + if $first_time; then + rm -rf squad_inputs/${project_run['project']} + fi + first_time=false +} + +# --------------------------- GET INFOS -------------------------------- +get_infos_from_artifacts () +{ + local artifacts_dir=$1 + local artifacts_sha1=$2 + local artifacts_branch=$3 + + echo "# base-artifact : rev=$sha1 ($artifacts_branch)" + git -C $artifact_dir reset -q --hard + git -C $artifact_dir checkout -q -f $artifacts_sha1 + + local manifest + [ -f "$artifacts_dir/manifest.sh" ] && manifest=manifest.sh + [ -f "$artifacts_dir/jenkins/manifest.sh" ] && manifest=jenkins/manifest.sh + [ x"$manifest" == x"" ] && error "Manifest not found" + + declare -A rr + # shellcheck disable=SC1090 + source $artifacts_dir/$manifest + + if [ -d $artifacts_dir/dashboard ]; then + # Use dashboard directory + : + else + + # general + project_run['artifact_topdir']="$artifacts_dir" + project_run['project']=$(echo $BUILD_URL |cut -d/ -f5) + project_run['run']=$(echo $BUILD_URL |cut -d/ -f6) + project_run['url']="$BUILD_URL" + + # tool versions + project_run['version_binutils']=${rr[binutils_rev]} + project_run['version_gcc']=${rr[gcc_rev]} + project_run['version_glibc']=${rr[glibc_rev]} + project_run['version_linux']=${rr[linux_rev]} + [ -v "rr[llvm_rev]" ] && project_run['version_llvm']=${rr[llvm_rev]} + + # url + project_run['artifacts_url_base']="https://git.linaro.org/toolchain/ci/base-artifacts.git/tree" + project_run['artifacts_url_branch_and_version']="h=$artifacts_branch&id=$artifacts_sha1" + project_run['artifacts_url_results']="${project_run['artifacts_url_base']}/?${project_run['artifacts_url_branch_and_version']}" + + # relative to artifact topdir + benchmark_dir=$(cd $artifacts_dir && find . -maxdepth 2 -name "*-benchmark") + [ x"$benchmark_dir" == x"" ] && error "Benchmark dir not found" + + check_regression_dir=$(cd $artifacts_dir && find . -maxdepth 2 -name "*-check_regression") + [ x"$check_regression_dir" == x"" ] && error "Regression dir not found" + + [ -f "$artifacts_dir/manifest.sh" ] && project_run['manifest']="manifest.sh" + [ -f "$artifacts_dir/jenkins/manifest.sh" ] && project_run['manifest']="jenkins/manifest.sh" + [ x"${project_run['manifest']}" == x"" ] && error "Manifest not found" + + [ -f "$artifacts_dir/$benchmark_dir/benchmark-build.log" ] && project_run['benchmark_log']="$benchmark_dir/benchmark-build.log" + [ -f "$artifacts_dir/$benchmark_dir/benchmark.log" ] && project_run['benchmark_log']="$benchmark_dir/benchmark.log" + [ x"${project_run['benchmark_log']}" == x"" ] && error "Benchmark log not found" + + [ -f "$artifacts_dir/$check_regression_dir/csv-results-1/results.csv" ] && project_run['results_csv']="$check_regression_dir/csv-results-1/results.csv" + [ -f "$artifacts_dir/$check_regression_dir/results-1.csv" ] && project_run['results_csv']="$check_regression_dir/results-1.csv" + [ x"${project_run['results_csv']}" == x"" ] && error "results.csv log not found" + + project_run['datetime']=$(git -C $artifact_dir show --no-patch --format=%cI) + fi +} + + +# --------------------------- GET INFOS -------------------------------- +# uses RESULTS_FILE argument +# outputs project_results +parse_results_csv_files() +{ + echo " * parsing base-artifacts inputs" + echo " ${project_run['artifact_topdir']}/${project_run['results_csv']}" + + IFS=',' + # benchmark, symbol, sample, size, num_vect_loops, symbol_md5sum + while read bench symb perf size vect md5sum + do + [[ $symb == *.default* ]] || continue + + if [[ -v project_results["func:$bench"] ]] && [ ${project_results["func:$bench"]} != "pass" ]; then + : # do nothing if fail is already + else + case $perf in + 999999999) + project_results["func:$bench"]="build_fail" + # project_results["perf:$bench"]="" + # project_results["size:$bench"]="" + # project_results["vect:$bench"]="" + ;; + 888888888) + project_results["func:$bench"]="run_fail" + [ $vect != "-1" ] && project_results["vect:$bench"]="$vect" + [ $size != "-1" ] && project_results["size:$bench"]="$size" + # project_results["perf:$bench"]="" + ;; + *) + project_results["func:$bench"]="pass" + [ $vect != "-1" ] && project_results["vect:$bench"]="$vect" + [ $size != "-1" ] && project_results["size:$bench"]="$size" + [ $perf != "-1" ] && project_results["perf:$bench"]="$perf" + ;; + esac + fi + + #echo " # $bench,$symb,$perf,$size,$vect,$md5sum" + verbose " $bench : ${project_results["func:$bench"]}, ${project_results["perf:$bench"]}, " \ + "${project_results["size:$bench"]}, ${project_results["vect:$bench"]}, $md5sum" + done < "${project_run['artifact_topdir']}/${project_run['results_csv']}" + unset IFS + + # hh:mm:ss Finished: SUCCESS + project_run['run_status']=$(grep '^Finished: ' ${project_run['artifact_topdir']}/${project_run['benchmark_log']} | tail -1 | cut -d: -f2) +} + + + +# --------------------------- WRITING OUTPUTS -------------------------------- +# uses project_results[] project_run[] +# dump files in ${project_run['project_run_dir']} +dump_json_for_squad() +{ + local nbtest=0 nbpass=0 nbfail=0 nbskip=0 + + outdir=squad_inputs/${project_run['project']}/${project_run['run']} + rm -rf $outdir + mkdir -p $outdir + + clean_squad_project_if_first_time + + echo " * writing json for squad" + + for key in "${!project_results[@]}"; do + + kind=$(echo $key | cut -d: -f1) + bench=$(echo $key | cut -d: -f2) + [[ "$kind" == "func" ]] || continue + + # echo " - writing $outdir/$bench # ${project_results["func:$bench"]}" + mkdir -p $outdir/$bench + + case ${project_results["func:$bench"]} in + "build_fail") + build_func="fail"; ((nbfail+=1)); + run_func="skip"; ((nbskip+=1)); + ((nbtest+=2)); + ;; + "run_fail") + build_func="pass"; ((nbpass+=1)); + run_func="fail"; ((nbfail+=1)); + ((nbtest+=2)); + ;; + "pass") + build_func="pass"; ((nbpass+=1)); + run_func="pass"; ((nbpass+=1)); + ((nbtest+=2)); + ;; + esac + + resfile=$outdir/$bench/results-functional.json + echo "{" >> $resfile + echo " \"build\" : \"$build_func\"", >> $resfile + echo " \"run\" : \"$run_func\"" >> $resfile + echo "}" >> $resfile + + resfile=$outdir/$bench/results-metrics.json + echo "{" >> $resfile + [[ -v project_results["perf:$bench"] ]] && echo " \"perf\" : \"${project_results["perf:$bench"]}\"," >> $resfile + [[ -v project_results["size:$bench"] ]] && echo " \"size\" : \"${project_results["size:$bench"]}\"," >> $resfile + [[ -v project_results["vect:$bench"] ]] && echo " \"vect\" : \"${project_results["vect:$bench"]}\"" >> $resfile + echo "}" >> $resfile + + resfile=$outdir/$bench/results-metadata.json + echo "{" >> $resfile + # general infos + echo " \"datetime\": \"${project_run['datetime']}\"," >> $resfile + echo " \"build_url\": \"${project_run['url']}\"," >> $resfile + echo " \"build_log\": \"${project_run['url']}console\"," >> $resfile + echo " \"job_status\": \"${project_run['run_status']}\"," >> $resfile + + # tool versions + echo " \"version_binutils\": \"${project_run['version_binutils']}\"," >> $resfile + echo " \"version_gcc\": \"${project_run['version_gcc']}\"," >> $resfile + echo " \"version_glibc\": \"${project_run['version_glibc']}\"," >> $resfile + echo " \"version_linux\": \"${project_run['version_linux']}\"," >> $resfile + [[ -v project_run['version_llvm'] ]] && echo " \"version_llvm\": \"${project_run['version_llvm']}\"," >> $resfile + + # links to base-artifacts + echo " \"artifact_results\": \"${project_run['artifacts_url_base']}/" \ + "?${project_run['artifacts_url_branch_and_version']}\"," >> $resfile + echo " \"build_manifest\": \"${project_run['artifacts_url_base']}/" \ + "${project_run['manifest']}?${project_run['artifacts_url_branch_and_version']}\"," >> $resfile + echo " \"benchmark_log\": \"${project_run['artifacts_url_base']}/"\ + "${project_run['benchmark_log']}?${project_run['artifacts_url_branch_and_version']}\"" >> $resfile + + echo "}" >> $resfile + + done + project_run['squad_input_dir']=$outdir + + echo " $outdir ($nbtest test, $nbpass pass, $nbfail fail, $nbskip skip)" +} + + +push_results_to_squad_server() +{ + local outdir=squad_inputs/${project_run['project']}/${project_run['run']} + local pushcmdfile=$outdir/push_results_to_squad.sh + rm -f $pushcmdfile + + echo " * generating cmd to push results" + + + SQUAD_TOKEN="e1e2856837b6c89c08e1031f3263e290b5aeefb5" + + squad_server=https://qa-reports.linaro.org/ + GRP="~laurent.alfonsi" + + echo "#!/bin/bash -f" > $pushcmdfile + echo "cd $(pwd)" >> $pushcmdfile + echo "" >> $pushcmdfile + + for key in "${!project_results[@]}"; do + + kind=$(echo $key | cut -d: -f1) + bench=$(echo $key | cut -d: -f2) + [[ "$kind" == "func" ]] || continue + + # A VERIFIER + PRJ="${project_run['project']}" + BLD="${project_run['run']}" + ENV="$bench" + + SQUAD_URL=$squad_server/api/submit/$GRP/$PRJ/$BLD/$ENV + + # result files + RESULTS_RESULTS=${project_run['squad_input_dir']}/$bench/results-functional.json + RESULTS_METRICS=${project_run['squad_input_dir']}/$bench/results-metrics.json + RESULTS_METADATA=${project_run['squad_input_dir']}/$bench/results-metadata.json + + echo $curl \ + --header \"Authorization: Token $SQUAD_TOKEN\" \ + --form tests=@$RESULTS_RESULTS \ + --form metrics=@$RESULTS_METRICS \ + --form metadata=@$RESULTS_METADATA \ + $SQUAD_URL >> $pushcmdfile + echo "echo $SQUAD_URL pushed" >> $pushcmdfile + + done + echo " $pushcmdfile" + chmod a+x $pushcmdfile + + # push command for all project + pushcmdfile=squad_inputs/${project_run['project']}/push_results_to_squad.sh + if [ ! -f $pushcmdfile ]; then + echo "#!/bin/bash -fx" > $pushcmdfile + chmod a+x $pushcmdfile + fi + + echo "" >> $pushcmdfile + echo "./squad_inputs/${project_run['project']}/${project_run['run']}/push_results_to_squad.sh" >> $pushcmdfile + + echo "" +} + +get_and_push_one_project () +{ + artifact_dir=$1 + artifact_branch=$2 + + local project + + echo "# base_artifact : fetch $artifact_branch" + git -C $artifact_dir fetch -q origin $artifact_branch + git -C $artifact_dir checkout FETCH_HEAD + git -C $artifact_dir branch -f $artifact_branch FETCH_HEAD + + for sha1 in $(git -C $artifact_dir log FETCH_HEAD --pretty=%H | tac); do + declare -A project_run project_results + + get_infos_from_artifacts $artifact_dir $sha1 $artifact_branch + project=${project_run['project']} + + parse_results_csv_files + + dump_json_for_squad + + push_results_to_squad_server + + unset project_run project_results + done + + echo "# TO PUSH ALL RESULTS, PLEASE RUN :" + echo "squad_inputs/$project/push_results_to_squad.sh" + echo "" +} + + +gitserver=https://git-us.linaro.org/toolchain/ci/base-artifacts.git +artifact_dir=base-artifacts + +# restart from scrach. Checkout single branch. +# rm -rf $artifact_dir + +if [ ! -d $artifact_dir ]; then + git clone --single-branch --branch empty $gitserver $artifact_dir +fi + + +get_and_push_one_project $artifact_dir $branch |