tcwg-wip: First script to push a results to squad (WIP)

Change-Id: If30c34480211f3b638eed97724c4043fbcd1962d

1 files changed, 341 insertions, 0 deletions

diff --git a/tcwg-wip/push-results-to-squad.sh b/tcwg-wip/push-results-to-squad.sh
new file mode 100755
index 00000000..976c8649
--- /dev/null
+++ b/tcwg-wip/push-results-to-squad.sh
@@ -0,0 +1,341 @@
+#!/bin/bash -f
+
+# Example to push a base-artifacts branch :
+#   tcwg-wip/push-results-to-squad.sh --branch linaro-local/ci/tcwg_bmk_gnu_apm/gnu-master-aarch64-spec2k6-Os_LTO --push true
+
+
+set -e
+
+scripts=$(dirname "$0")/..
+# shellcheck source=jenkins-helpers.sh
+. "$scripts/jenkins-helpers.sh"
+
+convert_args_to_variables "$@"
+obligatory_variables branch
+
+branch=${branch-}
+push=${push-false}
+dryrun=${dryrun-false}
+verbose=${verbose-false}
+
+if [ "$dryrun" == "false" ]; then
+  curl="curl"
+else
+  echo "DRYRUN : Will show but not push result"
+  curl="echo curl"
+fi
+
+
+# --------------------------- UTILS --------------------------------
+verbose()
+{
+  if $verbose; then
+    echo "$*"
+  fi
+}
+
+first_time=true
+clean_squad_project_if_first_time()
+{
+  if $first_time; then
+    rm -rf squad_inputs/${project_run['project']}
+  fi
+  first_time=false
+}
+
+# --------------------------- GET INFOS --------------------------------
+declare -A project_run
+get_infos_from_artifacts ()
+{
+  local artifacts_dir=$1
+  local artifacts_sha1=$2
+  local artifacts_branch=$3
+
+  echo "# base-artifact : rev=$sha1 ($artifacts_branch)"
+  git -C $artifact_dir reset -q --hard
+  git -C $artifact_dir checkout -q -f $artifacts_sha1
+
+  local fn=manifest.sh
+  if [ ! -f $fn ]; then
+    fn=jenkins/manifest.sh
+  fi
+
+  declare -A rr
+  # shellcheck disable=SC1090
+  source $artifacts_dir/$fn
+
+  if [ -d $artifacts_dir/dashboard ]; then
+    # Use dashboard directory
+    :
+  else
+
+    # general
+    project_run['artifact_topdir']="$artifacts_dir"
+    project_run['project']=$(echo $BUILD_URL |cut -d/ -f5)
+    project_run['run']=$(echo $BUILD_URL |cut -d/ -f6)
+    project_run['url']="$BUILD_URL"
+
+    # tool versions
+    set -x
+    project_run['version_binutils']=${rr[binutils_rev]}
+    project_run['version_gcc']=${rr[gcc_rev]}
+    set +x
+    project_run['version_glibc']=${rr[glibc_rev]}
+    project_run['version_linux']=${rr[linux_rev]}
+    [ -v "rr[llvm_rev]" ] && project_run['version_llvm']=${rr[llvm_rev]}
+
+    # url
+    project_run['artifacts_baseurl']="https://git.linaro.org/toolchain/ci/base-artifacts.git/tree"
+    project_run['artifacts_branch_and_version']="h=$artifacts_branch&id=$artifacts_sha1"
+    project_run['artifacts_results']="${project_run['artifacts_baseurl']}/?${project_run['artifacts_branch_and_version']}"
+
+    # relative to artifact topdir
+    benchmark_dir=$(cd $artifacts_dir && find . -maxdepth 2 -name "*-benchmark")
+    check_regression_dir=$(cd $artifacts_dir && find . -maxdepth 2 -name "*-check_regression")
+    project_run['manifest']="manifest.sh"
+    if [ ! -f $artifacts_dir/${project_run['manifest']} ]; then
+      project_run['manifest']="jenkins/manifest.sh"
+    fi
+
+    project_run['benchmark_log']="$benchmark_dir/benchmark-build.log"
+    project_run['results_csv']="$check_regression_dir/csv-results-1/results.csv"
+    project_run['datetime']=$(date --iso-8601=seconds -r $artifacts_dir/${project_run['manifest']})
+  fi
+}
+
+
+# --------------------------- GET INFOS --------------------------------
+# uses RESULTS_FILE argument
+# outputs project_results
+declare -A project_results
+
+parse_results_csv_files()
+{
+  echo " * parsing base-artifacts inputs"
+  echo "   ${project_run['artifact_topdir']}/${project_run['results_csv']}"
+
+  IFS=','
+  # benchmark, symbol, sample, size, num_vect_loops, symbol_md5sum
+  while read bench symb perf size vect md5sum
+  do
+    [[ $symb == *.default* ]] || continue
+
+    if [ -v project_results["func:$bench"] ] && [ ${project_results["func:$bench"]} != "pass" ]; then
+       : # do nothing if fail is already
+    else
+       case $perf in
+          999999999)
+             project_results["func:$bench"]="build_fail"
+             # project_results["perf:$bench"]=""
+             # project_results["size:$bench"]=""
+             # project_results["vect:$bench"]=""
+	     ;;
+	  888888888)
+             project_results["func:$bench"]="run_fail"
+             [ $size != "-1" ] && project_results["size:$bench"]="$size"
+             [ $vect != "-1" ] && project_results["vect:$bench"]="$vect"
+             # project_results["perf:$bench"]=""
+	     ;;
+	  *)
+             project_results["func:$bench"]="pass"
+             [ $size != "-1" ] && project_results["size:$bench"]="$size"
+             [ $vect != "-1" ] && project_results["vect:$bench"]="$vect"
+             [ $perf != "-1" ] && project_results["perf:$bench"]="$perf"
+	     ;;
+       esac
+    fi
+
+    #echo "  # $bench,$symb,$perf,$size,$vect,$md5sum"
+    verbose "  $bench : ${project_results["func:$bench"]}, ${project_results["perf:$bench"]}, " \
+          "${project_results["size:$bench"]}, ${project_results["vect:$bench"]}, $md5sum"
+  done < "${project_run['artifact_topdir']}/${project_run['results_csv']}"
+  unset IFS
+
+  # hh:mm:ss Finished: SUCCESS
+  project_run['run_status']=$(grep '^Finished: ' ${project_run['artifact_topdir']}/${project_run['benchmark_log']} | tail -1 | cut -d: -f2)
+}
+
+
+
+# --------------------------- WRITING OUTPUTS --------------------------------
+# uses project_results[] project_run[]
+# dump files in ${project_run['project_run_dir']}
+dump_json_for_squad()
+{
+  local nbtest=0 nbpass=0 nbfail=0 nbskip=0
+
+  outdir=squad_inputs/${project_run['project']}/${project_run['run']}
+  rm -rf $outdir
+  mkdir -p $outdir
+
+  clean_squad_project_if_first_time
+
+  echo " * writing json for squad"
+
+  for key in "${!project_results[@]}"; do
+
+    kind=$(echo $key | cut -d: -f1)
+    bench=$(echo $key | cut -d: -f2)
+    [[ "$kind" == "func" ]] || continue
+
+    # echo " - writing $outdir/$bench	# ${project_results["func:$bench"]}"
+    mkdir -p $outdir/$bench
+
+    case ${project_results["func:$bench"]} in
+      "build_fail")
+	build_func="fail"; ((nbfail+=1));
+	run_func="skip";   ((nbskip+=1));
+	((nbtest+=2));
+	;;
+      "run_fail")
+	build_func="pass"; ((nbpass+=1));
+	run_func="fail";   ((nbfail+=1));
+	((nbtest+=2));
+	;;
+      "pass")
+	build_func="pass"; ((nbpass+=1));
+	run_func="pass";   ((nbpass+=1));
+	((nbtest+=2));
+	;;
+    esac
+
+    resfile=$outdir/$bench/results-functional.json
+    echo "{"										>> $resfile
+    echo "  \"build\" : \"$build_func\"",						>> $resfile
+    echo "  \"run\" : \"$run_func\""							>> $resfile
+    echo "}"										>> $resfile
+
+    resfile=$outdir/$bench/results-metrics.json
+    echo "{"														>> $resfile
+    [ -v project_results["perf:$bench"] ] && echo "  \"perf\" : \"${project_results["perf:$bench"]}\","			>> $resfile
+    [ -v project_results["vect:$bench"] ] && echo "  \"vect\" : \"${project_results["vect:$bench"]}\","			>> $resfile
+    [ -v project_results["size:$bench"] ] && echo "  \"size\" : \"${project_results["size:$bench"]}\""			>> $resfile
+    echo "}"														>> $resfile
+
+    resfile=$outdir/$bench/results-metadata.json
+    echo "{"														>> $resfile
+    # general infos
+    echo "  \"datetime\": \"${project_run['datetime']}\","								>> $resfile
+    echo "  \"build_url\": \"${project_run['url']}\","									>> $resfile
+    echo "  \"build_log\": \"${project_run['url']}console\","								>> $resfile
+    echo "  \"job_status\": \"${project_run['run_status']}\","								>> $resfile
+
+    # tool versions
+    echo "  \"version_binutils\": \"${project_run['version_binutils']}\","  						>> $resfile
+    echo "  \"version_gcc\": \"${project_run['version_gcc']}\","							>> $resfile
+    echo "  \"version_glibc\": \"${project_run['version_glibc']}\","							>> $resfile
+    echo "  \"version_linux\": \"${project_run['version_linux']}\","							>> $resfile
+    [ -v project_run['version_llvm'] ] && echo "  \"version_llvm\": \"${project_run['version_llvm']}\","  		>> $resfile
+
+    # links to base-artifacts
+    echo "  \"artifact_results\": " \
+     "\"${project_run['artifacts_baseurl']}/?${project_run['artifacts_branch_and_version']}\","				>> $resfile
+    echo "  \"build_manifest\": "\
+     "\"${project_run['artifacts_baseurl']}/${project_run['manifest']}?${project_run['artifacts_branch_and_version']}\","	>> $resfile
+    echo "  \"benchmark_log\": "\
+     "\"${project_run['artifacts_baseurl']}/${project_run['benchmark_log']}?${project_run['artifacts_branch_and_version']}\""	>> $resfile
+
+    echo "}"														>> $resfile
+
+  done
+  project_run['squad_input_dir']=$outdir
+
+  echo "   $outdir ($nbtest test, $nbpass pass, $nbfail fail, $nbskip skip)"
+}
+
+
+push_results_to_squad_server()
+{
+  local outdir=squad_inputs/${project_run['project']}/${project_run['run']}
+  local pushcmdfile=$outdir/push_results_to_squad.sh
+  rm -f $pushcmdfile
+
+  echo " * generating cmd to push results"
+
+
+  SQUAD_TOKEN="e1e2856837b6c89c08e1031f3263e290b5aeefb5"
+
+  squad_server=https://qa-reports.linaro.org/
+  GRP="~laurent.alfonsi"
+
+  echo "#!/bin/bash -f"				> $pushcmdfile
+  echo "cd $(pwd)" 				>> $pushcmdfile
+  echo "" 					>> $pushcmdfile
+
+  for key in "${!project_results[@]}"; do
+
+    kind=$(echo $key | cut -d: -f1)
+    bench=$(echo $key | cut -d: -f2)
+    [[ "$kind" == "func" ]] || continue
+
+    # A VERIFIER
+    PRJ="${project_run['project']}"
+    BLD="${project_run['run']}"
+    ENV="$bench"
+
+    SQUAD_URL=$squad_server/api/submit/$GRP/$PRJ/$BLD/$ENV
+
+    # result files
+    RESULTS_RESULTS=${project_run['squad_input_dir']}/$bench/results-functional.json
+    RESULTS_METRICS=${project_run['squad_input_dir']}/$bench/results-metrics.json
+    RESULTS_METADATA=${project_run['squad_input_dir']}/$bench/results-metadata.json
+
+    echo $curl \
+      --header \"Authorization: Token $SQUAD_TOKEN\" \
+      --form tests=@$RESULTS_RESULTS \
+      --form metrics=@$RESULTS_METRICS \
+      --form metadata=@$RESULTS_METADATA \
+      $SQUAD_URL 					>> $pushcmdfile
+    echo "echo $SQUAD_URL pushed"			>> $pushcmdfile
+
+  done
+  echo "   $pushcmdfile"
+  chmod a+x $pushcmdfile
+
+  # push command for all project
+  pushcmdfile=squad_inputs/${project_run['project']}/push_results_to_squad.sh
+  if [ ! -f $pushcmdfile ]; then
+    echo "#!/bin/bash -fx"										> $pushcmdfile
+    chmod a+x $pushcmdfile
+  fi
+
+  echo ""												>> $pushcmdfile
+  echo "./squad_inputs/${project_run['project']}/${project_run['run']}/push_results_to_squad.sh"	>> $pushcmdfile
+
+  echo ""
+}
+
+get_and_push_one_project ()
+{
+  artifact_dir=$1
+  artifact_branch=$2
+
+  echo "# base_artifact : fetch $artifact_branch"
+  git -C $artifact_dir fetch -q origin $artifact_branch
+  git -C $artifact_dir checkout FETCH_HEAD
+
+  for sha1 in $(git -C $artifact_dir log FETCH_HEAD --pretty=%H | tac); do
+    get_infos_from_artifacts $artifact_dir $sha1 $artifact_branch
+    parse_results_csv_files
+    dump_json_for_squad
+    push_results_to_squad_server
+  done
+
+  echo "# TO PUSH ALL RESULTS, PLEASE RUN :"
+  echo "squad_inputs/${project_run['project']}/push_results_to_squad.sh"
+  echo ""
+}
+
+
+gitserver=https://git-us.linaro.org/toolchain/ci/base-artifacts.git
+artifact_dir=base-artifacts
+
+# restart from scrach. Checkout single branch.
+# rm -rf $artifact_dir
+
+if [ ! -d $artifact_dir ]; then
+ git clone --single-branch --branch empty $gitserver $artifact_dir
+fi
+
+
+get_and_push_one_project $artifact_dir $branch