path: root/tcwg-wip/push-results-to-squad.sh

#!/bin/bash -f

# Example to push a base-artifacts branch :
#   tcwg-wip/push-results-to-squad.sh --branch linaro-local/ci/tcwg_bmk_gnu_apm/gnu-master-aarch64-spec2k6-Os_LTO --push true


set -e

scripts=$(dirname "$0")/..
# shellcheck source=jenkins-helpers.sh
. "$scripts/jenkins-helpers.sh"

convert_args_to_variables "$@"
obligatory_variables branch

branch=${branch-}
push=${push-false}
dryrun=${dryrun-false}
verbose=${verbose-false}

if [ "$dryrun" == "false" ]; then
  curl="curl"
else
  echo "DRYRUN : Will show but not push result"
  curl="echo curl"
fi


# --------------------------- UTILS --------------------------------
verbose()
{
  if $verbose; then
    echo "$*"
  fi
}

error()
{
  echo ""
  echo "ERROR : $*"
  echo ""
  exit 1
}

first_time=true
clean_squad_project_if_first_time()
{
  if $first_time; then
    rm -rf squad_inputs/${project_run['project']}
  fi
  first_time=false
}

# --------------------------- GET INFOS --------------------------------
get_infos_from_artifacts ()
{
  local artifacts_dir=$1
  local artifacts_sha1=$2
  local artifacts_branch=$3

  echo "# base-artifact : rev=$sha1 ($artifacts_branch)"
  git -C $artifact_dir reset -q --hard
  git -C $artifact_dir checkout -q -f $artifacts_sha1

  local manifest
  [ -f "$artifacts_dir/manifest.sh" ] && manifest=manifest.sh
  [ -f "$artifacts_dir/jenkins/manifest.sh" ] && manifest=jenkins/manifest.sh
  [ x"$manifest" == x"" ] && error "Manifest not found"

  declare -A rr
  # shellcheck disable=SC1090
  source $artifacts_dir/$manifest

  if [ -d $artifacts_dir/dashboard ]; then
    # Use dashboard directory
    :
  else

    # general
    project_run['artifact_topdir']="$artifacts_dir"
    project_run['project']=$(echo $BUILD_URL |cut -d/ -f5)
    project_run['run']=$(echo $BUILD_URL |cut -d/ -f6)
    project_run['url']="$BUILD_URL"

    # tool versions
    project_run['version_binutils']=${rr[binutils_rev]}
    project_run['version_gcc']=${rr[gcc_rev]}
    project_run['version_glibc']=${rr[glibc_rev]}
    project_run['version_linux']=${rr[linux_rev]}
    [ -v "rr[llvm_rev]" ] && project_run['version_llvm']=${rr[llvm_rev]}

    # url
    project_run['artifacts_url_base']="https://git.linaro.org/toolchain/ci/base-artifacts.git/tree"
    project_run['artifacts_url_branch_and_version']="h=$artifacts_branch&id=$artifacts_sha1"
    project_run['artifacts_url_results']="${project_run['artifacts_url_base']}/?${project_run['artifacts_url_branch_and_version']}"

    # relative to artifact topdir
    benchmark_dir=$(cd $artifacts_dir && find . -maxdepth 2 -name "*-benchmark")
    [ x"$benchmark_dir" == x"" ] && error "Benchmark dir not found"

    check_regression_dir=$(cd $artifacts_dir && find . -maxdepth 2 -name "*-check_regression")
    [ x"$check_regression_dir" == x"" ] && error "Regression dir not found"

    [ -f "$artifacts_dir/manifest.sh" ] && project_run['manifest']="manifest.sh"
    [ -f "$artifacts_dir/jenkins/manifest.sh" ] && project_run['manifest']="jenkins/manifest.sh"
    [ x"${project_run['manifest']}" == x"" ] && error "Manifest not found"

    [ -f "$artifacts_dir/$benchmark_dir/benchmark-build.log" ] && project_run['benchmark_log']="$benchmark_dir/benchmark-build.log"
    [ -f "$artifacts_dir/$benchmark_dir/benchmark.log" ] && project_run['benchmark_log']="$benchmark_dir/benchmark.log"
    [ x"${project_run['benchmark_log']}" == x"" ] && error "Benchmark log not found"

    [ -f "$artifacts_dir/$check_regression_dir/csv-results-1/results.csv" ] && project_run['results_csv']="$check_regression_dir/csv-results-1/results.csv"
    [ -f "$artifacts_dir/$check_regression_dir/results-1.csv" ] && project_run['results_csv']="$check_regression_dir/results-1.csv"
    [ x"${project_run['results_csv']}" == x"" ] && error "results.csv log not found"

    project_run['datetime']=$(git -C $artifact_dir show --no-patch --format=%cI)
  fi
}


# --------------------------- GET INFOS --------------------------------
# uses RESULTS_FILE argument
# outputs project_results
parse_results_csv_files()
{
  echo " * parsing base-artifacts inputs"
  echo "   ${project_run['artifact_topdir']}/${project_run['results_csv']}"

  IFS=','
  # benchmark, symbol, sample, size, num_vect_loops, symbol_md5sum
  while read bench symb perf size vect md5sum
  do
    [[ $symb == *.default* ]] || continue

    if [[ -v project_results["func:$bench"] ]] && [ ${project_results["func:$bench"]} != "pass" ]; then
       : # do nothing if fail is already
    else
       case $perf in
          999999999)
             project_results["func:$bench"]="build_fail"
             # project_results["perf:$bench"]=""
             # project_results["size:$bench"]=""
             # project_results["vect:$bench"]=""
	     ;;
	  888888888)
             project_results["func:$bench"]="run_fail"
             [ $vect != "-1" ] && project_results["vect:$bench"]="$vect"
             [ $size != "-1" ] && project_results["size:$bench"]="$size"
             # project_results["perf:$bench"]=""
	     ;;
	  *)
             project_results["func:$bench"]="pass"
             [ $vect != "-1" ] && project_results["vect:$bench"]="$vect"
             [ $size != "-1" ] && project_results["size:$bench"]="$size"
             [ $perf != "-1" ] && project_results["perf:$bench"]="$perf"
	     ;;
       esac
    fi

    #echo "  # $bench,$symb,$perf,$size,$vect,$md5sum"
    verbose "    $bench : ${project_results["func:$bench"]}, ${project_results["perf:$bench"]}, " \
          "${project_results["size:$bench"]}, ${project_results["vect:$bench"]}, $md5sum"
  done < "${project_run['artifact_topdir']}/${project_run['results_csv']}"
  unset IFS

  # hh:mm:ss Finished: SUCCESS
  project_run['run_status']=$(grep '^Finished: ' ${project_run['artifact_topdir']}/${project_run['benchmark_log']} | tail -1 | cut -d: -f2)
}



# --------------------------- WRITING OUTPUTS --------------------------------
# uses project_results[] project_run[]
# dump files in ${project_run['project_run_dir']}
dump_json_for_squad()
{
  local nbtest=0 nbpass=0 nbfail=0 nbskip=0

  outdir=squad_inputs/${project_run['project']}/${project_run['run']}
  rm -rf $outdir
  mkdir -p $outdir

  clean_squad_project_if_first_time

  echo " * writing json for squad"

  for key in "${!project_results[@]}"; do

    kind=$(echo $key | cut -d: -f1)
    bench=$(echo $key | cut -d: -f2)
    [[ "$kind" == "func" ]] || continue

    # echo " - writing $outdir/$bench	# ${project_results["func:$bench"]}"
    mkdir -p $outdir/$bench

    case ${project_results["func:$bench"]} in
      "build_fail")
	build_func="fail"; ((nbfail+=1));
	run_func="skip";   ((nbskip+=1));
	((nbtest+=2));
	;;
      "run_fail")
	build_func="pass"; ((nbpass+=1));
	run_func="fail";   ((nbfail+=1));
	((nbtest+=2));
	;;
      "pass")
	build_func="pass"; ((nbpass+=1));
	run_func="pass";   ((nbpass+=1));
	((nbtest+=2));
	;;
    esac

    resfile=$outdir/$bench/results-functional.json
    echo "{"										>> $resfile
    echo "  \"build\" : \"$build_func\"",						>> $resfile
    echo "  \"run\" : \"$run_func\""							>> $resfile
    echo "}"										>> $resfile

    resfile=$outdir/$bench/results-metrics.json
    echo "{"														>> $resfile
    [[ -v project_results["perf:$bench"] ]] && echo "  \"perf\" : \"${project_results["perf:$bench"]}\","			>> $resfile
    [[ -v project_results["size:$bench"] ]] && echo "  \"size\" : \"${project_results["size:$bench"]}\","			>> $resfile
    [[ -v project_results["vect:$bench"] ]] && echo "  \"vect\" : \"${project_results["vect:$bench"]}\""			>> $resfile
    echo "}"														>> $resfile

    resfile=$outdir/$bench/results-metadata.json
    echo "{"														>> $resfile
    # general infos
    echo "  \"datetime\": \"${project_run['datetime']}\","								>> $resfile
    echo "  \"build_url\": \"${project_run['url']}\","									>> $resfile
    echo "  \"build_log\": \"${project_run['url']}console\","								>> $resfile
    echo "  \"job_status\": \"${project_run['run_status']}\","								>> $resfile

    # tool versions
    echo "  \"version_binutils\": \"${project_run['version_binutils']}\","  						>> $resfile
    echo "  \"version_gcc\": \"${project_run['version_gcc']}\","							>> $resfile
    echo "  \"version_glibc\": \"${project_run['version_glibc']}\","							>> $resfile
    echo "  \"version_linux\": \"${project_run['version_linux']}\","							>> $resfile
    [[ -v project_run['version_llvm'] ]] && echo "  \"version_llvm\": \"${project_run['version_llvm']}\","  		>> $resfile

    # links to base-artifacts
    echo "  \"artifact_results\": \"${project_run['artifacts_url_base']}/" \
     "?${project_run['artifacts_url_branch_and_version']}\","								>> $resfile
    echo "  \"build_manifest\": \"${project_run['artifacts_url_base']}/" \
      "${project_run['manifest']}?${project_run['artifacts_url_branch_and_version']}\","				>> $resfile
    echo "  \"benchmark_log\": \"${project_run['artifacts_url_base']}/"\
       "${project_run['benchmark_log']}?${project_run['artifacts_url_branch_and_version']}\""				>> $resfile

    echo "}"														>> $resfile

  done
  project_run['squad_input_dir']=$outdir

  echo "   $outdir ($nbtest test, $nbpass pass, $nbfail fail, $nbskip skip)"
}


push_results_to_squad_server()
{
  local outdir=squad_inputs/${project_run['project']}/${project_run['run']}
  local pushcmdfile=$outdir/push_results_to_squad.sh
  rm -f $pushcmdfile

  echo " * generating cmd to push results"


  SQUAD_TOKEN="e1e2856837b6c89c08e1031f3263e290b5aeefb5"

  squad_server=https://qa-reports.linaro.org/
  GRP="~laurent.alfonsi"

  echo "#!/bin/bash -f"				> $pushcmdfile
  echo "cd $(pwd)" 				>> $pushcmdfile
  echo "" 					>> $pushcmdfile

  for key in "${!project_results[@]}"; do

    kind=$(echo $key | cut -d: -f1)
    bench=$(echo $key | cut -d: -f2)
    [[ "$kind" == "func" ]] || continue

    # A VERIFIER
    PRJ="${project_run['project']}"
    BLD="${project_run['run']}"
    ENV="$bench"

    SQUAD_URL=$squad_server/api/submit/$GRP/$PRJ/$BLD/$ENV

    # result files
    RESULTS_RESULTS=${project_run['squad_input_dir']}/$bench/results-functional.json
    RESULTS_METRICS=${project_run['squad_input_dir']}/$bench/results-metrics.json
    RESULTS_METADATA=${project_run['squad_input_dir']}/$bench/results-metadata.json

    echo $curl \
      --header \"Authorization: Token $SQUAD_TOKEN\" \
      --form tests=@$RESULTS_RESULTS \
      --form metrics=@$RESULTS_METRICS \
      --form metadata=@$RESULTS_METADATA \
      $SQUAD_URL 					>> $pushcmdfile
    echo "echo $SQUAD_URL pushed"			>> $pushcmdfile

  done
  echo "   $pushcmdfile"
  chmod a+x $pushcmdfile

  # push command for all project
  pushcmdfile=squad_inputs/${project_run['project']}/push_results_to_squad.sh
  if [ ! -f $pushcmdfile ]; then
    echo "#!/bin/bash -fx"										> $pushcmdfile
    chmod a+x $pushcmdfile
  fi

  echo ""												>> $pushcmdfile
  echo "./squad_inputs/${project_run['project']}/${project_run['run']}/push_results_to_squad.sh"	>> $pushcmdfile

  echo ""
}

get_and_push_one_project ()
{
  artifact_dir=$1
  artifact_branch=$2

  local project
  
  echo "# base_artifact : fetch $artifact_branch"
  git -C $artifact_dir fetch -q origin $artifact_branch
  git -C $artifact_dir checkout FETCH_HEAD
  git -C $artifact_dir branch -f $artifact_branch FETCH_HEAD

  for sha1 in $(git -C $artifact_dir log FETCH_HEAD --pretty=%H | tac); do
    declare -A project_run project_results

    get_infos_from_artifacts $artifact_dir $sha1 $artifact_branch
    project=${project_run['project']}

    parse_results_csv_files

    dump_json_for_squad

    push_results_to_squad_server

    unset project_run project_results
  done

  echo "# TO PUSH ALL RESULTS, PLEASE RUN :"
  echo "squad_inputs/$project/push_results_to_squad.sh"
  echo ""
}


gitserver=https://git-us.linaro.org/toolchain/ci/base-artifacts.git
artifact_dir=base-artifacts

# restart from scrach. Checkout single branch.
# rm -rf $artifact_dir

if [ ! -d $artifact_dir ]; then
 git clone --single-branch --branch empty $gitserver $artifact_dir
fi


get_and_push_one_project $artifact_dir $branch