round-robin.sh, dashboard.sh: factorize all dashboard routines

... and simplify.

During round-robin.sh CHECK_REGRESSION stage :
- Creates file artifacts/dashboard/dashboard-generate.sh
  Simple call to jenkins-scripts/dashboard-generate-squad.sh with the
  proper parameters.
- And simply execute it.
  This creates artifacts/dashboard/squad/ that contains all json files
  and a scripts push to squad (dashboard-push-squad.sh)

During round-robin.sh UPDATE_BASELINE stage :
- Creates file artifacts/jenkins/dashboard-push.sh
  Simple call to dashboard-push-squad.sh

During last steps of the jenkins projects :
- invoke artifacts/jenkins/dashboard-push.sh

After the job end :
- If we want to generate a dashboard info from an artifacts dir, we would
  have to do :
  $ artifacts/dashboard/dashboard-generate.sh

- If we want to push a dashboard results from an artifacts dir, we would
  have to do :
  $ artifacts/jenkins/dashboard-push.sh

- If we want to iterate over a full branch of base-artifacts, and for each
  one generate and push the dashboard, you would ave to do :
  $ jenkins-scripts/dashboard-push-one-branch.sh  --top_artifacts <base-artifacts>
    --baseline_branch <git branch>

Change-Id: I84b6da3460d1b9c6676a4ad93b2894e5bac968d5

1 files changed, 62 insertions, 345 deletions

diff --git a/dashboard-push-one-branch.sh b/dashboard-push-one-branch.sh
index 007eff1b..46905733 100755
--- a/dashboard-push-one-branch.sh
+++ b/dashboard-push-one-branch.sh
@@ -1,382 +1,99 @@
-#!/bin/bash -f
+#!/bin/bash
 
 # This scripts iterates over a base-artifact BRANCH and for each base-artifact :
 # - recreates the dashboard directory if necessary
 # - push the results to squad
 #
 # Example to push a base-artifacts branch :
-#   tcwg-wip/tcwg-push-one-branch-to-squad.sh --branch linaro-local/ci/tcwg_bmk_gnu_apm/gnu-master-aarch64-spec2k6-Os_LTO
+#   dashboard-push-one-branch.sh --top_artifacts base-artifacts --baseline_branch linaro-local/ci/tcwg_bmk_gnu_apm/gnu-master-aarch64-spec2k6-Os_LTO
 #
 # This required the squad project to exist
 
 set -e
 
 # shellcheck source=jenkins-helpers.sh
-. "$(dirname "$0")/jenkins-helpers.sh"
+. "$(dirname $0)/jenkins-helpers.sh"
 # shellcheck source=round-robin.sh
-. "$(dirname "$0")/round-robin.sh"
+. "$(dirname $0)/round-robin.sh"
 
 convert_args_to_variables "$@"
-obligatory_variables branch
 
-declare branch
+obligatory_variables top_artifacts baseline_branch
 
-# --------------------------- UTILS --------------------------------
-error()
-{
-    echo ""
-    echo "ERROR : $*"
-    echo ""
-    exit 1
-}
+declare top_artifacts baseline_branch
 
-# --------------------HOOK CREATE DASHBOARD FILES -------------------
-# Writing dashboard files : json & cmd
-dashboard_create_dashboard_files()
+# rebuild component list from the artifacts state.
+get_components()
 {
-    (
-    local dashboard_dir=$1
-    local dashboard_env_file=$dashboard_dir/dashboard-env.sh
-
-    declare -A project_results dbd
-
-    # shellcheck disable=SC1090
-    source $dashboard_env_file
-
-    echo " * dashboard_push_create_dashboard_files"
-
-    # -----------------------------
-    # Parse results.csv
-    local header_verified=false
-    local results_csv=$run_step_artifacts/${dbd[metrics_file]}
-
-    IFS=','
-    # benchmark, symbol, sample, size, num_vect_loops, symbol_md5sum
-    while read bench symb perf size vect md5sum
-    do
-      # echo "  # $bench,$symb,$perf,$size,$vect,$md5sum"
-      # first line : check headers "benchmark, symbol, sample, size, num_vect_loops, symbol_md5sum"
-      if ! $header_verified; then
-        if [ "$bench" != benchmark ] || [ "$symb" != symbol ] ||
-           [ "$perf" != sample ] || [ "$size" != size ] ||
-           [ "$vect" != num_vect_loops ]; then
-           echo ""
-	   echo "ERROR: results.csv header to make sure format is conform to expected format"
-	   echo " $bench,$symb,$perf,$size,$vect,$md5sum"
-	   echo " benchmark,symbol,sample,size,num_vect_loops,symbol_md5sum"
-	   echo ""
-	   return
-        fi
-        header_verified=true
-      fi
-      [[ $symb == *_base.default* ]] || continue
-
-      if [ -v project_results["func:$bench"] ] && [ ${project_results["func:$bench"]} != "pass" ]; then
-         : # do nothing if fail is already
-      else
-         case $perf in
-            999999999)
-               project_results["func:$bench"]="build_fail"
-               # project_results["perf:$bench"]=""
-               # project_results["size:$bench"]=""
-               # project_results["vect:$bench"]=""
-	       ;;
-	    888888888)
-               project_results["func:$bench"]="run_fail"
-               [ $vect != "-1" ] && project_results["vect:$bench"]="$vect"
-               [ $size != "-1" ] && project_results["size:$bench"]="$size"
-               # project_results["perf:$bench"]=""
-	       ;;
-	    *)
-               project_results["func:$bench"]="pass"
-               [ $vect != "-1" ] && project_results["vect:$bench"]="$vect"
-               [ $size != "-1" ] && project_results["size:$bench"]="$size"
-               [ $perf != "-1" ] && project_results["perf:$bench"]="$perf"
-	       ;;
-         esac
-      fi
-
-      # echo "    $bench : ${project_results["func:$bench"]}, ${project_results["perf:$bench"]}, " \
-      #                    "${project_results["size:$bench"]}, ${project_results["vect:$bench"]}, $md5sum"
-
-    done < "$results_csv"
-    unset IFS
-
-    # -----------------------------
-    # Write json files
-    # for each bench :
-    #   results-functional.json, results-metrics.json, results-metadata.json
-    local nbtest=0 nbpass=0 nbfail=0 nbskip=0
-    local build_func run_func
-
-    squaddir=$dashboard_dir/squad
-    rm -rf $squaddir
-    mkdir -p $squaddir
-
-    echo "  - writing json for squad"
-
-    for key in "${!project_results[@]}"; do
-
-      kind=$(echo $key | cut -d: -f1)
-      bench=$(echo $key | cut -d: -f2)
-      [[ "$kind" == "func" ]] || continue
-
-      # echo " - writing $squaddir/$bench	# ${project_results["func:$bench"]}"
-      mkdir -p $squaddir/$bench
-
-      case ${project_results["func:$bench"]} in
-	"build_fail")
-	  build_func="fail"; ((nbfail+=1));
-	  run_func="skip";   ((nbskip+=1));
-	  ((nbtest+=2));
-	  ;;
-	"run_fail")
-	  build_func="pass"; ((nbpass+=1));
-	  run_func="fail";   ((nbfail+=1));
-	  ((nbtest+=2));
-	  ;;
-	"pass")
-	  build_func="pass"; ((nbpass+=1));
-	  run_func="pass";   ((nbpass+=1));
-	  ((nbtest+=2));
-	  ;;
-      esac
-
-      # Generate 3 results files : functional, metrics, metadata
-      #
-      resfile=$squaddir/$bench/results-functional.json
-      cat > $resfile << EOF
-{
-  "build" : "$build_func",
-  "run" :   "$run_func"
-}
-EOF
-
-      resfile=$squaddir/$bench/results-metrics.json
-      echo "{" > "$resfile"
-      [ -v project_results["perf:$bench"] ] && echo "  \"perf\" : \"${project_results["perf:$bench"]}\"," >>"$resfile"
-      [ -v project_results["size:$bench"] ] && echo "  \"size\" : \"${project_results["size:$bench"]}\"," >>"$resfile"
-      [ -v project_results["vect:$bench"] ] && echo "  \"vect\" : \"${project_results["vect:$bench"]}\"" >>"$resfile"
-      echo "}" >> "$resfile"
-
-      resfile=$squaddir/$bench/results-metadata.json
-      cat > $resfile << EOF
-{
-  "datetime":         "${dbd['datetime']}",
-  "build_url":        "${dbd['master_job_url']}",
-  "build_log":        "${dbd['master_job_url']}console",
-  "job_status":       "${dbd['status']}",
-EOF
-
-      [ -v dbd['binutils_rev'] ] && echo  "  \"version_binutils\":\"${dbd['binutils_rev']}\"," >> "$resfile"
-      [ -v dbd['gcc_rev'] ]   && echo  "  \"version_gcc\":     \"${dbd['gcc_rev']}\","   >> "$resfile"
-      [ -v dbd['glibc_rev'] ] && echo  "  \"version_glibc\":   \"${dbd['glibc_rev']}\"," >> "$resfile"
-      [ -v dbd['llvm_rev'] ]  && echo  "  \"version_llvm\":    \"${dbd['llvm_rev']}\","  >> "$resfile"
-      [ -v dbd['linux_rev'] ] && echo  "  \"version_linux\":   \"${dbd['linux_rev']}\"," >> "$resfile"
-      [ -v dbd['qemu_rev'] ]  && echo  "  \"version_qemu\":    \"${dbd['qemu_rev']}\","  >> "$resfile"
-
-      local base_artifacts_url=https://git-us.linaro.org/toolchain/ci/base-artifacts.git
-      cat >> $squaddir/$bench/results-metadata.json << EOF
-  "artifact_results": "$base_artifacts_url/?${dbd['base-artifacts_branch']}"
-}
-EOF
-
-    done
-    echo "    [$nbtest test : $nbpass pass, $nbfail fail, $nbskip skip]"
-
-    # Generate one annotation file
-    #
-    resfile=$squaddir/summary-annotation.txt
-    touch $resfile
-    if [ -f $dashboard_dir/../mail/mail-subject.txt ]; then
-      cat $dashboard_dir/../mail/mail-subject.txt >> $resfile
-    fi
-    if [ -f $run_step_artifacts/exe.regressions ]; then
-      echo ""							      >> $resfile
-      echo "Regressions :"					      >> $resfile
-      cut -d, -f5 $run_step_artifacts/exe.regressions		      >> $resfile
-    fi
-    if [ -f $run_step_artifacts/exe.improvements ]; then
-      echo ""							      >> $resfile
-      echo "Improvements :"					      >> $resfile
-      cut -d, -f5 $run_step_artifacts/exe.improvements		      >> $resfile
-    fi
-
-    # Squad annotations are limited to 1024 chars.
-    # Thus, limiting to : 1018 chars + "\n...\n" (<1024)
-    if [ "$(cat $resfile | wc -c)" -gt 1024 ]; then
-      tmpfile=tempfile
-      cat > $tmpfile << EOF
-$(head -c 1018 $resfile)
-...
-EOF
-      mv $tmpfile $resfile
+  (
+  local components=""
+
+  manifest=""
+  [ -f "$top_artifacts/manifest.sh" ] && manifest=manifest.sh
+  [ -f "$top_artifacts/jenkins/manifest.sh" ] && manifest=jenkins/manifest.sh
+  [ x"$manifest" == x"" ] && error "Manifest not found"
+  # shellcheck disable=SC1090
+  source $top_artifacts/$manifest
+
+  for c in binutils gcc glibc llvm linux; do
+    if [ -f $top_artifacts/git/${c}_rev ] || [ -v rr[${c}_rev] ] ; then
+      components="$components $c"
     fi
-
-    # Generate one push command file for all bench
-    #
-    local pushcmdfile=$squaddir/push_results_to_squad.sh
-    rm -f $pushcmdfile
-
-    echo "  - generating cmd to push results"
-
-    local squad_server prj grp bld env squad_url
-    local results_results results_metrics results_metadata
-
-    squad_server=https://qa-reports.linaro.org/
-    # SQUAD_TOKEN will be defined in user environment
-    # SQUAD_GRP is exported in the job definition
-    grp="$SQUAD_GRP"
-    prj="$(echo ${dbd[master_job_url]}|cut -d/ -f5)"
-    bld="$(echo ${dbd[master_job_url]}|cut -d/ -f6)"
-
-    cat > $pushcmdfile << EOF
-#!/bin/bash -f
-cd \$(dirname \$0)
-
-squad_server=$squad_server
-
-EOF
-
-    for key in "${!project_results[@]}"; do
-
-      kind=$(echo $key | cut -d: -f1)
-      bench=$(echo $key | cut -d: -f2)
-      [[ "$kind" == "func" ]] || continue
-
-      env="$bench"
-
-      squad_url=$squad_server/api/submit/$grp/$prj/$bld/$env
-
-      # result files
-      results_results="$bench/results-functional.json"
-      results_metrics="$bench/results-metrics.json"
-      results_metadata="$bench/results-metadata.json"
-
-      cat >> $pushcmdfile << EOF
-echo "pushing $squad_url"
-curl  --header "Authorization: Token \$SQUAD_TOKEN" \\
-      --form tests=@$results_results \\
-      --form metrics=@$results_metrics \\
-      --form metadata=@$results_metadata \\
-      $squad_url
-
-EOF
-    done
-
-    cat >> $pushcmdfile << EOF
-
-# Add annotation for this build
-api_of_this_build=\$(curl -s \$squad_server/$grp/$prj/build/$bld/ | \\
-   grep 'api view of this build' | sed -e 's|.*<a href="\(.*\)">.*|\1|')
-
-curl --header "Authorization: Token \$SQUAD_TOKEN" --data "description=
-\$(cat summary-annotation.txt)
-&build=\$api_of_this_build" \$squad_server/api/annotations/
-
-EOF
-
-    chmod a+x $pushcmdfile
-    )
+  done
+  echo $components
+  )
 }
 
-dashboard_create_push_cmd ()
-{
-    # Generate artifacts/jenkins/dashboard-push.sh, which the CI job
-    # will use to update dashboard in squad server.
-    cat > ${rr[top_artifacts]}/jenkins/dashboard-push.sh << EOF
-#!/bin/bash -f
-cd \$(dirname \$0)
-../dashboard/squad/push_results_to_squad.sh &
-wait \$! || true
-EOF
-   chmod a+x ${rr[top_artifacts]}/jenkins/dashboard-push.sh
-}
-
-# --------------------------- SETUP CONTEXT --------------------------------
-# Function that setup a minimal acceptable context in order to invoke
-# round-robin.sh create_dashboard_dir() routine
-dashboard_setup_context ()
-{
-    local top_artifact=$1
-
-    # look for results.csv, and copy it at top level.
-    check_regression_dir=$(cd $top_artifact && find . -maxdepth 2 -name "*-check_regression")
-    [ x"$check_regression_dir" == x"" ] && error "Regression dir not found"
-    run_step_artifacts=$top_artifact/$check_regression_dir
-
-    # copy result file
-    [ -f $top_artifact/$check_regression_dir/csv-results-1/results.csv ] &&
-       results_csv=$top_artifact/$check_regression_dir/csv-results-1/results.csv
-    [ -f $top_artifact/$check_regression_dir/results-1.csv ] &&
-       results_csv=$top_artifact/$check_regression_dir/results-1.csv
-    [ x"$results_csv" == x"" ] && error "no results_csv file"
-    cp $results_csv $run_step_artifacts/results.csv
-
-    # look for manifest file and source it
-    local manifest
-    [ -f "$top_artifact/manifest.sh" ] && manifest=manifest.sh
-    [ -f "$top_artifact/jenkins/manifest.sh" ] && manifest=jenkins/manifest.sh
-    [ x"$manifest" == x"" ] && error "Manifest not found"
-    # shellcheck disable=SC1090
-    source $top_artifact/$manifest
-
-    # default values
-    if [[ ! -v rr[components] ]]; then
-      rr[components]=""
-      for c in binutils gcc glibc llvm qemu; do
-        [ -v rr[${c}_rev] ] && rr[components]="${rr[components]} $c"
-      done
-    fi
-    rr[binutils_url]=${rr[binutils_url]-"git://sourceware.org/git/binutils-gdb.git"}
-    rr[gcc_url]=${rr[gcc_url]-"https://github.com/gcc-mirror/gcc.git"}
-    rr[glibc_url]=${rr[glibc_url]-"git://sourceware.org/git/glibc.git"}
-    rr[baseline_branch]=$branch
-
-    # override by top_artifact in this context
-    rr[top_artifacts]=$top_artifact
-    rr[create_dashboard_files]=dashboard_create_dashboard_files
-}
+# --------------------------- MAIN PROCEDURE --------------------------------
+gitserver=https://git-us.linaro.org/toolchain/ci/base-artifacts.git
+top_artifacts=base-artifacts
 
+if [ ! -d $top_artifacts ]; then
+ git clone --single-branch --branch empty $gitserver $top_artifacts
+fi
 
-# --------------------------- MAIN PROCEDURE --------------------------------
+echo "# base_artifact : fetch $baseline_branch"
+git -C $top_artifacts reset -q --hard
+git -C $top_artifacts fetch -q origin $baseline_branch
+git -C $top_artifacts checkout -q FETCH_HEAD
+git -C $top_artifacts branch -f $baseline_branch FETCH_HEAD
 
-declare -A rr
+for sha1 in $(git -C $top_artifacts log FETCH_HEAD --pretty=%H | tac); do
+  (
+  echo "# base-artifact : rev=$sha1 ($baseline_branch)"
+  git -C $top_artifacts reset -q --hard
+  git -C $top_artifacts checkout -q -f $sha1
 
-gitserver=https://git-us.linaro.org/toolchain/ci/base-artifacts.git
-artifacts_dir=base-artifacts
+  if [ -f $top_artifacts/dashboard/dashboard-generate.sh ] &&
+     [ -f $top_artifacts/dashboard/dashboard-push.sh ]; then
 
-if [ ! -d $artifacts_dir ]; then
- git clone --single-branch --branch empty $gitserver $artifacts_dir
-fi
+    # If dashboard-generate.sh already exists : just run it.
+    $top_artifacts/dashboard/dashboard-generate.sh
 
-echo "# base_artifact : fetch $branch"
-git -C $artifacts_dir reset -q --hard
-git -C $artifacts_dir fetch -q origin $branch
-git -C $artifacts_dir checkout -q FETCH_HEAD
-git -C $artifacts_dir branch -f $branch FETCH_HEAD
+    $top_artifacts/jenkins/dashboard-push.sh
 
-for sha1 in $(git -C $artifacts_dir log FETCH_HEAD --pretty=%H | tac); do
+  else
 
-  echo "# base-artifact : rev=$sha1 ($branch)"
-  git -C $artifacts_dir reset -q --hard
-  git -C $artifacts_dir checkout -q -f $sha1
+    # Otherwise, create dashboard-generate.sh, run it.
+    # and create dashboard-push.sh
+    declare -A rr
 
-  rm -rf $artifacts_dir/dashboard
-  if [ ! -d $artifacts_dir/dashboard ] || [ ! -d $artifacts_dir/dashboard/squad ]; then
-    (
-    # Context normally setup by round-robin.sh context
-    dashboard_setup_context $artifacts_dir
+    rr[top_artifacts]=$top_artifacts
+    rr[baseline_branch]=$baseline_branch
+    rr[components]=$(get_components)
+    rr[run_date]=$(git -C $top_artifacts show --no-patch --pretty="%cI" HEAD)
 
-    # Call normally invoked from CHECK_REGRESSION step (round-robin.sh)
+    # create dashboard-generate.sh, and run it
     create_dashboard_dir
 
-    # Call normally invoked from UPDATE_BASELINE step
-    dashboard_create_push_cmd
-    )
-  fi
+    # create dashboard-push.sh
+    create_dashboard_push_cmd
 
-  # push results on squad by simply executing dashboard-push.sh
-  $artifacts_dir/jenkins/dashboard-push.sh
+    # .. and run it
+    $top_artifacts/jenkins/dashboard-push.sh
 
+    # clean artifacts dir
+    git -C $top_artifacts clean -d -f
+  fi
+  )
 done