path: root/tcwg-report-stale-rr-jobs.sh

#!/bin/bash

set -euf -o pipefail

scripts=$(dirname "$0")
# shellcheck source=jenkins-helpers.sh
. "$scripts/jenkins-helpers.sh"

convert_args_to_variables "$@"

days="${days-10}"
human="${human-true}"
refs_prefix="${refs_prefix-refs/heads/linaro-local/ci/}"
refs_url_prefix="${refs_url_prefix-https://git.linaro.org/toolchain/ci}"
repos=("${repos[@]-default}")
verbose="${verbose-false}"
classify="${classify-false}"
only="${only-false}"
keep_tmp="${keep_tmp-false}"
tmpdir="${tmpdir-""}"

if [ x"${repos[*]}" = x"default" ]; then
    repos=(base-artifacts binutils-gdb gcc glibc linux llvm-project newlib qemu)
fi

process_git_url ()
{
    (
    set -euf -o pipefail
    local refs_url="$1"
    local days_for_master days_for_release days_limit

    # Initialize base-artifacts repo (by cloning its "empty" branch).
    refs_repo=$(basename "$refs_url" .git)
    clone_or_update_repo_no_checkout "$refs_repo" "$refs_url" auto empty origin \
				     >/dev/null 2>&1
    git -C "$refs_repo" reset --hard >/dev/null 2>&1

    days_for_master="$days"
    days_for_release="$((days+days))"

    # Walk through all commits of all tcwg_bmk* branches and mark results
    # referenced in those results with "used_by" file.
    while IFS= read -r ref; do
	dst_ref="refs/remotes/origin/${ref#refs/heads/}"
	git -C "$refs_repo" fetch -q origin "+$ref:$dst_ref" >/dev/null 2>&1
	commit_stamp=$(git -C "$refs_repo" show --no-patch --pretty=%ct "$dst_ref")
	days_ago=$((($(date +%s) - $commit_stamp) / (24 * 3600)))
        if [[ $ref =~ -release- ]]; then
            days_limit=$days_for_release
        else
            days_limit=$days_for_master
        fi
	if [ "$days_ago" -gt "$days_limit" ]; then
	    if $human; then
		echo "$refs_repo: ${ref#$refs_prefix}: last updated $days_ago days ago"
	    else
		echo "$refs_repo:$ref"
	    fi
	fi
    done < <(git ls-remote "$refs_url" "${refs_prefix}*" | awk '{ print $2 }')
    )
}


jenkins_base_url="https://ci.linaro.org"
use_last_build="${use_last_build-no}"

count_all=0
list_err_noproject=()

declare -A test
declare -A alldiags

################## UTILITY FUNCTIONS
classify_get_project()
{
    verbose " * $1"

    # # zero-initialize test var
    test=(['gitproject']="" ['jkproject']="" ['branch']="" ['last_updated']=""
       ['poll_date']=""
       #
       ['run_nb']="" ['run_date']="" ['run_status']="" ['run_title']="" ['run_check_regression']=""
       #
       ['last_run']="" ['diag']="" )
       
    test['last_updated']=$(echo $1 | sed -e 's|.*: last updated ||')
    test['gitproject']=$(echo $1 |cut -d: -f 1)
    test['branch']=$(echo $1 |cut -d: -f 2)
    test['branch']=${test['branch']:1}
    test['jkproject']=$(echo ${test['branch']} | sed \
         -e's|tcwg_bmk_gnu_apm/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_apm-|' \
         -e's|tcwg_bmk_gnu_eabi_stm32/|tcwg_bmk_ci_gnu_eabi-XXX-tcwg_bmk_stm32-|' \
         -e's|tcwg_bmk_gnu_tk1/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_tk1-|' \
         -e's|tcwg_bmk_gnu_tx1/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_tx1-|' \
         -e's|tcwg_bmk_llvm_apm/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_apm-|' \
         -e's|tcwg_bmk_llvm_tk1/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_tk1-|' \
         -e's|tcwg_bmk_llvm_tx1/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_tx1-|' \
         -e's|tcwg_bmk_gnu_fx/|tcwg_bmk_ci_fujitsu_gnu-XXX-tcwg_bmk_fx-|' \
         -e's|tcwg_bmk_llvm_fx/|tcwg_bmk_ci_fujitsu_llvm-XXX-tcwg_bmk_fx-|' \
         -e's|tcwg_bmk_gnu_sq/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_sq-|' \
         -e's|tcwg_bmk_llvm_sq/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_sq-|' \
         -e's|tcwg_gcc_check/|tcwg_gcc_check-XXX-|' \
         -e's|tcwg_gcc_bootstrap/|tcwg_gcc_bootstrap-XXX-|' \
         -e's|tcwg_gcc_check_bootstrap/|tcwg_gcc_check_bootstrap-XXX-|' \
         -e's|tcwg_gnu_cross_build/|tcwg_gnu_cross_build-XXX-|' \
         -e's|tcwg_gnu_cross_check_gcc/|tcwg_gnu_cross_build-XXX-|' \
         -e's|tcwg_gnu_cross_check_gcc/|tcwg_gnu_cross_check_gcc-XXX-|' \
         -e's|tcwg_gnu_native_build/|tcwg_gnu_native_build-XXX-|' \
         -e's|tcwg_gnu_native_check_binutils/|tcwg_gnu_native_check_binutils-XXX-|' \
         -e's|tcwg_gnu_native_check_gcc/|tcwg_gnu_native_check_gcc-XXX-|' \
         -e's|tcwg_gnu_native_check_gdb/|tcwg_gnu_native_check_gdb-XXX-|' \
         -e's|tcwg_kernel/gnu|tcwg_kernel-gnu-XXX-gnu|' \
         -e's|tcwg_kernel/llvm|tcwg_kernel-llvm-XXX-llvm|' \
         -e's|tcwg_gnu_native_build/|tcwg_gnu_native_build-XXX-|')

    test['jkproject']=$(echo ${test['jkproject']} |sed -e 's|-XXX-|-build-|')
    verbose " : $jenkins_base_url/job/${test['jkproject']}"
    verbose " : $tmpdir/""${test['jkproject']}"

    mkdir -p $tmpdir/"${test['jkproject']}" ; cd $tmpdir/"${test['jkproject']}"
}

set_diag()
{
    diag_error="$1"
    test['diag']="$diag_error"

    if [ "${test['diag']}" == "ERROR (project doesnot exist)" ] &&
       [[ ! ${list_err_noproject[*]} =~ (^|[[:space:]])${test['branch']}($|[[:space:]]) ]]; then
        list_err_noproject+=("${test['branch']}");
    fi

    [ -z "${alldiags["$diag_error"]+set}" ] && alldiags["$diag_error"]=0
    alldiags["$diag_error"]="$(( alldiags["$diag_error"] + 1 ))"
    verbose "   ==> diag=$diag_error"
}

verbose ()
{
   if [ $verbose != false ]; then
     echo "$@"
   fi
}

download_project_file ()
{
    local filename=$1
    local local_file=$filename
    local remote_file

    remote_file="$(echo $filename | sed -e 's|__toppage__|.|')"

    cd $tmpdir/"${test['jkproject']}"
    [ -f "$local_file" ] && return

    mkdir -p "$(dirname "$local_file")"
    # echo $(pwd)/$local_file
    wget -O "$(pwd)/$local_file" -o /dev/null "$jenkins_base_url/job/${test['jkproject']}/$remote_file" || true
}

classify ()
{
    local condition="$1"
    local filename="$2"
    local expression="$3"
    local diag_error="$4"

    # Only if not already classified 
    if [ ! -z "${test['diag']}" ]; then return; fi

   download_project_file "$filename"

   if [ "$condition" == "exist" ]; then
      if [ ! -s "$filename" ]; then
         set_diag "$diag_error"
      fi
   fi
   if [ "$condition" == "grep" ] &&  [ -f "$filename" ]; then
      nb=$(grep -c "$expression" $filename || true)
      if [ "$nb" != "0" ]; then
         set_diag "$diag_error"
      fi
   fi
   if [ "$condition" == "xzgrep" ] &&  [ -s "$filename" ]; then
      nb=$(xzcat $filename | grep -c "$expression" || true)
      if [ "$nb" != "0" ]; then
         set_diag "$diag_error"
      fi
   fi
}


################## GET INFO FROM THE BUILD
get_project_info ()
{
     count_all=$((count_all+1))

     # Diag has been already classified. Probably means no project. don't go further
     if [ ! -z "${test['diag']}" ]; then return; fi

     # Last poll
     download_project_file scmPollLog
     test['poll_date']=$(grep 'Started on' scmPollLog | sed -e 's|.*Started on ||' -e 's|,||g' || true)
     test['poll_date']=$(echo ${test['poll_date']} | sed -e 's|,||g' -e 's| mo | month |g' -e 's| hr | hour |g' || true)
     test['poll_date']=$(date --date="${test['poll_date']}" +"%x %R")

     # LastBuild run date
     download_project_file lastBuild/__toppage__
     test['run_date']=$(grep 'Started .* ago' lastBuild/__toppage__ | sed -e 's|.*Started \(.*\) ago.*|\1 ago|'|head -1)
}
get_artifact_dir ()
{
     lookfor=$1
     download_project_file ${test['run_nb']}/artifact/artifacts/__toppage__
     local i nb stepname
     for i in {1..15}; do
        stepname=$(printf "%02d" $i)-$lookfor
        nb=$(grep -c "href=\"$stepname\"" ${test['run_nb']}/artifact/artifacts/__toppage__)
        if [ $nb != 0 ]; then
          test["run_dir_$lookfor"]="$stepname"
	  echo "$stepname"
	  #echo "$jenkins_base_url/job/${test['jkproject']}/${test['run_nb']}/artifact/artifacts/${test[run_dir_$lookfor]}"
          break
        fi
     done
}
get_run_title_and_status ()
{
     run=$1
     
     verbose "  - get_run_title_and_status() : $run"
     # Last run
     download_project_file $run/__toppage__
     if [ -s "$run/__toppage__" ]; then
       test['run_title']=$(grep '<title>.*</title>' $run/__toppage__ | head -1 | sed -e 's|.*<title>||' -e 's|</title>.*||'||true)
       test['run_title']=$(echo ${test['run_title']}|sed -e 's|.* #||' -e 's| \[Jenkins\].*||')
       test['run_nb']=$(echo ${test['run_title']}|sed -e 's|\([0-9]*\)-.*|\1|')
       test['run_status']=$(grep 'tooltip' $run/__toppage__ | head -1 | sed -e 's|.*tooltip="||' -e 's|"* .*||' ||true)
     fi
     verbose "   > [${test['run_status']}] ${test['run_title']}"
}
get_last_interesting_run ()
{
     gitprojectshort=$(echo ${test['gitproject']}|cut -d- -f1)

     test['last_run']="lastBuild"
     get_run_title_and_status "lastBuild"

     [ "x${test['run_nb']}" = "x" ] && return
     [[ "${test['gitproject']}" =~ base-artifacts ]] && return

     verbose "   . last interesting run() : ${test['run_nb']}"

     export r   # to avoid shellcheck unused warning
     for r in {1..8}; do
        get_run_title_and_status ${test['run_nb']}
        if [[ "${test['run_title']}" =~ $gitprojectshort ]]; then
          test['last_run']=${test['run_nb']}
          verbose "  > ${test['run_nb']}"
          return
        fi
        test['run_nb']=$((test['run_nb']-1))
     done
     verbose "  > ${test['run_nb']}"
}

################## CLASSIFY FUNCTIONS


classify_polling_error ()
{
     if [ ! -z "${test['diag']}" ]; then return; fi
     verbose "  - classify_polling_error()"
     classify grep scmPollLog "Connection timed out" "ERROR(timeout while polling)"
     classify grep scmPollLog "fatal: read error: Connection reset by peer" "ERROR(fatal polling error)"
}
classify_project_deleted ()
{
     if [ ! -z "${test['diag']}" ]; then return; fi
     verbose "  - classify_project_deleted()"
     classify exist __toppage__ "x" "ERROR (project doesnot exist)"
}
classify_project_disabled ()
{
     if [ ! -z "${test['diag']}" ]; then return; fi
     verbose "  - classify_project_disabled()"
     classify grep __toppage__ "Project DELETE ME" "ERROR (project disabled)"
}
classify_gcc_boostrap_timeout ()
{
     if [ ! -z "${test['diag']}" ]; then return; fi
     verbose "  - classify_gcc_boostrap_timeout()"
     classify grep __toppage__ "tcwg_gcc_bootstrap" "ERROR (bootstrap timeout)"
}

classify_analyse_console ()
{
     if [ ! -z "${test['diag']}" ]; then return; fi
     verbose "  - classify_analyse_console()"
     classify grep lastBuild/console "Build timed out" "ERROR (build timeout)"
     classify grep lastBuild/console "FATAL: \[ssh-agent\] Unable to start agent" "ERROR (cannot start ssh-agent)"
}

classify_analyse_result_file ()
{
     local stage
     if [ ! -z "${test['diag']}" ]; then return; fi
     if [ "${test['run_status']}" == "Success" ]; then return; fi

     verbose "  - classify_analyse_result_file()"
     download_project_file ${test['run_nb']}/artifact/artifacts/results

     while read line
     do
        # stage line
	pat='^# .*(reset_artifacts|build_abe|build_llvm|benchmark)'
	if [[ $line =~ $pat ]]; then
	   stage="$(echo $line|sed -e 's|# ||' -e 's| --.*||' -e 's|:.*||')"
	   #echo " $line => $stage"
	fi

        # Error line
	pat='^# Benchmarking infra is offline'
	if [[ $line =~ $pat ]]; then
	   set_diag "ERROR (infra offline)"
	fi

        # Error line
	pat='# .* error: patch failed'
	if [[ $line =~ $pat ]]; then
	   set_diag "ERROR (git patch failed)"
	fi

     done < "${test['run_nb']}/artifact/artifacts/results"

     # If diag is set
     if [ ! -z "${test['diag']}" ]; then return; fi

     # otherwise fill with the stage
     if [[ -v stage ]]; then
        set_diag "ERROR ($stage)";
     fi
}

## Detection quite fragile for the moment
classify_no_change_in_sources ()
{
     local pjt days_limit

     if [ ! -z "${test['diag']}" ]; then return; fi

     verbose "  - classify_no_change_in_sources()"
     # how many days
     if [[ ${test['jkproject']} =~ -release- ]]; then
        days_limit="$((days+days))"
     else
        days_limit="$days"
     fi

     # get date
     download_project_file ${test['run_nb']}/artifact/artifacts/jenkins/manifest.sh

     # shellcheck disable=SC2034
     declare -A rr debug
     # shellcheck disable=SC1090
     source ${test['run_nb']}/artifact/artifacts/jenkins/manifest.sh

     # set diag if appopriate
     pjt=$(echo ${test['gitproject']}|cut -d- -f1)
     if [ "${rr[debug_${pjt}_date]+abc}" ]; then
        local last_commit_date start_warn_date
        last_commit_date="${rr[debug_${pjt}_date]}"
        start_warn_date=$(date +%s --date="$days_limit days ago")

        if [ "$last_commit_date" -lt "$start_warn_date" ]; then
           set_diag "ERROR (no change in sources)";
        fi
     fi
}


################## PRINT SUGGESTIONS
print_suggestions ()
{
  local gitbase="ssh://git.linaro.org/toolchain/ci"

  if [ ${#list_err_noproject[@]} -ne 0 ]; then
    echo "1) For deleted projects you may want to DELETE the stored results branches:"
    tmpdir=/tmp/empty_git
    echo "mkdir -p $tmpdir && cd $tmpdir && git init"
    for repo in "${repos[@]}"; do
      echo "git push $gitbase/$repo.git \\"
      for br in "${list_err_noproject[@]}"; do
        echo "  --delete refs/heads/linaro-local/ci/$br \\"
      done
      echo ""
    done 
    echo "rm -rf $tmpdir"
  fi
}

################## MAIN CLASSIFY FUNCTIONS 
classify_failures ()
{
  local stale_jobs_file

  stale_jobs_file="$(pwd)/$1"

  [ "x$tmpdir" = "x" ] &&
    tmpdir="$(mktemp -d -t tmpdir-XXXXXXXXXX)"

  echo "working in $tmpdir"
  cd $tmpdir

  printf "\n"
  printf "%-35s | %-13s | %-16s | %-16s | %-50s\n" "AUTOMATIC DIAGNOSTIC" "LAST UPDATED" "LAST POLLING" "LAST RUN" "PROJECT NAME"
  printf "====================================================================================================================================================================\n"

  while read -r line;
  do
     # For now, we classify only base-artifacts stale jobs
     #[[ "$line" =~ ^base-artifacts: ]] || continue

     classify_get_project "$line"

     if [ $only != false ] && [[ ! ${test['jkproject']} =~ $only ]]; then
       continue
     fi

     if [ $verbose == "all" ]; then set -x; fi

     # Check if project exist before getting the infos
     classify_project_deleted

     # Get info from project
     get_project_info

     # Classify
     classify_polling_error

     # is disabled ?
     classify_project_disabled

     # deeper analyse
     get_last_interesting_run
     classify_analyse_console
     classify_analyse_result_file
     classify_no_change_in_sources

     printf "%-35s | %-13s | %-16s | %-16s | %-16s %-50s\n" "${test['diag']}" "${test['last_updated']}" "${test['poll_date']}" "${test['run_date']}" "[${test['gitproject']}]" "$jenkins_base_url/job/${test['jkproject']}/${test['run_nb']}"
  done < $stale_jobs_file

  printf "====================================================================================================================================================================\n"
  printf "SUMMARY : \n"
  for K in "${!alldiags[@]}"; do
  printf "  %-28s : %-3s\n" "$K" "${alldiags[$K]}"
  done
  printf "  %-28s : %-3s\n" "TOTAL FAILURES" "$count_all"
  printf "====================================================================================================================================================================\n"
  printf "SUGGESTIONS : \n"
  print_suggestions
  printf "====================================================================================================================================================================\n"
  printf "\n"

  [ $keep_tmp ] || rm -rf $tmpdir
}


# If classify is specified, classify the failures
if [ $classify != false ]; then
  classify_failures $classify
  exit 0;
fi


# Otherwise process the git to find not updated results
for repo in "${repos[@]}"; do
    process_git_url "$refs_url_prefix/$repo" &
done | sort

wait