#!/bin/bash set -euf -o pipefail scripts=$(dirname "$0") # shellcheck source=jenkins-helpers.sh . "$scripts/jenkins-helpers.sh" convert_args_to_variables "$@" days="${days-10}" human="${human-true}" refs_prefix="${refs_prefix-refs/heads/linaro-local/ci/}" refs_url_prefix="${refs_url_prefix-https://git.linaro.org/toolchain/ci}" repos=("${repos[@]-default}") verbose="${verbose-false}" classify="${classify-false}" only="${only-false}" keep_tmp="${keep_tmp-false}" tmpdir="${tmpdir-""}" process_manifest () { components=$* iterate_again=false declare -A rr manifest=base-artifacts/manifest.sh [ -f base-artifacts/jenkins/manifest.sh ] && manifest=base-artifacts/jenkins/manifest.sh # shellcheck disable=SC1090 source $manifest for c in $components; do if [ ! -v rr[debug_${c}_date] ]; then iterate_again=true else commit_stamps[$c]=$(eval echo "\${rr[debug_${c}_date]}") fi done unset rr } process_all_artifacts () { ( set -euf -o pipefail local baseartifacts_url="$refs_url_prefix/base-artifacts" local days_for_master days_for_release days_limit # Initialize base-artifacts repo (by cloning its "empty" branch). clone_or_update_repo "base-artifacts" empty "$baseartifacts_url.git" >/dev/null 2>&1 git -C "base-artifacts" reset -q --hard days_for_master="$days" days_for_release="$((days+days))" # Walk through all commits of all tcwg_bmk* branches and mark results # referenced in those results with "used_by" file. while IFS= read -r ref; do if [ $only != false ] && [[ ! $ref =~ $only ]]; then continue fi dst_ref="refs/remotes/origin/${ref#refs/heads/}" git -C "base-artifacts" fetch -q origin "+$ref:$dst_ref" >/dev/null 2>&1 git -C "base-artifacts" checkout FETCH_HEAD >/dev/null 2>&1 # -- get components for this project components="" if [ ! -d "base-artifacts"/git ]; then # For old style artifacts, we consider, they have not been updated since # git directory has been created in base-artifacts. i.e. # 100fd17 round-robin.sh: Fix "git rev-parse" of short histories commit_stamp=1660557250 days_ago=$((($(date +%s) - $commit_stamp) / (24 * 3600))) echo "all: ${ref#$refs_prefix}: No successful run since $days_ago days" continue; fi while read -e c; do c=${c#base-artifacts/git/} c=${c%_rev} components="$components $c" done < <(find base-artifacts/git -name '*_rev') # -- get all component commit stamps declare -A commit_stamps iterate_again=true while $iterate_again ; do process_manifest $components if $iterate_again; then ( git -C "base-artifacts" checkout HEAD~1 >/dev/null 2>&1 ) || iterate_again=false fi done # -- dump the messages days_limit=$days_for_master [[ $ref =~ -release- ]] && days_limit=$days_for_release for c in $components; do if [[ -v commit_stamps[$c] ]]; then days_ago=$((($(date +%s) - ${commit_stamps[$c]}) / (24 * 3600))) if [ "$days_ago" -gt "$days_limit" ]; then echo "$c: ${ref#$refs_prefix}: last updated $days_ago days ago" fi else echo "$c: ${ref#$refs_prefix}: no date for this component" fi done done < <(git ls-remote "$baseartifacts_url" "${refs_prefix}*" | awk '{ print $2 }') ) } jenkins_base_url="https://ci.linaro.org" use_last_build="${use_last_build-no}" count_all=0 list_err_noproject=() declare -A test declare -A alldiags ################## UTILITY FUNCTIONS classify_get_project() { verbose " * $1" # # zero-initialize test var test=(['gitproject']="" ['jkproject']="" ['branch']="" ['last_updated']="" ['poll_date']="" # ['run_nb']="" ['run_date']="" ['run_status']="" ['run_title']="" ['run_check_regression']="" # ['last_run']="" ['diag']="" ) test['last_updated']=$(echo $1 | sed -e 's|.*: last updated ||' -e 's|.*: No successful run since ||') test['gitproject']=$(echo $1 |cut -d: -f 1) test['branch']=$(echo $1 |cut -d: -f 2) test['branch']=${test['branch']:1} test['jkproject']=$(echo ${test['branch']} | sed \ -e's|tcwg_bmk_gnu_apm/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_apm-|' \ -e's|tcwg_bmk_gnu_eabi_stm32/|tcwg_bmk_ci_gnu_eabi-XXX-tcwg_bmk_stm32-|' \ -e's|tcwg_bmk_gnu_tk1/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_tk1-|' \ -e's|tcwg_bmk_gnu_tx1/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_tx1-|' \ -e's|tcwg_bmk_llvm_apm/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_apm-|' \ -e's|tcwg_bmk_llvm_tk1/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_tk1-|' \ -e's|tcwg_bmk_llvm_tx1/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_tx1-|' \ -e's|tcwg_bmk_gnu_fx/|tcwg_bmk_ci_fujitsu_gnu-XXX-tcwg_bmk_fx-|' \ -e's|tcwg_bmk_llvm_fx/|tcwg_bmk_ci_fujitsu_llvm-XXX-tcwg_bmk_fx-|' \ -e's|tcwg_bmk_gnu_sq/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_sq-|' \ -e's|tcwg_bmk_llvm_sq/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_sq-|' \ -e's|tcwg_gcc_check/|tcwg_gcc_check-XXX-|' \ -e's|tcwg_gcc_bootstrap/|tcwg_gcc_bootstrap-XXX-|' \ -e's|tcwg_gcc_check_bootstrap/|tcwg_gcc_check_bootstrap-XXX-|' \ -e's|tcwg_gnu_cross_build/|tcwg_gnu_cross_build-XXX-|' \ -e's|tcwg_gnu_cross_check_gcc/|tcwg_gnu_cross_build-XXX-|' \ -e's|tcwg_gnu_cross_check_gcc/|tcwg_gnu_cross_check_gcc-XXX-|' \ -e's|tcwg_gnu_native_build/|tcwg_gnu_native_build-XXX-|' \ -e's|tcwg_gnu_native_check_binutils/|tcwg_gnu_native_check_binutils-XXX-|' \ -e's|tcwg_gnu_native_check_gcc/|tcwg_gnu_native_check_gcc-XXX-|' \ -e's|tcwg_gnu_native_check_gdb/|tcwg_gnu_native_check_gdb-XXX-|' \ -e's|tcwg_kernel/gnu|tcwg_kernel-gnu-XXX-gnu|' \ -e's|tcwg_kernel/llvm|tcwg_kernel-llvm-XXX-llvm|' \ -e's|tcwg_gnu_native_build/|tcwg_gnu_native_build-XXX-|') test['jkproject']=$(echo ${test['jkproject']} |sed -e 's|-XXX-|-build-|') verbose " : $jenkins_base_url/job/${test['jkproject']}" verbose " : $tmpdir/""${test['jkproject']}" mkdir -p $tmpdir/"${test['jkproject']}" ; cd $tmpdir/"${test['jkproject']}" } set_diag() { diag_error="$1" test['diag']="$diag_error" if [ "${test['diag']}" == "ERROR (project doesnot exist)" ] && [[ ! ${list_err_noproject[*]} =~ (^|[[:space:]])${test['branch']}($|[[:space:]]) ]]; then list_err_noproject+=("${test['branch']}"); fi [ -z "${alldiags["$diag_error"]+set}" ] && alldiags["$diag_error"]=0 alldiags["$diag_error"]="$(( alldiags["$diag_error"] + 1 ))" verbose " ==> diag=$diag_error" } verbose () { if [ $verbose != false ]; then echo "$@" fi } download_project_file () { local filename=$1 local local_file=$filename local remote_file remote_file="$(echo $filename | sed -e 's|__toppage__|.|')" cd $tmpdir/"${test['jkproject']}" [ -f "$local_file" ] && return mkdir -p "$(dirname "$local_file")" # echo $(pwd)/$local_file wget -O "$(pwd)/$local_file" -o /dev/null "$jenkins_base_url/job/${test['jkproject']}/$remote_file" || true } classify () { local condition="$1" local filename="$2" local expression="$3" local diag_error="$4" # Only if not already classified if [ ! -z "${test['diag']}" ]; then return; fi download_project_file "$filename" if [ "$condition" == "exist" ]; then if [ ! -s "$filename" ]; then set_diag "$diag_error" fi fi if [ "$condition" == "grep" ] && [ -f "$filename" ]; then nb=$(grep -c "$expression" $filename || true) if [ "$nb" != "0" ]; then set_diag "$diag_error" fi fi if [ "$condition" == "xzgrep" ] && [ -s "$filename" ]; then nb=$(xzcat $filename | grep -c "$expression" || true) if [ "$nb" != "0" ]; then set_diag "$diag_error" fi fi } ################## GET INFO FROM THE BUILD get_project_info () { count_all=$((count_all+1)) # Diag has been already classified. Probably means no project. don't go further if [ ! -z "${test['diag']}" ]; then return; fi # Last poll download_project_file scmPollLog test['poll_date']=$(grep 'Started on' scmPollLog | sed -e 's|.*Started on ||' -e 's|,||g' || true) test['poll_date']=$(echo ${test['poll_date']} | sed -e 's|,||g' -e 's| mo | month |g' -e 's| hr | hour |g' || true) test['poll_date']=$(date --date="${test['poll_date']}" +"%x %R") # LastBuild run date download_project_file lastBuild/__toppage__ test['run_date']=$(grep 'Started .* ago' lastBuild/__toppage__ | sed -e 's|.*Started \(.*\) ago.*|\1 ago|'|head -1) } get_artifact_dir () { lookfor=$1 download_project_file ${test['run_nb']}/artifact/artifacts/__toppage__ local i nb stepname for i in {1..15}; do stepname=$(printf "%02d" $i)-$lookfor nb=$(grep -c "href=\"$stepname\"" ${test['run_nb']}/artifact/artifacts/__toppage__) if [ $nb != 0 ]; then test["run_dir_$lookfor"]="$stepname" echo "$stepname" #echo "$jenkins_base_url/job/${test['jkproject']}/${test['run_nb']}/artifact/artifacts/${test[run_dir_$lookfor]}" break fi done } get_run_title_and_status () { run=$1 verbose " - get_run_title_and_status() : $run" # Last run download_project_file $run/__toppage__ if [ -s "$run/__toppage__" ]; then test['run_title']=$(grep '.*' $run/__toppage__ | head -1 | sed -e 's|.*||' -e 's|.*||'||true) test['run_title']=$(echo ${test['run_title']}|sed -e 's|.* #||' -e 's| \[Jenkins\].*||') test['run_nb']=$(echo ${test['run_title']}|sed -e 's|\([0-9]*\)-.*|\1|') test['run_status']=$(grep 'tooltip' $run/__toppage__ | head -1 | sed -e 's|.*tooltip="||' -e 's|"* .*||' ||true) fi verbose " > [${test['run_status']}] ${test['run_title']}" } get_last_interesting_run () { gitprojectshort=$(echo ${test['gitproject']}|cut -d- -f1) test['last_run']="lastBuild" get_run_title_and_status "lastBuild" [ "x${test['run_nb']}" = "x" ] && return [[ "${test['gitproject']}" =~ base-artifacts ]] && return verbose " . last interesting run() : ${test['run_nb']}" export r # to avoid shellcheck unused warning for r in {1..8}; do get_run_title_and_status ${test['run_nb']} if [[ "${test['run_title']}" =~ $gitprojectshort ]] || [ $gitprojectshort == "all" ]; then test['last_run']=${test['run_nb']} verbose " > ${test['run_nb']}" return fi test['run_nb']=$((test['run_nb']-1)) done verbose " > ${test['run_nb']}" } ################## CLASSIFY FUNCTIONS classify_polling_error () { if [ ! -z "${test['diag']}" ]; then return; fi verbose " - classify_polling_error()" classify grep scmPollLog "Connection timed out" "ERROR(timeout while polling)" classify grep scmPollLog "fatal: read error: Connection reset by peer" "ERROR(fatal polling error)" } classify_project_deleted () { if [ ! -z "${test['diag']}" ]; then return; fi verbose " - classify_project_deleted()" classify exist __toppage__ "x" "ERROR (project doesnot exist)" } classify_project_disabled () { if [ ! -z "${test['diag']}" ]; then return; fi verbose " - classify_project_disabled()" classify grep __toppage__ "Project DELETE ME" "ERROR (project disabled)" } classify_gcc_boostrap_timeout () { if [ ! -z "${test['diag']}" ]; then return; fi verbose " - classify_gcc_boostrap_timeout()" classify grep __toppage__ "tcwg_gcc_bootstrap" "ERROR (bootstrap timeout)" } classify_analyse_console () { if [ ! -z "${test['diag']}" ]; then return; fi verbose " - classify_analyse_console()" classify grep lastBuild/console "Build timed out" "ERROR (build timeout)" classify grep lastBuild/console "FATAL: \[ssh-agent\] Unable to start agent" "ERROR (cannot start ssh-agent)" } classify_analyse_result_file () { local stage if [ ! -z "${test['diag']}" ]; then return; fi if [ "${test['run_status']}" == "Success" ]; then return; fi verbose " - classify_analyse_result_file()" download_project_file ${test['run_nb']}/artifact/artifacts/results while read line do # stage line pat='^# .*(reset_artifacts|build_abe|build_llvm|benchmark|linux_n_obj)' if [[ $line =~ $pat ]]; then stage="$(echo $line|sed -e 's|# ||' -e 's| --.*||' -e 's|:.*||')" #echo " $line => $stage" fi # Clear error line pat='^# Benchmarking infra is offline' if [[ $line =~ $pat ]]; then set_diag "ERROR (infra offline)" fi pat='# .* error: patch failed' if [[ $line =~ $pat ]]; then set_diag "ERROR (git patch failed)" fi pat='# First few build errors in logs' if [[ $line =~ $pat ]]; then set_diag "ERROR ($stage build errors)" fi pat='^# .*grew in size.*' if [[ $line =~ $pat ]]; then set_diag "ERROR (grew in size)" fi pat='^# .*slowed down.*' if [[ $line =~ $pat ]]; then set_diag "ERROR (slowed down)" fi # single message before reset-artifact pat='^# FAILED' if [[ $line =~ $pat ]] && [ ! -v stage ]; then set_diag "ERROR (FAILED in reset_artifacts)" fi [ ! -z "${test['diag']}" ] && break done < "${test['run_nb']}/artifact/artifacts/results" # If diag is set if [ ! -z "${test['diag']}" ]; then return; fi # otherwise fill with the stage if [[ -v stage ]]; then set_diag "ERROR ($stage)"; fi } ## Detection quite fragile for the moment classify_no_change_in_sources () { local pjt days_limit if [ ! -z "${test['diag']}" ]; then return; fi verbose " - classify_no_change_in_sources()" # how many days if [[ ${test['jkproject']} =~ -release- ]]; then days_limit="$((days+days))" else days_limit="$days" fi # get date download_project_file ${test['run_nb']}/artifact/artifacts/jenkins/manifest.sh # shellcheck disable=SC2034 declare -A rr debug # shellcheck disable=SC1090 source ${test['run_nb']}/artifact/artifacts/jenkins/manifest.sh # set diag if appopriate pjt=$(echo ${test['gitproject']}|cut -d- -f1) if [ "${rr[debug_${pjt}_date]+abc}" ]; then local last_commit_date start_warn_date last_commit_date="${rr[debug_${pjt}_date]}" start_warn_date=$(date +%s --date="$days_limit days ago") if [ "$last_commit_date" -lt "$start_warn_date" ]; then set_diag "ERROR (no change in sources)"; fi fi } ################## PRINT SUGGESTIONS print_suggestions () { local gitbase="ssh://git.linaro.org/toolchain/ci" if [ ${#list_err_noproject[@]} -ne 0 ]; then echo "1) For deleted projects you may want to DELETE the stored results branches:" tmpdir=/tmp/empty_git echo "mkdir -p $tmpdir && cd $tmpdir && git init" for repo in "${repos[@]}"; do echo "git push $gitbase/$repo.git \\" for br in "${list_err_noproject[@]}"; do echo " --delete refs/heads/linaro-local/ci/$br \\" done echo "" done echo "rm -rf $tmpdir" fi } ################## MAIN CLASSIFY FUNCTIONS classify_failures () { local stale_jobs_file stale_jobs_file="$(pwd)/$1" [ "x$tmpdir" = "x" ] && tmpdir="$(mktemp -d -t tmpdir-XXXXXXXXXX)" echo "working in $tmpdir" cd $tmpdir printf "\n" printf "%-35s | %-13s | %-16s | %-16s | %-50s\n" "AUTOMATIC DIAGNOSTIC" "LAST UPDATED" "LAST POLLING" "LAST RUN" "PROJECT NAME" printf "====================================================================================================================================================================\n" while read -r line; do classify_get_project "$line" if [ $only != false ] && [[ ! ${test['jkproject']} =~ $only ]]; then continue fi if [ $verbose == "all" ]; then set -x; fi # Check if project exist before getting the infos classify_project_deleted # Get info from project get_project_info # Classify classify_polling_error # is disabled ? classify_project_disabled # deeper analyse get_last_interesting_run classify_analyse_console classify_analyse_result_file classify_no_change_in_sources printf "%-35s | %-13s | %-16s | %-16s | %-16s %-50s\n" "${test['diag']}" "${test['last_updated']}" "${test['poll_date']}" "${test['run_date']}" "[${test['gitproject']}]" "$jenkins_base_url/job/${test['jkproject']}/${test['run_nb']}" done < $stale_jobs_file printf "====================================================================================================================================================================\n" printf "SUMMARY : \n" for K in "${!alldiags[@]}"; do printf " %-28s : %-3s\n" "$K" "${alldiags[$K]}" done printf " %-28s : %-3s\n" "TOTAL FAILURES" "$count_all" printf "====================================================================================================================================================================\n" printf "SUGGESTIONS : \n" print_suggestions printf "====================================================================================================================================================================\n" printf "\n" [ $keep_tmp ] || rm -rf $tmpdir } # If classify is specified, classify the failures if [ $classify != false ]; then classify_failures $classify exit 0; fi process_all_artifacts