#!/bin/bash set -euf -o pipefail scripts=$(dirname "$0") # shellcheck source=jenkins-helpers.sh . "$scripts/jenkins-helpers.sh" convert_args_to_variables "$@" days="${days-10}" human="${human-true}" refs_prefix="${refs_prefix-refs/heads/linaro-local/ci/}" refs_url_prefix="${refs_url_prefix-https://git.linaro.org/toolchain/ci}" repos=("${repos[@]-default}") verbose="${verbose-false}" classify="${classify-false}" only="${only-false}" keep_tmp="${keep_tmp-false}" tmpdir="${tmpdir-""}" process_manifest () { components=$* iterate_again=false declare -A rr manifest=base-artifacts/manifest.sh [ -f base-artifacts/jenkins/manifest.sh ] && manifest=base-artifacts/jenkins/manifest.sh # shellcheck disable=SC1090 source $manifest for c in $components; do if [ ! -v rr[debug_${c}_date] ]; then iterate_again=true else commit_stamps[$c]=$(eval echo "\${rr[debug_${c}_date]}") fi done unset rr } process_all_artifacts () { ( set -euf -o pipefail local baseartifacts_url="$refs_url_prefix/base-artifacts" local days_for_master days_for_release days_limit # Initialize base-artifacts repo (by cloning its "empty" branch). clone_or_update_repo "base-artifacts" empty "$baseartifacts_url.git" >/dev/null 2>&1 git -C "base-artifacts" reset -q --hard days_for_master="$days" days_for_release="$((days+days))" # Walk through all commits of all tcwg_bmk* branches and mark results # referenced in those results with "used_by" file. while IFS= read -r ref; do if [ $only != false ] && [[ ! $ref =~ $only ]]; then continue fi dst_ref="refs/remotes/origin/${ref#refs/heads/}" git -C "base-artifacts" fetch -q origin "+$ref:$dst_ref" >/dev/null 2>&1 git -C "base-artifacts" checkout FETCH_HEAD >/dev/null 2>&1 # -- get components for this project components="" if [ ! -d "base-artifacts"/git ]; then # For old style artifacts, we consider, they have not been updated since # git directory has been created in base-artifacts. i.e. # 100fd17 round-robin.sh: Fix "git rev-parse" of short histories commit_stamp=1660557250 days_ago=$((($(date +%s) - $commit_stamp) / (24 * 3600))) echo "all: ${ref#$refs_prefix}: No successful run since $days_ago days" continue; fi while read -e c; do c=${c#base-artifacts/git/} c=${c%_rev} components="$components $c" done < <(find base-artifacts/git -name '*_rev') # -- get all component commit stamps declare -A commit_stamps iterate_again=true while $iterate_again ; do process_manifest $components if $iterate_again; then ( git -C "base-artifacts" checkout HEAD~1 >/dev/null 2>&1 ) || iterate_again=false fi done # -- dump the messages days_limit=$days_for_master [[ $ref =~ -release- ]] && days_limit=$days_for_release for c in $components; do if [[ -v commit_stamps[$c] ]]; then days_ago=$((($(date +%s) - ${commit_stamps[$c]}) / (24 * 3600))) if [ "$days_ago" -gt "$days_limit" ]; then echo "$c: ${ref#$refs_prefix}: last updated $days_ago days ago" fi else echo "$c: ${ref#$refs_prefix}: no date for this component" fi done done < <(git ls-remote "$baseartifacts_url" "${refs_prefix}*" | awk '{ print $2 }') ) } jenkins_base_url="https://ci.linaro.org" use_last_build="${use_last_build-no}" count_all=0 list_err_noproject=() declare -A test declare -A alldiags ################## UTILITY FUNCTIONS classify_get_project() { verbose " * $1" # #Â zero-initialize test var test=(['gitproject']="" ['jkproject']="" ['branch']="" ['last_updated']="" ['poll_date']="" # ['run_nb']="" ['run_date']="" ['run_status']="" ['run_title']="" ['run_check_regression']="" # ['last_run']="" ['diag']="" ) test['last_updated']=$(echo $1 | sed -e 's|.*: last updated ||' -e 's|.*: No successful run since ||') test['gitproject']=$(echo $1 |cut -d: -f 1) test['branch']=$(echo $1 |cut -d: -f 2) test['branch']=${test['branch']:1} test['jkproject']=$(echo ${test['branch']} | sed \ -e's|tcwg_bmk_gnu_apm/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_apm-|' \ -e's|tcwg_bmk_gnu_eabi_stm32/|tcwg_bmk_ci_gnu_eabi-XXX-tcwg_bmk_stm32-|' \ -e's|tcwg_bmk_gnu_tk1/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_tk1-|' \ -e's|tcwg_bmk_gnu_tx1/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_tx1-|' \ -e's|tcwg_bmk_llvm_apm/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_apm-|' \ -e's|tcwg_bmk_llvm_tk1/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_tk1-|' \ -e's|tcwg_bmk_llvm_tx1/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_tx1-|' \ -e's|tcwg_bmk_gnu_fx/|tcwg_bmk_ci_fujitsu_gnu-XXX-tcwg_bmk_fx-|' \ -e's|tcwg_bmk_llvm_fx/|tcwg_bmk_ci_fujitsu_llvm-XXX-tcwg_bmk_fx-|' \ -e's|tcwg_bmk_gnu_sq/|tcwg_bmk_ci_gnu-XXX-tcwg_bmk_sq-|' \ -e's|tcwg_bmk_llvm_sq/|tcwg_bmk_ci_llvm-XXX-tcwg_bmk_sq-|' \ -e's|tcwg_gcc_check/|tcwg_gcc_check-XXX-|' \ -e's|tcwg_gcc_bootstrap/|tcwg_gcc_bootstrap-XXX-|' \ -e's|tcwg_gcc_check_bootstrap/|tcwg_gcc_check_bootstrap-XXX-|' \ -e's|tcwg_gnu_cross_build/|tcwg_gnu_cross_build-XXX-|' \ -e's|tcwg_gnu_cross_check_gcc/|tcwg_gnu_cross_build-XXX-|' \ -e's|tcwg_gnu_cross_check_gcc/|tcwg_gnu_cross_check_gcc-XXX-|' \ -e's|tcwg_gnu_native_build/|tcwg_gnu_native_build-XXX-|' \ -e's|tcwg_gnu_native_check_binutils/|tcwg_gnu_native_check_binutils-XXX-|' \ -e's|tcwg_gnu_native_check_gcc/|tcwg_gnu_native_check_gcc-XXX-|' \ -e's|tcwg_gnu_native_check_gdb/|tcwg_gnu_native_check_gdb-XXX-|' \ -e's|tcwg_kernel/gnu|tcwg_kernel-gnu-XXX-gnu|' \ -e's|tcwg_kernel/llvm|tcwg_kernel-llvm-XXX-llvm|' \ -e's|tcwg_gnu_native_build/|tcwg_gnu_native_build-XXX-|') test['jkproject']=$(echo ${test['jkproject']} |sed -e 's|-XXX-|-build-|') verbose " : $jenkins_base_url/job/${test['jkproject']}" verbose " : $tmpdir/""${test['jkproject']}" mkdir -p $tmpdir/"${test['jkproject']}" ; cd $tmpdir/"${test['jkproject']}" } set_diag() { diag_error="$1" test['diag']="$diag_error" if [ "${test['diag']}" == "ERROR (project doesnot exist)" ] && [[ ! ${list_err_noproject[*]} =~ (^|[[:space:]])${test['branch']}($|[[:space:]]) ]]; then list_err_noproject+=("${test['branch']}"); fi [ -z "${alldiags["$diag_error"]+set}" ] && alldiags["$diag_error"]=0 alldiags["$diag_error"]="$(( alldiags["$diag_error"] + 1 ))" verbose " ==> diag=$diag_error" } verbose () { if [ $verbose != false ]; then echo "$@" fi } download_project_file () { local filename=$1 local local_file=$filename local remote_file remote_file="$(echo $filename | sed -e 's|__toppage__|.|')" cd $tmpdir/"${test['jkproject']}" [ -f "$local_file" ] && return mkdir -p "$(dirname "$local_file")" #Â echo $(pwd)/$local_file wget -O "$(pwd)/$local_file" -o /dev/null "$jenkins_base_url/job/${test['jkproject']}/$remote_file" || true } classify () { local condition="$1" local filename="$2" local expression="$3" local diag_error="$4" # Only if not already classified if [ ! -z "${test['diag']}" ]; then return; fi download_project_file "$filename" if [ "$condition" == "exist" ]; then if [ ! -s "$filename" ]; then set_diag "$diag_error" fi fi if [ "$condition" == "grep" ] && [ -f "$filename" ]; then nb=$(grep -c "$expression" $filename || true) if [ "$nb" != "0" ]; then set_diag "$diag_error" fi fi if [ "$condition" == "xzgrep" ] && [ -s "$filename" ]; then nb=$(xzcat $filename | grep -c "$expression" || true) if [ "$nb" != "0" ]; then set_diag "$diag_error" fi fi } ################## GET INFO FROM THE BUILD get_project_info () { count_all=$((count_all+1)) # Diag has been already classified. Probably means no project. don't go further if [ ! -z "${test['diag']}" ]; then return; fi #Â Last poll download_project_file scmPollLog test['poll_date']=$(grep 'Started on' scmPollLog | sed -e 's|.*Started on ||' -e 's|,||g' || true) test['poll_date']=$(echo ${test['poll_date']} | sed -e 's|,||g' -e 's| mo | month |g' -e 's| hr | hour |g' || true) test['poll_date']=$(date --date="${test['poll_date']}" +"%x %R") # LastBuild run date download_project_file lastBuild/__toppage__ test['run_date']=$(grep 'Started .* ago' lastBuild/__toppage__ | sed -e 's|.*Started \(.*\) ago.*|\1 ago|'|head -1) } get_artifact_dir () { lookfor=$1 download_project_file ${test['run_nb']}/artifact/artifacts/__toppage__ local i nb stepname for i in {1..15}; do stepname=$(printf "%02d" $i)-$lookfor nb=$(grep -c "href=\"$stepname\"" ${test['run_nb']}/artifact/artifacts/__toppage__) if [ $nb != 0 ]; then test["run_dir_$lookfor"]="$stepname" echo "$stepname" #echo "$jenkins_base_url/job/${test['jkproject']}/${test['run_nb']}/artifact/artifacts/${test[run_dir_$lookfor]}" break fi done } get_run_title_and_status () { run=$1 verbose " - get_run_title_and_status() : $run" # Last run download_project_file $run/__toppage__ if [ -s "$run/__toppage__" ]; then test['run_title']=$(grep '