tcwg-cleanup-stale-results.sh: Improve cleanup script.

- Add option --filter to look into part of the directories
- Speed up the find command by adding mindepth/maxdepth
- Always cleaning "git remote" pointers before going further

Change-Id: I38b32f671c10f4c8f71699346e3a7c49d2cc491c

1 files changed, 34 insertions, 15 deletions

diff --git a/tcwg-cleanup-stale-results.sh b/tcwg-cleanup-stale-results.sh
index c14c6272..4b86788d 100755
--- a/tcwg-cleanup-stale-results.sh
+++ b/tcwg-cleanup-stale-results.sh
@@ -15,19 +15,22 @@ results_top="${results_top-/home/tcwg-benchmark/results}"
 dryrun="${dryrun-true}"
 verbose="${verbose-false}"
 
+filter="${filter-}"
+
 if $verbose; then
     set -x
 fi
 
 # Delete "used_by" markers older than $days days.
-(set +f; find $results_top-* -name used_by -mtime "+$days" -delete)
+(set +f; find $results_top-* -maxdepth 4 -mindepth 4 -name used_by -mtime "+$days" -delete)
 
 inspect_base_artifact ()
 {
     local remote_name="$1"
     local remote_path="$2"
 
-    git -C "base-artifacts" remote add -f $remote_name "$remote_path"
+    # echo "# git -C base-artifacts remote add -f $remote_name $remote_path"
+    git -C "base-artifacts" remote add -f $remote_name "$remote_path" >& /dev/null
 
     while read br; do
 
@@ -35,11 +38,11 @@ inspect_base_artifact ()
 
         git -C "base-artifacts" checkout -q "$br" 2> /dev/null
 
-        readarray -t result_id_arr < <(get_git_history -0 base-artifacts results_id)
+        readarray -t result_id_arr < <(get_git_history -0 base-artifacts results_id 2> /dev/null)
 
         for result_id_file in "${result_id_arr[@]:1}"; do
             if ! [ -d "$results_top-$(cat $result_id_file)" ]; then
-                echo "WARNING: $results_top-$(cat $result_id_file) doesn't exist"
+                echo "WARNING: $results_top-$(cat $result_id_file) doesn't exists"
                 continue
             fi
             echo "$br ~ $(echo $result_id_file | cut -d/ -f4)" > "$results_top-$(cat $result_id_file)/used_by"
@@ -57,13 +60,24 @@ inspect_base_artifact ()
 clone_or_update_repo "base-artifacts" empty "$refs_url.git" auto empty
 git -C "base-artifacts" reset -q --hard
 
+# remove any existing remote repository.
+for r in $(git -C "base-artifacts" remote); do
+   git -C "base-artifacts" remote rm $r
+done
+
 # inspect and mark toolchain/ci/base-artifacts.git
-git -C "base-artifacts" remote rm origin
 inspect_base_artifact "origin" "$refs_url.git"
 
+# Prealably check check ssh connection
+ssh git.linaro.org help >& /dev/null
+
 # inspect and mark toolchain/ci/base-artifacts/<ci_project>/<ci_config>.git
 while read -r gitpath; do
 
+    if ! [[ $gitpath =~ $filter ]]; then
+        continue
+    fi
+
     ci_project_config="${gitpath#toolchain/ci/base-artifacts/}"
     remote_project_config="${ci_project_config/\//--}"
 
@@ -71,37 +85,42 @@ while read -r gitpath; do
 
 done < <(ssh git.linaro.org info toolchain/ci/base-artifacts|cut -f2|grep base-artifacts/tcwg)
 
+echo "All used_by placed .. preparing to delete useless directories"
+
 # dump/delete the necessary dirs
 while IFS= read -r -d '' dir; do
     # Skip already-deleted dirs (e.g., $dir's parent was deleted).
     if [ ! -d "$dir" ]; then
-	continue
+        continue
+    fi
+    if ! [[ $dir =~ $filter ]]; then
+        continue
     fi
 
     # Don't delete "used_by" dirs and dirs that have recent files
     # (i.e., "-mtime -$days").  E.g., in-progress benchmark might have uploaded
     # partial results.
-    if [ x"$(find "$dir" -name used_by -o -mtime "-$days" | head -n1)" != x"" ]; then
-	continue
+    if [ x"$(echo "$(find "$dir" -name used_by; find "$dir" -mindepth 1 -mtime "-$days")" | head -n1)" != x"" ]; then
+        continue
     fi
 
     # Don't delete subdirectories of a "used_by" parent.
     parent="$dir"
     used=false
     while [ x"$parent" != x"/home/tcwg-benchmark" ] && ! $used; do
-	parent=$(dirname "$parent")
-	if [ -f "$parent/used_by" ]; then
-	    used=true
-	fi
+        parent=$(dirname "$parent")
+        if [ -f "$parent/used_by" ]; then
+            used=true
+        fi
     done
     if $used; then
-	continue
+        continue
     fi
 
     echo "DELETE: $dir is not used"
     if $dryrun; then
-	echo "DRYRUN: rm -rf $dir"
+        echo "DRYRUN: rm -rf $dir"
     else
-	rm -rf "$dir"
+        rm -rf "$dir"
     fi
 done < <(set +f; find $results_top-* -maxdepth 3 -mindepth 3 -type d -print0)