From 7970c7fdae84e58f697cce9e59559cbecdceac47 Mon Sep 17 00:00:00 2001
From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Date: Mon, 29 May 2023 10:40:30 +0000
Subject: pw-helpers.sh (pw_yaml_get): Cache requests to PW to avoid throttling

Change-Id: Ic02c19ecac1e27a63d14f238cd97da2bf228ad84
---
 pw-helpers.sh | 43 ++++++++++++++++++++++++++++++++++++-------
 pw-trigger.sh |  9 +++++++++
 2 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/pw-helpers.sh b/pw-helpers.sh
index 0a3cfdfb..893dbd69 100755
--- a/pw-helpers.sh
+++ b/pw-helpers.sh
@@ -56,6 +56,8 @@ pw_init ()
 	"https://patchwork.sourceware.org/api/1.2/"
     git -C "$project" config pw.project "$project"
 
+    pw_clear_cache
+
     # BE CAREFUL WITH $token
     set +x
     local token="$2"
@@ -73,10 +75,23 @@ pw_deinit ()
     set -euf -o pipefail
     local project="$1"
 
+    rm -rf "/tmp/pw-yaml-cache-$$"
+
     git -C "$project" config --unset pw.token
     )
 }
 
+# Clear pw_yaml cache.
+pw_clear_cache ()
+{
+    (
+    set -euf -o pipefail
+
+    rm -rf "/tmp/pw-yaml-cache-$$"
+    mkdir "/tmp/pw-yaml-cache-$$"
+    )
+}
+
 # Get specified piece of data from git-pw yaml output.
 # This is reasonably unstable and relies heavily on git-pw's yaml format and
 # field names not changing.
@@ -113,11 +128,6 @@ pw_yaml_get ()
     # Reduce noise in the logs
     set +x
 
-    local yaml
-    yaml=$(mktemp)
-    # shellcheck disable=SC2064
-    trap "rm -f $yaml" EXIT
-
     local -a cmd
     case "$id" in
 	"--owner"*)
@@ -129,8 +139,25 @@ pw_yaml_get ()
 	    ;;
     esac
 
-    # Crash if PW throttles connection.  Otherwise, this will hang indefinitely.
-    timeout 1m git -C "$project" pw "$section" "${cmd[@]}" -f yaml > "$yaml"
+    local -a git_cmd
+    git_cmd=(git -C "$project" pw "$section" "${cmd[@]}" -f yaml)
+
+    local yaml
+    yaml=$(IFS="_"; echo "${git_cmd[*]}" | tr "/" "_")
+    yaml="/tmp/pw-yaml-cache-$$/$yaml"
+    if [ -f "$yaml" ]; then
+	touch "$yaml"
+    else
+	# Timeout if PW throttles connection.  Otherwise, this would hang
+	# indefinitely.
+	timeout 1m "${git_cmd[@]}" > "$yaml" &
+	local res
+	res=0 && wait $! || res=$?
+	if [ $res != 0 ]; then
+	    rm -f "$yaml"
+	    return $res
+	fi
+    fi
 
     local len
     len=$(shyaml get-length < "$yaml")
@@ -314,6 +341,7 @@ pw_ci_bot_bundle_check_and_add ()
 	git -C "$project" pw bundle create --public -f yaml \
 	    "$bundle_name" "$patch_id"
     fi
+    pw_clear_cache
     )
 }
 
@@ -352,5 +380,6 @@ pw_ci_bot_bundle_remove ()
     elif $allow_delete; then
 	git -C "$project" pw bundle delete -f yaml "$bundle_id"
     fi
+    pw_clear_cache
     )
 }
diff --git a/pw-trigger.sh b/pw-trigger.sh
index 659c7bc8..e53bfb85 100755
--- a/pw-trigger.sh
+++ b/pw-trigger.sh
@@ -39,6 +39,8 @@ while [ "$queued_id" != "" ]; do
 	# "bundle remove" will refuse to delete the last patch.
 	break
     fi
+    # Clear pw_yaml cache after above "git pw bundle remove"
+    pw_clear_cache
 done
 
 triggered_name=$(pw_bundle_name triggered "$ci_bot")
@@ -54,6 +56,7 @@ mkdir -p "$out_dir"
 len=$(shyaml get-length < "$yaml")
 i=$len
 j=0
+queued_patches=()
 # Go through series with the oldest first.
 while [ "$i" -gt "0" ]; do
     i=$(($i - 1 ))
@@ -87,6 +90,12 @@ while [ "$i" -gt "0" ]; do
 ${project}_git=pw://series/$series_id
 EOF
 
+    # Delay adding patches to "queued" bundle to avoid clearing cache of
+    # pw_get_yaml.
+    queued_patches+=("$patch_id")
+done
+
+for patch_id in "${queued_patches[@]}"; do
     pw_ci_bot_bundle_check_and_add \
 	"$project" queued "$ci_bot" "$patch_id" true
 done
-- 
cgit v1.2.3