summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-01-03 05:06:25 +0000
committerMaxim Kuvyrkov <maxim.kuvyrkov@linaro.org>2021-01-03 10:14:32 +0000
commit50781c0bdac875936dab082d1043f7f4a3317546 (patch)
tree5b1237cbdb8219c6d43599167a51d30aa607461d
parentf769fb587484a8dd86d715edee4063b5c4a0b52d (diff)
Reduce "docker pull" requests to dockerhub
We attempt to run all our build using current versions of docker images. Unfortunately, now that dockerhub limits pull requests, we need to be more considerate to when we pull the image or attempt to use the local copy. This is the first stab at the problem. We use stamp files under /home/shared/docker/ to track times of last image pull and last image use. We then run "docker pull" on images that are older than a day to avoid using terribly-outdated local copies. We also switch to deleting images that haven't been used for 3+ days in tcwg-cleanup-stale-containers.sh -- instead of deleting all not-used-at- the-moment images. Change-Id: I3839fe030b30adaee318f7453c6d2e47185f4ffc
-rwxr-xr-xstart-container-docker.sh42
-rwxr-xr-xtcwg-cleanup-stale-containers.sh17
2 files changed, 57 insertions, 2 deletions
diff --git a/start-container-docker.sh b/start-container-docker.sh
index 6fbec8e4..bc3f6145 100755
--- a/start-container-docker.sh
+++ b/start-container-docker.sh
@@ -215,6 +215,33 @@ fi
image=linaro/ci-${container_arch}-tcwg-build-ubuntu:${distro}
+stamp_dir=/home/shared/docker
+if ! [ -d "$stamp_dir" ]; then
+ sudo mkdir -p "$stamp_dir"
+ sudo chmod 0777 "$stamp_dir"
+fi
+
+# We use two stamp files per image:
+# - $image_stamp.pull -- time of last image pull
+# - $image_stamp.use -- time of last image use
+image_stamp="$stamp_dir/$(echo "$image" | tr "/:" "_")"
+
+# We attempt to run all our builds using current versions of docker images.
+# Unfortunately, now that dockerhub limits pull requests, we need to be more
+# considerate to when we pull the image or attempt to use a local copy.
+# Also note that "docker run" below will automatically pull the image if there
+# is no local copy.
+pull_image=false
+
+# For starters, let's try to pull images once a day. This guarantees
+# that any change to master docker images will be deployed within a day.
+pull_if_older_than=$(($(date +%s) - 1*24*60*60))
+# Use negative comparison to handle non-existent stamp files.
+if ! [ "$(stat -c %Z "$image_stamp.pull" 2>/dev/null)" \
+ -gt $pull_if_older_than ]; then
+ pull_image=true
+fi
+
# Avoid connexion sharing because of race conditions with parallel
# builds
SSH="ssh -S none"
@@ -224,7 +251,15 @@ SSH="ssh -S none"
# Instead of:
# "foo bar docker" <...>
DOCKER="$dryruncmd $SSH $session_host docker"
-$DOCKER pull $image || ssh_error $?
+
+if $pull_image; then
+ $DOCKER pull $image || ssh_error $?
+ # Remove the stamp to avoid permission issues (we have rwx permissions
+ # for all on the directory, so we can always remove a file, but only
+ # owner can modify files.
+ rm -f "$image_stamp.pull"
+ touch "$image_stamp.pull"
+fi
SECURITY="--cap-add=SYS_PTRACE"
# We need this because of a bug in libgo's configure script:
@@ -367,6 +402,11 @@ if $ssh_info; then
ssh_info_opt="ssh_host=${user}$session_host ssh_port=$session_port"
fi
+# Update the time of image use, so that we don't remove the image in
+# tcwg-cleanup-stale-containers .
+rm -f "$image_stamp.use"
+touch "$image_stamp.use"
+
# Restore stdout/stderr
exec 1>&3 2>&4
diff --git a/tcwg-cleanup-stale-containers.sh b/tcwg-cleanup-stale-containers.sh
index 2fa246cf..fe626621 100755
--- a/tcwg-cleanup-stale-containers.sh
+++ b/tcwg-cleanup-stale-containers.sh
@@ -176,7 +176,22 @@ else
fi
if $cleanup_images; then
- docker image prune -a -f
+ # See start-container-docker.sh for background on image stamp files.
+ stamp_dir=/home/shared/docker
+
+ # Untag and prune images that haven't been used for 3 days or more.
+ for image in $(docker images --format "{{.Repository}}:{{.Tag}}"); do
+ image_stamp="$stamp_dir/$(echo "$image" | tr "/:" "_")"
+ remove_if_not_used_since=$(($(date +%s) - 3*24*60*60))
+ # Use negative comparison to handle non-existent stamp files.
+ if ! [ "$(stat -c %Z "$image_stamp.use" 2>/dev/null)" \
+ -gt $remove_if_not_used_since ]; then
+ # Untag the image.
+ docker rmi "$image"
+ fi
+ done
+ # Prune untagged images.
+ docker image prune -f
else
echo "DRY_RUN: NOT REMOVING UNUSED IMAGES"
fi