#!/bin/bash set -euf -o pipefail # shellcheck source=jenkins-helpers.sh . "$(dirname $0)"/jenkins-helpers.sh convert_args_to_variables "$@" shift "$SHIFT_CONVERTED_ARGS" obligatory_variables container image declare container image dryrun="${dryrun-false}" keep_existing="${keep_existing-true}" verbose="${verbose-true}" additional_options="${additional_options-}" test_docker="${test_docker-false}" if $verbose; then set -x; fi # Check that docker can start a container. test_docker() { timeout 30s /root/docker-wrapper ps /root/docker-wrapper maybepull "$image" /root/docker-wrapper run --rm --entrypoint=/bin/sh "$image" echo "NOTE: Docker seems to be OK" } if [ -f /root/docker-wrapper ]; then # /root/docker-wrapper is created by dockerfiles/tcwg-base/tcwg-host/run.sh; # on benchmarking boards /root is bind-mounted inside "host" container. if $test_docker; then # The fact that we are here implies that we running as root on # a bare machine. test_docker & if ! wait $!; then storage_driver=$(timeout 30s /root/docker-wrapper info \ | grep "Storage Driver" | awk '{print $3}' \ || true) if [ x"$storage_driver" = x"" ] \ || [ x"$storage_driver" = x"devicemapper" ]; then # With the TK1's old kernel the only way to run docker # is to use devicemapper storage driver with loopback # backend, which is unfit for production usage. # Every few months the loopback file gets corrupted and # docker can't start. # To solve this we go nuclear on docker. timeout 30s /usr/sbin/service docker stop || true rm -rf /var/lib/docker/ # If below hangs, then we'll just wait for the eventual # power-cycle. If docker still doesn't work from a clean # state, then we need to investigate manually. /usr/sbin/service docker stop || true fi /usr/sbin/service docker restart test_docker & if ! wait $!; then echo "ERROR: Cannot make docker work on the system" exit 1 fi fi fi if [ x"$keep_existing" != x"false" ]; then # We have docker-wrapper available, so use it to workaround dockerhub's # limits on pull requests. This is important for benchmarking boards, # which call tcwg-update-bmk-containers.sh for every build. /root/docker-wrapper maybepull "$image" else # We are asked to update the container unconditionally. # Make sure we will use latest image. docker pull "$image" fi else docker pull "$image" fi rm_cnt="" if docker stats --no-stream "$container" >/dev/null 2>&1; then running=$(docker container inspect -f "{{.State.Running}}" "$container") case "$running:$keep_existing" in true:true) exit 0 ;; true:keep_if_same_image) old_image=$(docker container inspect -f "{{.Image}}" "$container") new_image=$(docker image inspect -f "{{.Id}}" "$image") if [ x"$old_image" = x"$new_image" ]; then exit 0 fi ;; esac if $dryrun; then exit $EXTERNAL_FAIL fi # Rename the current container to free-up the name for "docker run" below. # Use rename name starting with a number (seconds since epoch) so that # it'll be cleaned up even if something goes wrong here. rm_cnt="$(date +%Y-%m-%d)-$container.bak" docker rename "$container" "$rm_cnt" & res=0 && wait $! || res=$? if [ x"$res" != x"0" ]; then # Failure to rename a container is usually caused by container # restarting loop. This restarting container can't be the current # one, so just delete it. docker stop "$container" || true if ! docker rm -v "$container"; then docker rm -vf "$container" fi rm_cnt="" fi fi if $dryrun; then exit $EXTERNAL_FAIL fi qemu_mount="" qemu_bin=$(mktemp -p $HOME) case "$(uname -m):$image" in x86_64:*-arm64-tcwg-llvmbot-*) # See dockerfiles.git/tcwg-base/tcwg-llvmbot/start.sh for details # on how this works. cp "$(which qemu-aarch64-static)" "$qemu_bin" chmod +x "$qemu_bin" qemu_mount="-v $qemu_bin:/bin/qemu-aarch64-static" ;; esac start_sh=$(mktemp) docker run --rm $qemu_mount $image start.sh > "$start_sh" bash "$start_sh" --verbose "$verbose" --additional_options "$additional_options" -- "$@" rm "$start_sh" "$qemu_bin" if [ x"$rm_cnt" != x"" ]; then # With the new container started delete the old one. # Note that if both old and new containers need an exclusive resource # (e.g., tcp port or connection to jenkins), then the new container might # need to restart a couple of times to wait for removal of the old one. # # We first try to gracefully shutdown the container docker stop "$rm_cnt" || true if ! docker rm -v "$rm_cnt"; then # ... and force SIGKILL only when necessary. docker rm -fv "$rm_cnt" fi fi