start-container-qemu.sh: New script

Start QEMU KVM virtual machine inside docker container.
Use a copy of container's rootfs as VM's rootfs, which container
exports via NFS.  Likewise export any container's bind-mounts as
NFS shares to the VM.
VM is booted using provided initrd and kernel image.
E.g.,
./start-container-qemu.sh --session-name qemu-1 \
--label tcwg-apm_64-build --distro xenial \
--prefix qemu_ \
--kernel_url https://cloud-images.ubuntu.com/releases/16.04/release/unpacked/ubuntu-16.04-server-cloudimg-arm64-vmlinuz-generic \
--initrd_url https://cloud-images.ubuntu.com/releases/16.04/release/unpacked/ubuntu-16.04-server-cloudimg-arm64-initrd-generic \
> qemu-container.sh

Current motivation is to allow development and cross-testing of ILP32
toolchains, but there are other potential applications.

Use of KVM is not essential.  Non-KVM emulations work just as well,
though one needs a reasonably well-tested QEMU version (Xenial or later)
for ISA-level emulation.

Change-Id: I5ad0f19c5cd716cc2ae20420eada10105e74acff

1 files changed, 123 insertions, 0 deletions

diff --git a/start-container-qemu.sh b/start-container-qemu.sh
new file mode 100755
index 00000000..82a04a09
--- /dev/null
+++ b/start-container-qemu.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+set -ef -o pipefail
+
+# Save stdout/stderr file descriptors
+exec 3>&1 4>&2
+
+# Make sure all output goes to stderr
+exec 1>&2
+
+docker_args=()
+
+initrd_url=""
+kernel_url=""
+prefix=""
+task="build"
+weight="1"
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+	--initrd_url) initrd_url="$2" ;;
+	--kernel_url) kernel_url="$2" ;;
+	--prefix) prefix="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
+	--task) task="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
+	--weight) weight="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
+	*) docker_args=("${docker_args[@]}" "$1" "$2")
+    esac
+    shift 2
+done
+
+set -u
+
+container=$(mktemp)
+
+# Run container as privileged to mount host's /dev inside container to gain
+# access to /dev/kvm.
+# Publish container's port 2222, which is forwarded to VM's ssh server.
+$(dirname "$0")/start-container-docker.sh "${docker_args[@]}" --docker_opts "--privileged --publish 2222" > "$container"
+
+trap "rm $container; cleanup_all_containers" EXIT
+
+. "$container"
+
+eval "host=\$${prefix}container_host"
+eval "port=\$${prefix}container_port"
+eval "container_id=\$${prefix}container_id"
+
+# FIXME: install qemu-system-arm into docker images.
+${prefix}container_exec sudo apt-get update
+${prefix}container_exec sudo apt-get install -y qemu-system-arm nfs-kernel-server
+
+# /tmp is a scratch volume outside of / filesystem.
+nfsroot="/tmp/nfsroot"
+${prefix}container_exec sudo mkdir "$nfsroot"
+${prefix}container_exec sudo rsync -a --one-file-system / $nfsroot/
+
+nfsip=$(docker -H $host:2375 inspect --format {{.NetworkSettings.IPAddress}} "$container_id")
+
+# Start NFS server inside container.
+# Note, for NFS server to work inside container kernel should have
+# NFS module loaded, which seems easiest achieved by installing "nfs-kernel-server"
+# package on the host.
+bind_mounts=($(print_bind_mounts "$task"))
+echo "$nfsroot *(rw,async,no_root_squash,no_subtree_check,insecure)" | ${prefix}container_exec sudo tee -a /etc/exports
+for bind_mount in "${bind_mounts[@]}"; do
+    dir="${bind_mount##*:}"
+    mkdir -p "$nfsroot/$dir"
+
+    echo "$dir *(rw,async,no_root_squash,no_subtree_check,insecure)" | ${prefix}container_exec sudo tee -a /etc/exports
+    echo "$nfsip:$dir $dir nfs defaults,proto=udp 0 0" | ${prefix}container_exec sudo tee -a "$nfsroot/etc/fstab"
+done
+${prefix}container_exec sudo /etc/init.d/rpcbind start
+${prefix}container_exec sudo /etc/init.d/nfs-kernel-server start
+
+# Add tcwg-buildslave to "kvm" group to avoid running QEMU as root
+${prefix}container_exec sudo addgroup $USER kvm
+
+# CPU is limited (on contention) by docker container.
+ncpus=$(${prefix}container_exec nproc --all)
+memory=$(print_memory_limit "$task" "$weight")
+# Reduce memory limit for VM to leave something for QEMU itself.
+memory=$(($memory*2/3))
+
+wget_wildcard_url "$kernel_url"
+rsync -e "ssh -p$port" -az "$(ls $(basename "$kernel_url"))" "$host:/tmp/kernel"
+wget_wildcard_url "$initrd_url"
+rsync -e "ssh -p$port" -az "$(ls $(basename "$initrd_url"))" "$host:/tmp/initrd"
+
+# "-f" is to start qemu in the background.
+# "-Snone" is to avoid mixing this session with any other shared sessions.
+remote_exec "$host:$port:$(pwd):-f -Snone" \
+	    qemu-system-aarch64 -machine virt -enable-kvm -cpu host \
+	    -smp $ncpus -m $memory -nographic -monitor none \
+	    -kernel /tmp/kernel -initrd /tmp/initrd \
+	    -append "root=/dev/nfs nfsroot=$nfsip:$nfsroot,udp rw ip=dhcp" \
+	    -netdev user,id=unet,hostfwd=tcp::2222-:22 \
+	    -device virtio-net-device,netdev=unet \
+	    -s
+
+qemu_port=$(docker -H $host:2375 port $container_id 2222 | cut -d: -f 2)
+
+ret=0
+wait_for_ssh_server $host $qemu_port || ret=$?
+if [ x"$ret" != x"0" ]; then
+    echo "SSH server did not respond, exiting"
+    exit 1
+fi
+
+# Do not remove the container upon exit: it is now ready
+trap EXIT
+
+# Restore stdout/stderr
+exec 1>&3 2>&4
+
+cat "$container"
+cat <<EOF
+${prefix}container_port=$qemu_port
+${prefix}container_exec ()
+{
+  remote_exec "$host:$qemu_port:\$(pwd)" "\$@"
+}
+EOF
+rm "$container"