1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#!/bin/bash
set -ef -o pipefail
# Save stdout/stderr file descriptors
exec 3>&1 4>&2
# Make sure all output goes to stderr
exec 1>&2
docker_args=()
initrd_url=""
kernel_url=""
prefix=""
task="build"
weight="1"
while [ $# -gt 0 ]; do
case "$1" in
--initrd_url) initrd_url="$2" ;;
--kernel_url) kernel_url="$2" ;;
--prefix) prefix="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
--task) task="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
--weight) weight="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
*) docker_args=("${docker_args[@]}" "$1" "$2")
esac
shift 2
done
container=$(mktemp)
# Run container as privileged to mount host's /dev inside container to gain
# access to /dev/kvm.
# Publish container's port 2222, which is forwarded to VM's ssh server.
$(dirname "$0")/start-container-docker.sh "${docker_args[@]}" --docker_opts "--privileged --publish 2222" > "$container"
trap "rm $container; cleanup_all_containers" EXIT
. "$container"
eval "host=\$${prefix}container_host"
eval "port=\$${prefix}container_port"
eval "container_id=\$${prefix}container_id"
container_exec ()
{
remote_exec "$host:$port:/" "$@"
}
# FIXME: install qemu-system-arm into docker images.
container_exec sudo apt-get update
container_exec sudo apt-get install -y qemu-system-arm nfs-kernel-server
# /tmp is a scratch volume outside of / filesystem.
nfsroot="/tmp/nfsroot"
container_exec sudo mkdir "$nfsroot"
container_exec sudo rsync -a --one-file-system / $nfsroot/
nfsip=$(docker -H $host:2375 inspect --format {{.NetworkSettings.IPAddress}} "$container_id")
# Start NFS server inside container.
# Note, for NFS server to work inside container kernel should have
# NFS module loaded, which seems easiest achieved by installing "nfs-kernel-server"
# package on the host.
bind_mounts=($(print_bind_mounts "$task"))
echo "$nfsroot *(rw,async,no_root_squash,no_subtree_check,insecure)" | container_exec sudo tee -a /etc/exports
for bind_mount in "${bind_mounts[@]}"; do
dir="$(echo $bind_mount | cut -s -d: -f 2)"
rw="$(echo $bind_mount | cut -s -d: -f 3)"
rw="${rw:-rw}"
mkdir -p "$nfsroot/$dir"
echo "$dir *($rw,async,no_root_squash,no_subtree_check,insecure)" | container_exec sudo tee -a /etc/exports
echo "$nfsip:$dir $dir nfs defaults,$rw,vers=3,proto=udp 0 0" | container_exec sudo tee -a "$nfsroot/etc/fstab"
done
container_exec sudo /etc/init.d/rpcbind start
container_exec sudo /etc/init.d/nfs-kernel-server start
# Add tcwg-buildslave to "kvm" group to avoid running QEMU as root
container_exec sudo addgroup $USER kvm
# CPU is limited (on contention) by docker container.
ncpus=$(container_exec nproc --all)
memory=$(print_memory_limit "$task" "$weight")
# Reduce memory limit for VM to leave something for QEMU itself.
memory=$(($memory*2/3))
wget_wildcard_url "$kernel_url"
rsync -e "ssh -p$port" -az "$(ls $(basename "$kernel_url"))" "$host:/tmp/kernel"
wget_wildcard_url "$initrd_url"
rsync -e "ssh -p$port" -az "$(ls $(basename "$initrd_url"))" "$host:/tmp/initrd"
# "-f" is to start qemu in the background.
# "-Snone" is to avoid mixing this session with any other shared sessions.
remote_exec "$host:$port:/:-f -Snone" \
qemu-system-aarch64 -machine virt -enable-kvm -cpu host \
-smp $ncpus -m $memory -nographic -monitor none \
-kernel /tmp/kernel -initrd /tmp/initrd \
-append "root=/dev/nfs nfsroot=$nfsip:$nfsroot,udp rw ip=dhcp" \
-netdev user,id=unet,hostfwd=tcp::2222-:22 \
-device virtio-net-device,netdev=unet \
-s > ${prefix}qemu.log 2>&1
qemu_port=$(docker -H $host:2375 port $container_id 2222 | cut -d: -f 2)
ret=0
wait_for_ssh_server $host $qemu_port || ret=$?
if [ x"$ret" != x"0" ]; then
echo "SSH server did not respond, exiting"
exit 1
fi
# Do not remove the container upon exit: it is now ready
trap EXIT
# Restore stdout/stderr
exec 1>&3 2>&4
cat "$container"
cat <<EOF
${prefix}container_port=$qemu_port
${prefix}container_exec ()
{
remote_exec "$host:$qemu_port:\$(pwd)" "\$@"
}
EOF
rm "$container"
|
