summaryrefslogtreecommitdiff
path: root/start-container-qemu.sh
blob: 6e01d9c1506aa1048d9a98a45847357a7c02acc7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
#!/bin/bash

set -ef -o pipefail

# Save stdout/stderr file descriptors
exec 3>&1 4>&2

# Make sure all output goes to stderr
exec 1>&2

docker_args=()

initrd_url=""
kernel_url=""
prefix=""
qemu="qemu-aarch64_host"
task="build"
weight="1"

while [ $# -gt 0 ]; do
    case "$1" in
	--initrd_url) initrd_url="$2" ;;
	--kernel_url) kernel_url="$2" ;;
	--prefix) prefix="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
	--qemu) qemu="$2" ;;
	--task) task="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
	--weight) weight="$2"; docker_args=("${docker_args[@]}" "$1" "$2") ;;
	*) docker_args=("${docker_args[@]}" "$1" "$2")
    esac
    shift 2
done

set -u

container=$(mktemp)

# Run container as privileged to mount host's /dev inside container to gain
# access to /dev/kvm.
# Publish container's port 2222, which is forwarded to VM's ssh server.
"$(dirname "$0")"/start-container-docker.sh "${docker_args[@]}" --docker_opts "--privileged --publish 2222" > "$container"

# shellcheck disable=SC2064
trap "rm $container; cleanup_all_containers" EXIT

# shellcheck disable=SC1090
. "$container"

declare host
eval "host=\$${prefix}container_host"
declare port
eval "port=\$${prefix}container_port"
declare container_id
eval "container_id=\$${prefix}container_id"
container_exec ()
{
    remote_exec "$host:$port:/" "$@"
}

# /tmp is a scratch volume outside of / filesystem.
nfsroot="/tmp/nfsroot"
container_exec sudo mkdir "$nfsroot"
container_exec sudo rsync -a --one-file-system / $nfsroot/

nfsip=$(ssh $host docker inspect --format "{{.NetworkSettings.IPAddress}}" "$container_id")

# Start NFS server inside container.
# Note, for NFS server to work inside container kernel should have
# NFS module loaded, which seems easiest achieved by installing "nfs-kernel-server"
# package on the host.
IFS=" " read -r -a bind_mounts <<< "$(print_bind_mounts "$task" "ssh $host")"
echo "$nfsroot *(rw,async,no_root_squash,no_subtree_check,insecure)" | container_exec sudo tee -a /etc/exports
test_array bind_mounts && for bind_mount in "${bind_mounts[@]}"; do
    dir="$(echo $bind_mount | cut -d: -f 1)"
    rw="$(echo $bind_mount | cut -s -d: -f 2)"
    rw="${rw:-rw}"
    mkdir -p "$nfsroot/$dir"

    echo "$dir *($rw,async,no_root_squash,no_subtree_check,insecure)" | container_exec sudo tee -a /etc/exports
    echo "$nfsip:$dir $dir nfs defaults,$rw,vers=3,proto=udp 0 0" | container_exec sudo tee -a "$nfsroot/etc/fstab"
done
container_exec sudo /etc/init.d/rpcbind start
container_exec sudo /etc/init.d/nfs-kernel-server start

# $qemu == qemu-ARCH_CPU
qemu_arch=$(echo "$qemu" | sed -e "s/^qemu-\(.*\)_\(.*\)/\1/")
qemu_cpu=$(echo "$qemu" | sed -e "s/^qemu-\(.*\)_\(.*\)/\2/")

kvm_opt=""
if [ x"$qemu_cpu" = x"host" ]; then
    # /dev/kvm is chowned to root:kvm when qemu-system-arm package is installed.
    # We have this packaged installed, but /dev/kvm owner and permissions are
    # not preserved in container image.
    container_exec sudo chown root:kvm /dev/kvm
    container_exec sudo chmod g+rw /dev/kvm
    kvm_opt="-enable-kvm"
fi

# CPU is limited (on contention) by docker container.
ncpus=$(container_exec nproc --all)
memory=$(print_memory_limit "$task" "$weight")
# Reduce memory limit for VM to leave something for QEMU itself.
memory=$(($memory*2/3))

wget_wildcard_url "$kernel_url"
rsync -e "ssh -p$port" -az "$(ls "$(basename "$kernel_url")")" "$host:/tmp/kernel"
wget_wildcard_url "$initrd_url"
rsync -e "ssh -p$port" -az "$(ls "$(basename "$initrd_url")")" "$host:/tmp/initrd"

# "-f" is to start qemu in the background.
# "-Snone" is to avoid mixing this session with any other shared sessions.
remote_exec "$host:$port:/:-f -Snone" \
	    qemu-system-$qemu_arch -machine virt $kvm_opt -cpu $qemu_cpu \
	    -smp $ncpus -m $memory -nographic -monitor none \
	    -kernel /tmp/kernel -initrd /tmp/initrd \
	    -append "root=/dev/nfs nfsroot=$nfsip:$nfsroot,udp rw ip=dhcp" \
	    -netdev user,id=unet,hostfwd=tcp::2222-:22 \
	    -device virtio-net-device,netdev=unet \
	    -s > ${prefix}qemu.log 2>&1

qemu_port=$(ssh $host docker port $container_id 2222 | cut -d: -f 2)

ret=0
wait_for_ssh_server $host $qemu_port || ret=$?
if [ $ret != 0 ]; then
    echo "SSH server did not respond, exiting"
    exit $ret
fi

# Do not remove the container upon exit: it is now ready
trap EXIT

# Restore stdout/stderr
exec 1>&3 2>&4

cat "$container"
cat <<EOF
${prefix}container_port=$qemu_port
${prefix}container_exec ()
{
  remote_exec "$host:$qemu_port:\$(pwd)" "\$@"
}
EOF
rm "$container"