tests: add test for pivot_root in initramfs support

The only way to run in the proper initramfs is to start a VM using a
custom initrd that runs runc. This should be a fairly reasonable
smoke-test that matches what minikube and kata do.

Unfortunately, running the right qemu for the native architecture on
various distros is a little different, so we need a helper function to
get it to work on both Debian and AlmaLinux.

Signed-off-by: Aleksa Sarai <[email protected]>

Author: cyphar

.cirrus.yml

@@ -79,7 +79,7 @@ task:
     CIRRUS_WORKING_DIR: /home/runc
     GO_VERSION: "1.23"
     BATS_VERSION: "v1.9.0"
-    RPMS: gcc git iptables jq glibc-static libseccomp-devel make criu fuse-sshfs container-selinux
+    RPMS: gcc git iptables jq qemu-kvm glibc-static libseccomp-devel make criu fuse-sshfs container-selinux
     # yamllint disable rule:key-duplicates
     matrix:
       DISTRO: almalinux-8

.github/workflows/test.yml

@@ -120,7 +120,7 @@ jobs:
     - name: install deps
       run: |
         sudo apt update
-        sudo apt -y install libseccomp-dev sshfs uidmap
+        sudo apt -y install cpio libseccomp-dev qemu-kvm sshfs uidmap
 
     - name: install CRIU
       if: ${{ matrix.criu == '' }}

Dockerfile

@@ -16,6 +16,7 @@ RUN KEYFILE=/usr/share/keyrings/criu-repo-keyring.gpg; \
         criu \
         gcc \
         gcc-multilib \
+        cpio \
         curl \
         gawk \
         gperf \
@@ -24,6 +25,7 @@ RUN KEYFILE=/usr/share/keyrings/criu-repo-keyring.gpg; \
         kmod \
         pkg-config \
         python3-minimal \
+        qemu-kvm \
         sshfs \
         sudo \
         uidmap \

Makefile

@@ -172,6 +172,7 @@ localunittest: all
 integration: runcimage
 	$(CONTAINER_ENGINE) run $(CONTAINER_ENGINE_RUN_FLAGS) \
 		-t --privileged --rm \
+		-v /boot:/boot:ro \
 		-v /lib/modules:/lib/modules:ro \
 		-v $(CURDIR):/go/src/$(PROJECT) \
 		$(RUNC_IMAGE) make localintegration TESTPATH="$(TESTPATH)"

tests/integration/helpers.bash

@@ -13,6 +13,7 @@ eval "$IMAGES"
 unset IMAGES
 
 : "${RUNC:="${INTEGRATION_ROOT}/../../runc"}"
+: "${RUNC_STATIC:="${INTEGRATION_ROOT}/../../runc.static"}"
 RECVTTY="${INTEGRATION_ROOT}/../../tests/cmd/recvtty/recvtty"
 SD_HELPER="${INTEGRATION_ROOT}/../../tests/cmd/sd-helper/sd-helper"
 SECCOMP_AGENT="${INTEGRATION_ROOT}/../../tests/cmd/seccompagent/seccompagent"
@@ -39,18 +40,25 @@ ARCH=$(uname -m)
 # Seccomp agent socket.
 SECCCOMP_AGENT_SOCKET="$BATS_TMPDIR/seccomp-agent.sock"
 
-# Wrapper for runc.
-function runc() {
-	run __runc "$@"
+function sane_run() {
+	local cmd="$1"
+	shift
+
+	run "$cmd" "$@"
 
 	# Some debug information to make life easier. bats will only print it if the
 	# test failed, in which case the output is useful.
 	# shellcheck disable=SC2154
-	echo "$(basename "$RUNC") $* (status=$status):" >&2
+	echo "$cmd $* (status=$status):" >&2
 	# shellcheck disable=SC2154
 	echo "$output" >&2
 }
 
+# Wrapper for runc.
+function runc() {
+	sane_run __runc "$@"
+}
+
 # Raw wrapper for runc.
 function __runc() {
 	"$RUNC" ${RUNC_USE_SYSTEMD+--systemd-cgroup} \

tests/integration/initramfs.bats

@@ -0,0 +1,151 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+	INITRAMFS_ROOT="$(mktemp -d "$BATS_RUN_TMPDIR/runc-initramfs.XXXXXX")"
+}
+
+function teardown() {
+	[ -v INITRAMFS_ROOT ] && rm -rf "$INITRAMFS_ROOT"
+}
+
+function find_vmlinuz() {
+	shopt -s nullglob
+	candidates=(
+		/boot/vmlinuz
+		/boot/vmlinuz-$(uname -r)*
+		/usr/lib*/modules/$(uname -r)/vmlinu* # vmlinuz / vmlinux.{gz,xz,*}
+	)
+	shopt -u nullglob
+
+	for candidate in "${candidates[@]}"; do
+		[ -e "$candidate" ] || continue
+		echo "$candidate"
+		return 0
+	done
+
+	skip "could not find host vmlinuz kernel"
+}
+
+function qemu_native() {
+	# Different distributions put qemu-kvm in different locations and with
+	# different names. Debian and Ubuntu have a "kvm" binary, while AlmaLinux
+	# has /usr/libexec/qemu-kvm.
+	qemu_candidates=("kvm" "qemu-kvm" "/usr/libexec/qemu-kvm")
+
+	qemu_binary=
+	for candidate in "${qemu_candidates[@]}"; do
+		"$candidate" -help &>/dev/null || continue
+		qemu_binary="$candidate"
+		break
+	done
+	# TODO: Maybe we should also try to call qemu-system-FOO for the current
+	# architecture if qemu-kvm is missing?
+	[ -n "$qemu_binary" ] || skip "could not find qemu-kvm binary"
+
+	machine_args=()
+	case "$(go env GOARCH)" in
+	386 | amd64)
+		machine_args=("-machine" "pc")
+		;;
+	arm | arm64)
+		machine_args=("-machine" "virt")
+		;;
+	*)
+		echo "could not figure out -machine argument for qemu -- using default" >&2
+		;;
+	esac
+
+	sane_run "$qemu_binary" "${machine_args[@]}" "$@"
+	if [ "$status" -ne 0 ]; then
+		# To help with debugging, output the set of valid machine values.
+		"$qemu_binary" -machine help >&2
+	fi
+}
+
+@test "runc run [initramfs + pivot_root]" {
+	requires root
+
+	KERNEL="$(find_vmlinuz)"
+
+	# Configure our minimal initrd.
+	mkdir -p "$INITRAMFS_ROOT/initrd"
+	pushd "$INITRAMFS_ROOT/initrd"
+
+	# Use busybox as a base for our initrd.
+	tar --exclude './dev/*' -xf "$BUSYBOX_IMAGE"
+	# Make sure that "sh" and "poweroff" are installed, otherwise qemu will
+	# boot loop when init stops.
+	[ -x ./bin/sh ] || skip "busybox image is missing /bin/sh"
+	[ -x ./bin/poweroff ] || skip "busybox image is missing /bin/poweroff"
+
+	# Copy the runc binary into the container. In theory we would prefer to
+	# copy a static binary, but some distros (like openSUSE) don't ship
+	# libseccomp-static so requiring a static build for any integration test
+	# run wouldn't work. Instead, we copy all of the library dependencies into
+	# the rootfs (note that we also have to copy ld-linux-*.so because runc was
+	# probably built with a newer glibc than the one in our busybox image.
+	cp "$RUNC" ./bin/runc
+	readarray -t runclibs \
+		<<<"$(ldd "$RUNC" | grep -Eo '/[^ ]*lib[^ ]*.so.[^ ]*')"
+	mkdir -p ./lib{,64}
+	cp -vt ./lib/ "${runclibs[@]}"
+	cp -vt ./lib64/ "${runclibs[@]}"
+
+	# Create a container bundle using the same busybox image.
+	mkdir -p ./run/bundle
+	pushd ./run/bundle
+	mkdir -p rootfs
+	tar --exclude './dev/*' -C rootfs -xf "$BUSYBOX_IMAGE"
+	runc spec
+	update_config '.process.args = ["/bin/echo", "hello from inside the container"]'
+	popd
+
+	# Build a custom /init script.
+	cat >./init <<-EOF
+		#!/bin/sh
+
+		set -x
+		echo "==START INIT SCRIPT=="
+
+		mkdir -p /proc /sys
+		mount -t proc proc /proc
+		mkdir -p /sys
+		mount -t sysfs sysfs /sys
+
+		mkdir -p /sys/fs/cgroup
+		mount -t cgroup2 cgroup2 /sys/fs/cgroup
+
+		mkdir -p /tmp
+		mount -t tmpfs tmpfs /tmp
+
+		mkdir -p /dev
+		mount -t devtmpfs devtmpfs /dev
+		mkdir -p /dev/pts
+		mount -t devpts -o newinstance devpts /dev/pts
+		mkdir -p /dev/shm
+		mount --bind /tmp /dev/shm
+
+		runc run -b /run/bundle ctr
+
+		echo "==END INIT SCRIPT=="
+		poweroff -f
+	EOF
+	chmod +x ./init
+
+	find . | cpio -o -H newc >"$INITRAMFS_ROOT/initrd.cpio"
+	popd
+
+	# Now we can just run the image (use qemu-kvm so that we run on the same
+	# architecture as the host system). We can just reuse the host kernel.
+	qemu_native \
+		-initrd "$INITRAMFS_ROOT/initrd.cpio" \
+		-kernel "$KERNEL" \
+		-m 512M \
+		-nographic -append console=ttyS0
+	[ "$status" -eq 0 ]
+	[[ "$output" = *"==START INIT SCRIPT=="* ]]
+	[[ "$output" = *"hello from inside the container"* ]]
+	[[ "$output" = *"==END INIT SCRIPT=="* ]]
+}