libct/nsenter: namespace the bindfd shuffle

Processes can watch /proc/self/mounts or /mountinfo, and the kernel
will notify them whenever the namespace's mount table is modified. The
notified process still needs to read and parse the mountinfo to
determine what changed once notified. Many such processes, including
udisksd and SystemD < v248, make no attempt to rate-limit their
mountinfo notifications. This tends to not be a problem on many systems,
where mount tables are small and mounting and unmounting is uncommon.
Every runC exec which successfully uses the try_bindfd container-escape
mitigation performs two mount()s and one umount() in the host's mount
namespace, causing any mount-watching processes to wake up and parse the
mountinfo file three times in a row. Consequently, using 'exec' health
checks on containers has a larger-than-expected impact on system load
when such mount-watching daemons are running. Furthermore, the size of
the mount table in the host's mount namespace tends to be proportional
to the number of OCI containers as a unique mount is required for the
rootfs of each container. Therefore, on systems with mount-watching
processes, the system load increases *quadratically* with the number of
running containers which use health checks!

Prevent runC from incidentally modifying the host's mount namespace for
container-escape mitigations by setting up the mitigation in a private
mount namespace.

Signed-off-by: Cory Snider <[email protected]>

Author: corhere

libcontainer/nsenter/cloned_binary.c

@@ -49,14 +49,20 @@
 #include <fcntl.h>
 #include <errno.h>
 
+#include <sched.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/statfs.h>
 #include <sys/vfs.h>
 #include <sys/mman.h>
 #include <sys/mount.h>
 #include <sys/sendfile.h>
+#include <sys/socket.h>
 #include <sys/syscall.h>
+#include <sys/wait.h>
+
+#include "clone.h"
+#include "ipc.h"
 
 /* Use our own wrapper for memfd_create. */
 #ifndef SYS_memfd_create
@@ -399,6 +405,8 @@ static int try_bindfd(void)
 	int fd, ret = -1;
 	char template[PATH_MAX] = { 0 };
 	char *prefix = getenv("_LIBCONTAINER_STATEDIR");
+	jmp_buf env;
+	int pipe[2], is_child;
 
 	if (!prefix || *prefix != '/')
 		prefix = "/tmp";
@@ -415,29 +423,87 @@ static int try_bindfd(void)
 	close(fd);
 
 	/*
-	 * For obvious reasons this won't work in rootless mode because we haven't
-	 * created a userns+mntns -- but getting that to work will be a bit
-	 * complicated and it's only worth doing if someone actually needs it.
+	 * Daemons such as systemd and udisks2 watch /proc/self/mountinfo and
+	 * re-parse it on every change, which gets expensive when the mount table
+	 * is large and/or changes frequently. Set up the bind-mounts in a new,
+	 * private mount namespace so as not to wake up those processes every
+	 * time we nsexec into a container. We clone a child process into a new
+	 * mount namespace to do the dirty work so the side effects of unsharing
+	 * the mount namespace do not leak into the current process.
 	 */
-	ret = -EPERM;
-	if (mount("/proc/self/exe", template, "", MS_BIND, "") < 0)
+	if ((ret = socketpair(AF_LOCAL, SOCK_STREAM, 0, pipe)) < 0)
 		goto out;
-	if (mount("", template, "", MS_REMOUNT | MS_BIND | MS_RDONLY, "") < 0)
-		goto out_umount;
 
-	/* Get read-only handle that we're sure can't be made read-write. */
-	ret = open(template, O_PATH | O_CLOEXEC);
+	is_child = setjmp(env);
+	if (is_child) {
+		close(pipe[0]);
+		fd = -1;
+		/*
+		 * For obvious reasons this won't work in rootless mode because we haven't
+		 * created a userns -- but getting that to work will be a bit complicated and
+		 * it's only worth doing if someone actually needs it.
+		 */
+		if ((ret = mount("none", "/", NULL, MS_PRIVATE | MS_REC, NULL)) < 0)
+			goto out_child;
+		/*
+		 * The kernel refuses to bind-mount from the magic symlink when the process has
+		 * been cloned (or unshared) into a new mount namespace, at least on Linux 4.4.
+		 */
+		char linkbuf[PATH_MAX + 1] = { 0 };
+		ssize_t linkpathlen = readlink("/proc/self/exe", linkbuf, sizeof(linkbuf));
+		if (linkpathlen == sizeof(linkbuf)) {
+			/*
+			 * The link path is longer than PATH_MAX, and the contents of
+			 * linkbuf might have been truncated. A truncated path could
+			 * happen to be a valid path to a different file, which could
+			 * allow for local privilege escalation if we were to exec it.
+			 * The mount syscall doesn't accept paths longer than PATH_MAX,
+			 * anyway.
+			 */
+			ret = -ENAMETOOLONG;
+			goto out_child;
+		}
+		linkbuf[linkpathlen] = '\0';
+		if ((ret = mount(linkbuf, template, "", MS_BIND, "")) < 0)
+			goto out_child;
+		if ((ret = mount("", template, "", MS_REMOUNT | MS_BIND | MS_RDONLY, "")) < 0)
+			goto out_child;
+
+		/* Get read-only handle that we're sure can't be made read-write. */
+		fd = open(template, O_PATH | O_CLOEXEC);
+		if (fd < 0) {
+			ret = fd;
+			goto out_child;
+		}
 
-out_umount:
-	/*
-	 * Make sure the MNT_DETACH works, otherwise we could get remounted
-	 * read-write and that would be quite bad (the fd would be made read-write
-	 * too, invalidating the protection).
-	 */
-	if (umount2(template, MNT_DETACH) < 0) {
-		if (ret >= 0)
-			close(ret);
-		ret = -ENOTRECOVERABLE;
+		/*
+		 * Make sure the MNT_DETACH works, otherwise we could get remounted
+		 * read-write and that would be quite bad (the fd would be made read-write
+		 * too, invalidating the protection).
+		 */
+		if (umount2(template, MNT_DETACH) < 0) {
+			ret = -ENOTRECOVERABLE;
+			goto out_child;
+		}
+		send_fd(pipe[1], fd);
+out_child:
+		if (fd >= 0)
+			close(fd);
+		close(pipe[1]);
+		exit(-ret);
+	} else {
+		int cpid = clone_longjmp(&env, 1, CLONE_NEWNS | SIGCHLD);
+		if (cpid < 0)
+			goto out;
+		int wstatus = 0;
+		if ((ret = waitpid(cpid, &wstatus, 0)) < 0)
+			goto out;
+		if ((ret = -WEXITSTATUS(wstatus)))
+			goto out;
+		fd = receive_fd(pipe[0]);
+		if ((ret = fcntl(ret, F_SETFD, O_PATH | O_CLOEXEC)) < 0)
+			goto out;
+		ret = fd;
 	}
 
 out:
@@ -446,6 +512,8 @@ static int try_bindfd(void)
 	 * there's an empty file left around in STATEDIR.
 	 */
 	unlink(template);
+	close(pipe[0]);
+	close(pipe[1]);
 	return ret;
 }