restore: parallelize memfd content restore via async daemon

memfds are restored as regular files, so their content can be filled in
parallel at the file level. Filling memfd data is the slowest part of
restore, and since the other restore steps do not depend on it, it can be
done asynchronously by a helper daemon.

Each restoring task creates its memfd and hands the descriptor to an
asynchronous daemon (asyncd) that fills the content (mmap + pread) from a
pool of worker threads. The daemon mirrors the usernsd design: tasks ship
work over a SEQPACKET socket via SCM_RIGHTS, and the content fill runs
out-of-band while the tasks keep restoring.

The daemon's worker threads must not hold a TID that the restorer later
needs. The restorer recreates each application thread at its original TID
via clone3(set_tid); if a daemon worker still occupies that TID the clone
fails with EEXIST ("Unable to create a thread: -17"). To guarantee the
daemon is gone before any application thread is cloned, a new restore
stage CR_STATE_PRE_RESTORER is introduced between CR_STATE_FORKING and
CR_STATE_RESTORE. The root task starts the daemon and switches all tasks
to PRE_RESTORER, during which the tasks ship their fills. The root task
then drains and reaps the daemon (stop_asyncd) before switching to
CR_STATE_RESTORE, where threads are cloned. Because the daemon is reaped
first, its worker TIDs are free by the time clone3(set_tid) runs.

Keeping the daemon inside the restored task tree (rather than forking it
from the coordinator) keeps the content fill charged to the restored
container's memory cgroup instead of to criu.

The fill-thread pool is sized to min(online_cpus, 16). stop_asyncd() fails
the restore if the daemon exits abnormally, so a failed content fill is
never silently lost, and it is idempotent so the restore paths can call it
unconditionally.

criu-image-streamer serves the image in a single sequential pass, which is
incompatible with the daemon's out-of-band reads, so the daemon is not
started when restoring from a stream and the content is filled inline
instead.

Co-developed-by: Dan Feigin <dfeigin@nvidia.com>
Signed-off-by: Dan Feigin <dfeigin@nvidia.com>
Signed-off-by: Andrei Vagin <avagin@google.com>

criu/Makefile.crtools

@@ -44,6 +44,7 @@ obj-y			+= mount.o
 obj-y			+= mount-v2.o
 obj-y			+= filesystems.o
 obj-y			+= namespaces.o
+obj-y			+= asyncd.o
 obj-y			+= netfilter.o
 obj-y			+= net.o
 obj-y			+= pagemap-cache.o

criu/asyncd.c

@@ -0,0 +1,211 @@
+#include <unistd.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <pthread.h>
+#include <sys/socket.h>
+#include <sys/syscall.h>
+#include <sched.h>
+
+#include "namespaces.h"
+#include "fdstore.h"
+#include "log.h"
+#include "util.h"
+#include "rst_info.h"
+#include "asyncd.h"
+#include "common/compiler.h"
+
+static void *asyncd_thread(void *arg)
+{
+	int sk = (long)arg;
+
+	while (1) {
+		struct unsc_msg um;
+		char msg[MAX_UNSFD_MSG_SIZE];
+		uns_call_t call;
+		int flags, fd, ret;
+		pid_t pid;
+
+		unsc_msg_init_nocreds(&um, &call, &flags, msg, sizeof(msg), 0);
+		ret = recvmsg(sk, &um.h, 0);
+		if (ret == 0)
+			break;
+		if (ret < 0) {
+			pr_perror("async: recv req error");
+			syscall(SYS_exit_group, 1);
+		}
+
+		unsc_msg_pid_fd(&um, &pid, &fd);
+		pr_debug("async: daemon calls %p (%d, %d, %x)\n", call, pid, fd, flags);
+
+		/*
+		 * Caller has sent us bare address of the routine it
+		 * wants to call. Since the caller is fork()-ed from the
+		 * same process as the daemon is, the latter has exactly
+		 * the same code at exactly the same address as the
+		 * former guy has. So go ahead and just call one!
+		 */
+
+		ret = call(msg, fd, pid);
+
+		if (fd >= 0)
+			close(fd);
+
+		if (ret < 0) {
+			pr_err("async: Async call failed. Exiting\n");
+			syscall(SYS_exit_group, 1);
+		}
+	}
+	return (void *)0;
+}
+
+/*
+ * The daemon is forked before the number of restore jobs is known, so it
+ * cannot size its fill pool from the job count. Size it to the number of
+ * available CPUs instead, capped at ASYNC_THREAD_NR_MAX. Content fill is the
+ * bottleneck, so matching the pool to the available cores is a good default.
+ */
+#define ASYNC_THREAD_NR_MAX 16
+static int get_avail_cpus(void)
+{
+	cpu_set_t cpuset;
+
+	CPU_ZERO(&cpuset);
+	if (sched_getaffinity(0, sizeof(cpuset), &cpuset) == 0)
+		return CPU_COUNT(&cpuset);
+
+	return sysconf(_SC_NPROCESSORS_ONLN);
+}
+
+static int asyncd(int sk)
+{
+	pthread_t threads[ASYNC_THREAD_NR_MAX];
+	int nr_threads = min_t(int, get_avail_cpus(), ASYNC_THREAD_NR_MAX);
+	int i;
+
+	if (nr_threads < 1)
+		nr_threads = 1;
+
+	for (i = 0; i < nr_threads; i++) {
+		if (pthread_create(&threads[i], NULL, asyncd_thread, (void *)(long)sk)) {
+			pr_perror("async: pthread_create");
+			return 1;
+		}
+	}
+
+	pr_info("async: Daemon started with %d threads\n", nr_threads);
+	for (i = 0; i < nr_threads; i++) {
+		if (pthread_join(threads[i], NULL)) {
+			pr_perror("async: pthread_join");
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+int __async_call(const char *func_name, uns_call_t call, int flags, void *arg, size_t arg_size, int fd)
+{
+	int ret, sk = -1;
+	struct unsc_msg um;
+
+	if (unlikely(arg_size > MAX_UNSFD_MSG_SIZE)) {
+		pr_err("async: message size exceeded\n");
+		return -1;
+	}
+
+	sk = fdstore_get(task_entries->asyncd_sk_id);
+	if (sk < 0) {
+		pr_err("async: cannot get ASYNCD_SK fd\n");
+		return -1;
+	}
+	pr_debug("async: calling %s (%d, %x)\n", func_name, fd, flags);
+
+	unsc_msg_init_nocreds(&um, &call, &flags, arg, arg_size, fd);
+	ret = sendmsg(sk, &um.h, 0);
+	if (ret <= 0) {
+		pr_perror("async: send req error");
+		ret = -1;
+		goto out;
+	}
+
+	ret = 0;
+out:
+	close_safe(&sk);
+	return ret;
+}
+
+static int asyncd_pid;
+int start_asyncd(void)
+{
+	int sk, id;
+
+	sk = start_unix_cred_daemon(&asyncd_pid, asyncd, false);
+	if (sk < 0) {
+		pr_err("failed to start asyncd\n");
+		return -1;
+	}
+
+	id = fdstore_add(sk);
+	close(sk);
+	if (id < 0) {
+		if (waitpid(asyncd_pid, NULL, 0) < 0)
+			pr_perror("async: waitpid");
+		asyncd_pid = 0;
+
+		return -1;
+	}
+
+	task_entries->asyncd_sk_id = id;
+
+	return 0;
+}
+
+int stop_asyncd(void)
+{
+	int exit_code = -1;
+	int sk = -1, status = -1;
+	sigset_t blockmask, oldmask;
+
+	/* No daemon was started (e.g. stream restore, or already stopped). */
+	if (asyncd_pid == 0)
+		return 0;
+
+	/*
+	 * Don't let the sigchld_handler() mess with us
+	 * calling waitpid() on the exited daemon. The
+	 * same is done in cr_system().
+	 */
+
+	sigemptyset(&blockmask);
+	sigaddset(&blockmask, SIGCHLD);
+	sigprocmask(SIG_BLOCK, &blockmask, &oldmask);
+
+	sk = fdstore_get(task_entries->asyncd_sk_id);
+	if (sk < 0) {
+		pr_err("async: cannot get ASYNCD_SK fd\n");
+		goto out;
+	}
+	if (shutdown(sk, SHUT_WR)) {
+		pr_perror("async: shutdown");
+		goto out;
+	}
+	if (waitpid(asyncd_pid, &status, 0) < 0) {
+		pr_perror("async: waitpid");
+		goto out;
+	}
+
+	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+		pr_err("async: daemon exited abnormally (status %#x)\n", status);
+		goto out;
+	}
+
+	pr_info("async: daemon stopped\n");
+	exit_code = 0;
+out:
+	close_safe(&sk);
+	asyncd_pid = 0;
+	sigprocmask(SIG_SETMASK, &oldmask, NULL);
+
+	return exit_code;
+}

criu/cgroup.c

@@ -2091,7 +2091,7 @@ static int prepare_cgroup_thread_sfd(void)
 {
 	int sk;
 
-	sk = start_unix_cred_daemon(&cgroupd_pid, cgroupd);
+	sk = start_unix_cred_daemon(&cgroupd_pid, cgroupd, true);
 	if (sk < 0) {
 		pr_err("failed to start cgroupd\n");
 		return -1;

criu/cr-restore.c

@@ -49,6 +49,7 @@
 #include "crtools.h"
 #include "uffd.h"
 #include "namespaces.h"
+#include "asyncd.h"
 #include "mem.h"
 #include "mount.h"
 #include "fsnotify.h"
@@ -143,6 +144,8 @@ static inline int stage_participants(int next_stage)
 		return 1;
 	case CR_STATE_FORKING:
 		return task_entries->nr_tasks + task_entries->nr_helpers;
+	case CR_STATE_PRE_RESTORER:
+		return task_entries->nr_tasks + task_entries->nr_helpers;
 	case CR_STATE_RESTORE:
 		return task_entries->nr_threads + task_entries->nr_helpers;
 	case CR_STATE_RESTORE_SIGCHLD:
@@ -158,16 +161,8 @@ static inline int stage_current_participants(int next_stage)
 {
 	switch (next_stage) {
 	case CR_STATE_FORKING:
+	case CR_STATE_PRE_RESTORER:
 		return 1;
-	case CR_STATE_RESTORE:
-		/*
-		 * Each thread has to be reported about this stage,
-		 * so if we want to wait all other tasks, we have to
-		 * exclude all threads of the current process.
-		 * It is supposed that we will wait other tasks,
-		 * before creating threads of the current task.
-		 */
-		return current->nr_threads;
 	}
 
 	BUG();
@@ -801,6 +796,7 @@ static int restore_one_zombie(CoreEntry *core)
 
 	prctl(PR_SET_NAME, (long)(void *)core->tc->comm, 0, 0, 0);
 
+	restore_finish_stage(task_entries, CR_STATE_PRE_RESTORER);
 	if (task_entries != NULL) {
 		wait_exiting_children();
 		zombie_prepare_signals();
@@ -978,6 +974,7 @@ static int restore_one_helper(void)
 	if (prepare_fds(current))
 		return -1;
 
+	restore_finish_stage(task_entries, CR_STATE_PRE_RESTORER);
 	if (wait_exiting_children())
 		return -1;
 
@@ -1670,8 +1667,6 @@ static int __restore_task_with_children(void *_arg)
 	if (populate_pid_proc())
 		goto err;
 
-	sfds_protected = true;
-
 	if (unmap_guard_pages(current))
 		goto err;
 
@@ -1694,18 +1689,32 @@ static int __restore_task_with_children(void *_arg)
 		if (restore_wait_other_tasks())
 			goto err;
 		fini_restore_mntns();
-		__restore_switch_stage(CR_STATE_RESTORE);
+
+		/* streamer serves each image once, one at a time; asyncd reads in parallel. */
+		if (!opts.stream && start_asyncd())
+			goto err;
+
+		__restore_switch_stage(CR_STATE_PRE_RESTORER);
 	} else {
 		if (restore_finish_stage(task_entries, CR_STATE_FORKING) < 0)
 			goto err;
 	}
 
+	sfds_protected = true;
+
 	if (restore_one_task(vpid(current), ca->core))
 		goto err;
 
 	return 0;
 
 err:
+	/*
+	 * Reap the async daemon before waking the coordinator: once the
+	 * abort is signalled the coordinator tears down the task tree and
+	 * may kill us mid-reap. stop_asyncd() is a no-op if no daemon was
+	 * started (asyncd_pid == 0).
+	 */
+	stop_asyncd();
 	if (current->parent == NULL)
 		futex_abort_and_wake(&task_entries->nr_in_progress);
 	exit(1);
@@ -2184,7 +2193,7 @@ skip_ns_bouncing:
 		goto out_kill;
 
 	/*
-	 * Zombies die after CR_STATE_RESTORE which is switched
+	 * Zombies die after CR_STATE_PRE_RESTORER which is switched
 	 * by root task, not by us. See comment before CR_STATE_FORKING
 	 * in the header for details.
 	 */
@@ -3241,6 +3250,12 @@ static int sigreturn_restore(pid_t pid, struct task_restore_args *task_args, uns
 			goto err_nv;
 		if (root_ns_mask & CLONE_NEWNS && remount_readonly_mounts())
 			goto err_nv;
+		if (stop_asyncd())
+			goto err_nv;
+		__restore_switch_stage(CR_STATE_RESTORE);
+	} else {
+		if (restore_finish_stage(task_entries, CR_STATE_PRE_RESTORER) < 0)
+			goto err;
 	}
 
 	/*
@@ -3603,6 +3618,8 @@ static int sigreturn_restore(pid_t pid, struct task_restore_args *task_args, uns
 err:
 	free_mappings(&self_vmas);
 err_nv:
+	/* Reap the async daemon if it is still running (no-op otherwise). */
+	stop_asyncd();
 	/* Just to be sure */
 	exit(1);
 	return -1;

criu/include/asyncd.h

@@ -0,0 +1,13 @@
+#ifndef __CR_ASYNCD_H__
+#define __CR_ASYNCD_H__
+
+#include "namespaces.h"
+
+extern int stop_asyncd(void);
+extern int start_asyncd(void);
+extern int __async_call(const char *func_name, uns_call_t call, int flags, void *arg, size_t arg_size, int fd);
+
+#define async_call(__call, __flags, __arg, __arg_size, __fd) \
+	__async_call(__stringify(__call), __call, __flags, __arg, __arg_size, __fd)
+
+#endif /* __CR_ASYNCD_H__ */

criu/include/namespaces.h

@@ -244,7 +244,8 @@ struct unsc_msg {
 };
 
 extern void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid);
+extern void unsc_msg_init_nocreds(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd);
 extern void unsc_msg_pid_fd(struct unsc_msg *um, pid_t *pid, int *fd);
-extern int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk));
+extern int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk), bool passcred);
 
 #endif /* __CR_NS_H__ */

criu/include/restorer.h

@@ -306,6 +306,7 @@ enum {
 	 * purely to make sure all tasks be in sync.
 	 */
 	CR_STATE_FORKING,
+	CR_STATE_PRE_RESTORER,
 	/*
 	 * Main restore stage. By the end of it all tasks are
 	 * almost ready and what's left is:

criu/include/rst_info.h

@@ -17,6 +17,7 @@ struct task_entries {
 	mutex_t userns_sync_lock;
 	mutex_t cgroupd_sync_lock;
 	mutex_t last_pid_mutex;
+	int asyncd_sk_id;
 };
 
 struct fdt {

criu/include/shmem.h

@@ -16,7 +16,14 @@ extern int fixup_sysv_shmems(void);
 extern int dump_one_memfd_shmem(int fd, unsigned long shmid, unsigned long size);
 extern int dump_one_sysv_shmem(void *addr, unsigned long size, unsigned long shmid);
 extern int restore_sysv_shmem_content(void *addr, unsigned long size, unsigned long shmid);
-extern int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size);
+extern int restore_shmem_fd_content(int fd, unsigned long shmid, unsigned long size);
+
+struct async_restore_shmem_args {
+	unsigned long shmid;
+	unsigned long size;
+};
+
+int async_restore_shmem_content(void *arg, int fd, pid_t pid);
 
 #define SYSV_SHMEM_SKIP_FD (0x7fffffff)
 

criu/memfd.c

@@ -3,6 +3,7 @@
 
 #include "common/compiler.h"
 #include "common/lock.h"
+#include "cr_options.h"
 #include "memfd.h"
 #include "fdinfo.h"
 #include "imgset.h"
@@ -17,6 +18,7 @@
 #include "fdstore.h"
 #include "file-ids.h"
 #include "namespaces.h"
+#include "asyncd.h"
 #include "shmem.h"
 #include "hugetlb.h"
 
@@ -258,7 +260,7 @@ static int memfd_open_inode_nocache(struct memfd_restore_inode *inode)
 {
 	MemfdInodeEntry *mie = NULL;
 	int fd = -1;
-	int ret = -1;
+	int ret, exit_code = -1;
 	int flags;
 
 	mie = inode->mie;
@@ -277,11 +279,34 @@ static int memfd_open_inode_nocache(struct memfd_restore_inode *inode)
 	fd = memfd_create(mie->name, flags);
 	if (fd < 0) {
 		pr_perror("Can't create memfd:%s", mie->name);
-		goto out;
+		goto err;
 	}
 
-	if (restore_memfd_shmem_content(fd, mie->shmid, mie->size))
-		goto out;
+	if (ftruncate(fd, mie->size) < 0) {
+		pr_perror("Can't resize shmem 0x%x size=%" PRIu64, mie->shmid, mie->size);
+		goto err;
+	}
+
+	if (opts.stream) {
+		/*
+		 * criu-image-streamer serves the image in a single sequential
+		 * pass and does not reopen it. The async fill daemon reads the
+		 * memfd content out-of-band from a separate process, which
+		 * breaks that contract, so fill inline when restoring from a
+		 * stream.
+		 */
+		if (restore_shmem_fd_content(fd, mie->shmid, mie->size))
+			goto err;
+	} else {
+		struct async_restore_shmem_args async_arg = {
+			.shmid = mie->shmid,
+			.size = mie->size,
+		};
+
+		if (async_call(async_restore_shmem_content, 0,
+			       &async_arg, sizeof(async_arg), fd))
+			goto err;
+	}
 
 	if (mie->has_mode)
 		ret = cr_fchperm(fd, mie->uid, mie->gid, mie->mode);
@@ -290,20 +315,19 @@ static int memfd_open_inode_nocache(struct memfd_restore_inode *inode)
 	if (ret) {
 		pr_perror("Can't set permissions { uid %d gid %d mode %#o } of memfd:%s", (int)mie->uid,
 			  (int)mie->gid, mie->has_mode ? (int)mie->mode : -1, mie->name);
-		goto out;
+		goto err;
 	}
 
 	inode->fdstore_id = fdstore_add(fd);
 	if (inode->fdstore_id < 0)
-		goto out;
+		goto err;
 
-	ret = fd;
+	exit_code = fd;
 	fd = -1;
 
-out:
-	if (fd != -1)
-		close(fd);
-	return ret;
+err:
+	close_safe(&fd);
+	return exit_code;
 }
 
 static int memfd_open_inode(struct memfd_restore_inode *inode)

criu/namespaces.c

@@ -1221,7 +1221,8 @@ static int write_id_map(pid_t pid, UidGidExtent **extents, int n, char *id_map)
 
 static int usernsd_pid;
 
-inline void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid)
+static void unsc_msg_init_flags(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid,
+				bool send_creds)
 {
 	struct cmsghdr *ch;
 	struct ucred *ucred;
@@ -1251,24 +1252,31 @@ inline void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg,
 	 */
 	memzero(&m->c, sizeof(m->c));
 
-	m->h.msg_controllen = CMSG_SPACE(sizeof(struct ucred));
+	m->h.msg_controllen = 0;
+	ch = NULL;
 
-	ch = CMSG_FIRSTHDR(&m->h);
-	ch->cmsg_len = CMSG_LEN(sizeof(struct ucred));
-	ch->cmsg_level = SOL_SOCKET;
-	ch->cmsg_type = SCM_CREDENTIALS;
+	if (send_creds) {
+		m->h.msg_controllen = CMSG_SPACE(sizeof(struct ucred));
+		ch = CMSG_FIRSTHDR(&m->h);
+		ch->cmsg_len = CMSG_LEN(sizeof(struct ucred));
+		ch->cmsg_level = SOL_SOCKET;
+		ch->cmsg_type = SCM_CREDENTIALS;
 
-	ucred = (struct ucred *)CMSG_DATA(ch);
-	if (pid)
-		ucred->pid = *pid;
-	else
-		ucred->pid = getpid();
-	ucred->uid = getuid();
-	ucred->gid = getgid();
+		ucred = (struct ucred *)CMSG_DATA(ch);
+		if (pid)
+			ucred->pid = *pid;
+		else
+			ucred->pid = getpid();
+		ucred->uid = getuid();
+		ucred->gid = getgid();
+	}
 
 	if (fd >= 0) {
 		m->h.msg_controllen += CMSG_SPACE(sizeof(int));
-		ch = CMSG_NXTHDR(&m->h, ch);
+		if (ch)
+			ch = CMSG_NXTHDR(&m->h, ch);
+		else
+			ch = CMSG_FIRSTHDR(&m->h);
 		BUG_ON(!ch);
 		ch->cmsg_len = CMSG_LEN(sizeof(int));
 		ch->cmsg_level = SOL_SOCKET;
@@ -1277,24 +1285,34 @@ inline void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg,
 	}
 }
 
+void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid)
+{
+	unsc_msg_init_flags(m, c, x, arg, asize, fd, pid, true);
+}
+
+void unsc_msg_init_nocreds(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd)
+{
+	unsc_msg_init_flags(m, c, x, arg, asize, fd, NULL, false);
+}
+
 void unsc_msg_pid_fd(struct unsc_msg *um, pid_t *pid, int *fd)
 {
 	struct cmsghdr *ch;
 	struct ucred *ucred;
 
 	ch = CMSG_FIRSTHDR(&um->h);
-	BUG_ON(!ch);
-	BUG_ON(ch->cmsg_len != CMSG_LEN(sizeof(struct ucred)));
-	BUG_ON(ch->cmsg_level != SOL_SOCKET);
-	BUG_ON(ch->cmsg_type != SCM_CREDENTIALS);
-
-	if (pid) {
-		ucred = (struct ucred *)CMSG_DATA(ch);
-		*pid = ucred->pid;
+	if (ch && ch->cmsg_level == SOL_SOCKET && ch->cmsg_type == SCM_CREDENTIALS) {
+		BUG_ON(ch->cmsg_len != CMSG_LEN(sizeof(struct ucred)));
+		if (pid) {
+			ucred = (struct ucred *)CMSG_DATA(ch);
+			*pid = ucred->pid;
+		}
+		ch = CMSG_NXTHDR(&um->h, ch);
+	} else {
+		if (pid)
+			*pid = -1;
 	}
 
-	ch = CMSG_NXTHDR(&um->h, ch);
-
 	if (ch && ch->cmsg_len == CMSG_LEN(sizeof(int))) {
 		BUG_ON(ch->cmsg_level != SOL_SOCKET);
 		BUG_ON(ch->cmsg_type != SCM_RIGHTS);
@@ -1447,7 +1465,7 @@ out:
 	return ret;
 }
 
-int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk))
+int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk), bool passcred)
 {
 	int sk[2];
 	int one = 1;
@@ -1470,14 +1488,20 @@ int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk))
 		return -1;
 	}
 
-	if (setsockopt(sk[0], SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) < 0) {
-		pr_perror("failed to setsockopt");
-		return -1;
-	}
+	if (passcred) {
+		if (setsockopt(sk[0], SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) < 0) {
+			pr_perror("failed to setsockopt");
+			close(sk[0]);
+			close(sk[1]);
+			return -1;
+		}
 
-	if (setsockopt(sk[1], SOL_SOCKET, SO_PASSCRED, &one, sizeof(1)) < 0) {
-		pr_perror("failed to setsockopt");
-		return -1;
+		if (setsockopt(sk[1], SOL_SOCKET, SO_PASSCRED, &one, sizeof(1)) < 0) {
+			pr_perror("failed to setsockopt");
+			close(sk[0]);
+			close(sk[1]);
+			return -1;
+		}
 	}
 
 	*pid = fork();
@@ -1506,7 +1530,7 @@ static int start_usernsd(void)
 	if (!(root_ns_mask & CLONE_NEWUSER))
 		return 0;
 
-	sk = start_unix_cred_daemon(&usernsd_pid, usernsd);
+	sk = start_unix_cred_daemon(&usernsd_pid, usernsd, true);
 	if (sk < 0) {
 		pr_err("failed to start usernsd\n");
 		return -1;

criu/pagemap.c

@@ -8,6 +8,7 @@
 #include <limits.h>
 
 #include "types.h"
+#include "atomic.h"
 #include "image.h"
 #include "cr_options.h"
 #include "servicefd.h"
@@ -857,7 +858,8 @@ int probe_pages_o_direct(int fd)
 int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int pr_flags)
 {
 	int flags, i_typ;
-	static unsigned ids = 1;
+	/* Shared across asyncd fill-daemon workers, which open page-reads concurrently. */
+	static atomic_t ids = { 0 };
 	bool remote = pr_flags & PR_REMOTE;
 
 	/*
@@ -943,7 +945,7 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p
 	pr->seek_pagemap = seek_pagemap;
 	pr->reset = reset_pagemap;
 	pr->io_complete = NULL; /* set up by the client if needed */
-	pr->id = ids++;
+	pr->id = atomic_inc_return(&ids);
 	pr->img_id = img_id;
 
 	if (remote)

criu/shmem.c

@@ -516,23 +516,18 @@ int restore_sysv_shmem_content(void *addr, unsigned long size, unsigned long shm
 	return do_restore_shmem_content(addr, round_up(size, PAGE_SIZE), shmid);
 }
 
-int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size)
+int restore_shmem_fd_content(int fd, unsigned long shmid, unsigned long size)
 {
-	void *addr = NULL;
-	int ret = 1;
+	void *addr = MAP_FAILED;
+	int exit_code = -1;
 
 	if (size == 0)
 		return 0;
 
-	if (ftruncate(fd, size) < 0) {
-		pr_perror("Can't resize shmem 0x%lx size=%ld", shmid, size);
-		goto out;
-	}
-
 	addr = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
 	if (addr == MAP_FAILED) {
-		pr_perror("Can't mmap shmem 0x%lx size=%ld", shmid, size);
-		goto out;
+		pr_perror("Can't mmap shmem 0x%lx size=%lu", shmid, size);
+		goto err;
 	}
 
 	/*
@@ -540,15 +535,24 @@ int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size)
 	 */
 	if (do_restore_shmem_content(addr, round_up(size, PAGE_SIZE), shmid) < 0) {
 		pr_err("Can't restore shmem content\n");
-		goto out;
+		goto err;
 	}
 
-	ret = 0;
-
-out:
-	if (addr)
+	exit_code = 0;
+err:
+	if (addr != MAP_FAILED)
 		munmap(addr, size);
-	return ret;
+	return exit_code;
+}
+
+int async_restore_shmem_content(void *arg, int fd, pid_t pid)
+{
+	struct async_restore_shmem_args *args = arg;
+
+	if (restore_shmem_fd_content(fd, args->shmid, args->size))
+		return -1;
+
+	return 0;
 }
 
 struct open_map_file_args {