diff --git a/criu/Makefile.crtools b/criu/Makefile.crtools index ba6132d2f..1a652f19b 100644 --- a/criu/Makefile.crtools +++ b/criu/Makefile.crtools @@ -44,6 +44,7 @@ obj-y += mount.o obj-y += mount-v2.o obj-y += filesystems.o obj-y += namespaces.o +obj-y += asyncd.o obj-y += netfilter.o obj-y += net.o obj-y += pagemap-cache.o diff --git a/criu/asyncd.c b/criu/asyncd.c new file mode 100644 index 000000000..8d3689776 --- /dev/null +++ b/criu/asyncd.c @@ -0,0 +1,211 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "namespaces.h" +#include "fdstore.h" +#include "log.h" +#include "util.h" +#include "rst_info.h" +#include "asyncd.h" +#include "common/compiler.h" + +static void *asyncd_thread(void *arg) +{ + int sk = (long)arg; + + while (1) { + struct unsc_msg um; + char msg[MAX_UNSFD_MSG_SIZE]; + uns_call_t call; + int flags, fd, ret; + pid_t pid; + + unsc_msg_init_nocreds(&um, &call, &flags, msg, sizeof(msg), 0); + ret = recvmsg(sk, &um.h, 0); + if (ret == 0) + break; + if (ret < 0) { + pr_perror("async: recv req error"); + syscall(SYS_exit_group, 1); + } + + unsc_msg_pid_fd(&um, &pid, &fd); + pr_debug("async: daemon calls %p (%d, %d, %x)\n", call, pid, fd, flags); + + /* + * Caller has sent us bare address of the routine it + * wants to call. Since the caller is fork()-ed from the + * same process as the daemon is, the latter has exactly + * the same code at exactly the same address as the + * former guy has. So go ahead and just call one! + */ + + ret = call(msg, fd, pid); + + if (fd >= 0) + close(fd); + + if (ret < 0) { + pr_err("async: Async call failed. Exiting\n"); + syscall(SYS_exit_group, 1); + } + } + return (void *)0; +} + +/* + * The daemon is forked before the number of restore jobs is known, so it + * cannot size its fill pool from the job count. Size it to the number of + * available CPUs instead, capped at ASYNC_THREAD_NR_MAX. Content fill is the + * bottleneck, so matching the pool to the available cores is a good default. + */ +#define ASYNC_THREAD_NR_MAX 16 +static int get_avail_cpus(void) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + if (sched_getaffinity(0, sizeof(cpuset), &cpuset) == 0) + return CPU_COUNT(&cpuset); + + return sysconf(_SC_NPROCESSORS_ONLN); +} + +static int asyncd(int sk) +{ + pthread_t threads[ASYNC_THREAD_NR_MAX]; + int nr_threads = min_t(int, get_avail_cpus(), ASYNC_THREAD_NR_MAX); + int i; + + if (nr_threads < 1) + nr_threads = 1; + + for (i = 0; i < nr_threads; i++) { + if (pthread_create(&threads[i], NULL, asyncd_thread, (void *)(long)sk)) { + pr_perror("async: pthread_create"); + return 1; + } + } + + pr_info("async: Daemon started with %d threads\n", nr_threads); + for (i = 0; i < nr_threads; i++) { + if (pthread_join(threads[i], NULL)) { + pr_perror("async: pthread_join"); + return 1; + } + } + + return 0; +} + +int __async_call(const char *func_name, uns_call_t call, int flags, void *arg, size_t arg_size, int fd) +{ + int ret, sk = -1; + struct unsc_msg um; + + if (unlikely(arg_size > MAX_UNSFD_MSG_SIZE)) { + pr_err("async: message size exceeded\n"); + return -1; + } + + sk = fdstore_get(task_entries->asyncd_sk_id); + if (sk < 0) { + pr_err("async: cannot get ASYNCD_SK fd\n"); + return -1; + } + pr_debug("async: calling %s (%d, %x)\n", func_name, fd, flags); + + unsc_msg_init_nocreds(&um, &call, &flags, arg, arg_size, fd); + ret = sendmsg(sk, &um.h, 0); + if (ret <= 0) { + pr_perror("async: send req error"); + ret = -1; + goto out; + } + + ret = 0; +out: + close_safe(&sk); + return ret; +} + +static int asyncd_pid; +int start_asyncd(void) +{ + int sk, id; + + sk = start_unix_cred_daemon(&asyncd_pid, asyncd, false); + if (sk < 0) { + pr_err("failed to start asyncd\n"); + return -1; + } + + id = fdstore_add(sk); + close(sk); + if (id < 0) { + if (waitpid(asyncd_pid, NULL, 0) < 0) + pr_perror("async: waitpid"); + asyncd_pid = 0; + + return -1; + } + + task_entries->asyncd_sk_id = id; + + return 0; +} + +int stop_asyncd(void) +{ + int exit_code = -1; + int sk = -1, status = -1; + sigset_t blockmask, oldmask; + + /* No daemon was started (e.g. stream restore, or already stopped). */ + if (asyncd_pid == 0) + return 0; + + /* + * Don't let the sigchld_handler() mess with us + * calling waitpid() on the exited daemon. The + * same is done in cr_system(). + */ + + sigemptyset(&blockmask); + sigaddset(&blockmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &blockmask, &oldmask); + + sk = fdstore_get(task_entries->asyncd_sk_id); + if (sk < 0) { + pr_err("async: cannot get ASYNCD_SK fd\n"); + goto out; + } + if (shutdown(sk, SHUT_WR)) { + pr_perror("async: shutdown"); + goto out; + } + if (waitpid(asyncd_pid, &status, 0) < 0) { + pr_perror("async: waitpid"); + goto out; + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + pr_err("async: daemon exited abnormally (status %#x)\n", status); + goto out; + } + + pr_info("async: daemon stopped\n"); + exit_code = 0; +out: + close_safe(&sk); + asyncd_pid = 0; + sigprocmask(SIG_SETMASK, &oldmask, NULL); + + return exit_code; +} diff --git a/criu/cgroup.c b/criu/cgroup.c index 9246be639..d1f65df34 100644 --- a/criu/cgroup.c +++ b/criu/cgroup.c @@ -2091,7 +2091,7 @@ static int prepare_cgroup_thread_sfd(void) { int sk; - sk = start_unix_cred_daemon(&cgroupd_pid, cgroupd); + sk = start_unix_cred_daemon(&cgroupd_pid, cgroupd, true); if (sk < 0) { pr_err("failed to start cgroupd\n"); return -1; diff --git a/criu/cr-restore.c b/criu/cr-restore.c index 072a72951..6f3666d8c 100644 --- a/criu/cr-restore.c +++ b/criu/cr-restore.c @@ -49,6 +49,7 @@ #include "crtools.h" #include "uffd.h" #include "namespaces.h" +#include "asyncd.h" #include "mem.h" #include "mount.h" #include "fsnotify.h" @@ -143,6 +144,8 @@ static inline int stage_participants(int next_stage) return 1; case CR_STATE_FORKING: return task_entries->nr_tasks + task_entries->nr_helpers; + case CR_STATE_PRE_RESTORER: + return task_entries->nr_tasks + task_entries->nr_helpers; case CR_STATE_RESTORE: return task_entries->nr_threads + task_entries->nr_helpers; case CR_STATE_RESTORE_SIGCHLD: @@ -158,16 +161,8 @@ static inline int stage_current_participants(int next_stage) { switch (next_stage) { case CR_STATE_FORKING: + case CR_STATE_PRE_RESTORER: return 1; - case CR_STATE_RESTORE: - /* - * Each thread has to be reported about this stage, - * so if we want to wait all other tasks, we have to - * exclude all threads of the current process. - * It is supposed that we will wait other tasks, - * before creating threads of the current task. - */ - return current->nr_threads; } BUG(); @@ -801,6 +796,7 @@ static int restore_one_zombie(CoreEntry *core) prctl(PR_SET_NAME, (long)(void *)core->tc->comm, 0, 0, 0); + restore_finish_stage(task_entries, CR_STATE_PRE_RESTORER); if (task_entries != NULL) { wait_exiting_children(); zombie_prepare_signals(); @@ -978,6 +974,7 @@ static int restore_one_helper(void) if (prepare_fds(current)) return -1; + restore_finish_stage(task_entries, CR_STATE_PRE_RESTORER); if (wait_exiting_children()) return -1; @@ -1670,8 +1667,6 @@ static int __restore_task_with_children(void *_arg) if (populate_pid_proc()) goto err; - sfds_protected = true; - if (unmap_guard_pages(current)) goto err; @@ -1694,18 +1689,32 @@ static int __restore_task_with_children(void *_arg) if (restore_wait_other_tasks()) goto err; fini_restore_mntns(); - __restore_switch_stage(CR_STATE_RESTORE); + + /* streamer serves each image once, one at a time; asyncd reads in parallel. */ + if (!opts.stream && start_asyncd()) + goto err; + + __restore_switch_stage(CR_STATE_PRE_RESTORER); } else { if (restore_finish_stage(task_entries, CR_STATE_FORKING) < 0) goto err; } + sfds_protected = true; + if (restore_one_task(vpid(current), ca->core)) goto err; return 0; err: + /* + * Reap the async daemon before waking the coordinator: once the + * abort is signalled the coordinator tears down the task tree and + * may kill us mid-reap. stop_asyncd() is a no-op if no daemon was + * started (asyncd_pid == 0). + */ + stop_asyncd(); if (current->parent == NULL) futex_abort_and_wake(&task_entries->nr_in_progress); exit(1); @@ -2184,7 +2193,7 @@ skip_ns_bouncing: goto out_kill; /* - * Zombies die after CR_STATE_RESTORE which is switched + * Zombies die after CR_STATE_PRE_RESTORER which is switched * by root task, not by us. See comment before CR_STATE_FORKING * in the header for details. */ @@ -3241,6 +3250,12 @@ static int sigreturn_restore(pid_t pid, struct task_restore_args *task_args, uns goto err_nv; if (root_ns_mask & CLONE_NEWNS && remount_readonly_mounts()) goto err_nv; + if (stop_asyncd()) + goto err_nv; + __restore_switch_stage(CR_STATE_RESTORE); + } else { + if (restore_finish_stage(task_entries, CR_STATE_PRE_RESTORER) < 0) + goto err; } /* @@ -3603,6 +3618,8 @@ static int sigreturn_restore(pid_t pid, struct task_restore_args *task_args, uns err: free_mappings(&self_vmas); err_nv: + /* Reap the async daemon if it is still running (no-op otherwise). */ + stop_asyncd(); /* Just to be sure */ exit(1); return -1; diff --git a/criu/include/asyncd.h b/criu/include/asyncd.h new file mode 100644 index 000000000..1a779d5b5 --- /dev/null +++ b/criu/include/asyncd.h @@ -0,0 +1,13 @@ +#ifndef __CR_ASYNCD_H__ +#define __CR_ASYNCD_H__ + +#include "namespaces.h" + +extern int stop_asyncd(void); +extern int start_asyncd(void); +extern int __async_call(const char *func_name, uns_call_t call, int flags, void *arg, size_t arg_size, int fd); + +#define async_call(__call, __flags, __arg, __arg_size, __fd) \ + __async_call(__stringify(__call), __call, __flags, __arg, __arg_size, __fd) + +#endif /* __CR_ASYNCD_H__ */ diff --git a/criu/include/namespaces.h b/criu/include/namespaces.h index cfd0b02e4..bcd4e908b 100644 --- a/criu/include/namespaces.h +++ b/criu/include/namespaces.h @@ -244,7 +244,8 @@ struct unsc_msg { }; extern void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid); +extern void unsc_msg_init_nocreds(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd); extern void unsc_msg_pid_fd(struct unsc_msg *um, pid_t *pid, int *fd); -extern int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk)); +extern int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk), bool passcred); #endif /* __CR_NS_H__ */ diff --git a/criu/include/restorer.h b/criu/include/restorer.h index 103b69df5..205e26be6 100644 --- a/criu/include/restorer.h +++ b/criu/include/restorer.h @@ -306,6 +306,7 @@ enum { * purely to make sure all tasks be in sync. */ CR_STATE_FORKING, + CR_STATE_PRE_RESTORER, /* * Main restore stage. By the end of it all tasks are * almost ready and what's left is: diff --git a/criu/include/rst_info.h b/criu/include/rst_info.h index c9b0c3fb4..691af2776 100644 --- a/criu/include/rst_info.h +++ b/criu/include/rst_info.h @@ -17,6 +17,7 @@ struct task_entries { mutex_t userns_sync_lock; mutex_t cgroupd_sync_lock; mutex_t last_pid_mutex; + int asyncd_sk_id; }; struct fdt { diff --git a/criu/include/shmem.h b/criu/include/shmem.h index 15cab1146..6b9226964 100644 --- a/criu/include/shmem.h +++ b/criu/include/shmem.h @@ -16,7 +16,14 @@ extern int fixup_sysv_shmems(void); extern int dump_one_memfd_shmem(int fd, unsigned long shmid, unsigned long size); extern int dump_one_sysv_shmem(void *addr, unsigned long size, unsigned long shmid); extern int restore_sysv_shmem_content(void *addr, unsigned long size, unsigned long shmid); -extern int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size); +extern int restore_shmem_fd_content(int fd, unsigned long shmid, unsigned long size); + +struct async_restore_shmem_args { + unsigned long shmid; + unsigned long size; +}; + +int async_restore_shmem_content(void *arg, int fd, pid_t pid); #define SYSV_SHMEM_SKIP_FD (0x7fffffff) diff --git a/criu/memfd.c b/criu/memfd.c index 9d9f0621f..0ed62b14c 100644 --- a/criu/memfd.c +++ b/criu/memfd.c @@ -3,6 +3,7 @@ #include "common/compiler.h" #include "common/lock.h" +#include "cr_options.h" #include "memfd.h" #include "fdinfo.h" #include "imgset.h" @@ -17,6 +18,7 @@ #include "fdstore.h" #include "file-ids.h" #include "namespaces.h" +#include "asyncd.h" #include "shmem.h" #include "hugetlb.h" @@ -258,7 +260,7 @@ static int memfd_open_inode_nocache(struct memfd_restore_inode *inode) { MemfdInodeEntry *mie = NULL; int fd = -1; - int ret = -1; + int ret, exit_code = -1; int flags; mie = inode->mie; @@ -277,11 +279,34 @@ static int memfd_open_inode_nocache(struct memfd_restore_inode *inode) fd = memfd_create(mie->name, flags); if (fd < 0) { pr_perror("Can't create memfd:%s", mie->name); - goto out; + goto err; } - if (restore_memfd_shmem_content(fd, mie->shmid, mie->size)) - goto out; + if (ftruncate(fd, mie->size) < 0) { + pr_perror("Can't resize shmem 0x%x size=%" PRIu64, mie->shmid, mie->size); + goto err; + } + + if (opts.stream) { + /* + * criu-image-streamer serves the image in a single sequential + * pass and does not reopen it. The async fill daemon reads the + * memfd content out-of-band from a separate process, which + * breaks that contract, so fill inline when restoring from a + * stream. + */ + if (restore_shmem_fd_content(fd, mie->shmid, mie->size)) + goto err; + } else { + struct async_restore_shmem_args async_arg = { + .shmid = mie->shmid, + .size = mie->size, + }; + + if (async_call(async_restore_shmem_content, 0, + &async_arg, sizeof(async_arg), fd)) + goto err; + } if (mie->has_mode) ret = cr_fchperm(fd, mie->uid, mie->gid, mie->mode); @@ -290,20 +315,19 @@ static int memfd_open_inode_nocache(struct memfd_restore_inode *inode) if (ret) { pr_perror("Can't set permissions { uid %d gid %d mode %#o } of memfd:%s", (int)mie->uid, (int)mie->gid, mie->has_mode ? (int)mie->mode : -1, mie->name); - goto out; + goto err; } inode->fdstore_id = fdstore_add(fd); if (inode->fdstore_id < 0) - goto out; + goto err; - ret = fd; + exit_code = fd; fd = -1; -out: - if (fd != -1) - close(fd); - return ret; +err: + close_safe(&fd); + return exit_code; } static int memfd_open_inode(struct memfd_restore_inode *inode) diff --git a/criu/namespaces.c b/criu/namespaces.c index 253270881..c4537ea2a 100644 --- a/criu/namespaces.c +++ b/criu/namespaces.c @@ -1221,7 +1221,8 @@ static int write_id_map(pid_t pid, UidGidExtent **extents, int n, char *id_map) static int usernsd_pid; -inline void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid) +static void unsc_msg_init_flags(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid, + bool send_creds) { struct cmsghdr *ch; struct ucred *ucred; @@ -1251,24 +1252,31 @@ inline void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, */ memzero(&m->c, sizeof(m->c)); - m->h.msg_controllen = CMSG_SPACE(sizeof(struct ucred)); + m->h.msg_controllen = 0; + ch = NULL; - ch = CMSG_FIRSTHDR(&m->h); - ch->cmsg_len = CMSG_LEN(sizeof(struct ucred)); - ch->cmsg_level = SOL_SOCKET; - ch->cmsg_type = SCM_CREDENTIALS; + if (send_creds) { + m->h.msg_controllen = CMSG_SPACE(sizeof(struct ucred)); + ch = CMSG_FIRSTHDR(&m->h); + ch->cmsg_len = CMSG_LEN(sizeof(struct ucred)); + ch->cmsg_level = SOL_SOCKET; + ch->cmsg_type = SCM_CREDENTIALS; - ucred = (struct ucred *)CMSG_DATA(ch); - if (pid) - ucred->pid = *pid; - else - ucred->pid = getpid(); - ucred->uid = getuid(); - ucred->gid = getgid(); + ucred = (struct ucred *)CMSG_DATA(ch); + if (pid) + ucred->pid = *pid; + else + ucred->pid = getpid(); + ucred->uid = getuid(); + ucred->gid = getgid(); + } if (fd >= 0) { m->h.msg_controllen += CMSG_SPACE(sizeof(int)); - ch = CMSG_NXTHDR(&m->h, ch); + if (ch) + ch = CMSG_NXTHDR(&m->h, ch); + else + ch = CMSG_FIRSTHDR(&m->h); BUG_ON(!ch); ch->cmsg_len = CMSG_LEN(sizeof(int)); ch->cmsg_level = SOL_SOCKET; @@ -1277,24 +1285,34 @@ inline void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, } } +void unsc_msg_init(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd, pid_t *pid) +{ + unsc_msg_init_flags(m, c, x, arg, asize, fd, pid, true); +} + +void unsc_msg_init_nocreds(struct unsc_msg *m, uns_call_t *c, int *x, void *arg, size_t asize, int fd) +{ + unsc_msg_init_flags(m, c, x, arg, asize, fd, NULL, false); +} + void unsc_msg_pid_fd(struct unsc_msg *um, pid_t *pid, int *fd) { struct cmsghdr *ch; struct ucred *ucred; ch = CMSG_FIRSTHDR(&um->h); - BUG_ON(!ch); - BUG_ON(ch->cmsg_len != CMSG_LEN(sizeof(struct ucred))); - BUG_ON(ch->cmsg_level != SOL_SOCKET); - BUG_ON(ch->cmsg_type != SCM_CREDENTIALS); - - if (pid) { - ucred = (struct ucred *)CMSG_DATA(ch); - *pid = ucred->pid; + if (ch && ch->cmsg_level == SOL_SOCKET && ch->cmsg_type == SCM_CREDENTIALS) { + BUG_ON(ch->cmsg_len != CMSG_LEN(sizeof(struct ucred))); + if (pid) { + ucred = (struct ucred *)CMSG_DATA(ch); + *pid = ucred->pid; + } + ch = CMSG_NXTHDR(&um->h, ch); + } else { + if (pid) + *pid = -1; } - ch = CMSG_NXTHDR(&um->h, ch); - if (ch && ch->cmsg_len == CMSG_LEN(sizeof(int))) { BUG_ON(ch->cmsg_level != SOL_SOCKET); BUG_ON(ch->cmsg_type != SCM_RIGHTS); @@ -1447,7 +1465,7 @@ out: return ret; } -int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk)) +int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk), bool passcred) { int sk[2]; int one = 1; @@ -1470,14 +1488,20 @@ int start_unix_cred_daemon(pid_t *pid, int (*daemon_func)(int sk)) return -1; } - if (setsockopt(sk[0], SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) < 0) { - pr_perror("failed to setsockopt"); - return -1; - } + if (passcred) { + if (setsockopt(sk[0], SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) < 0) { + pr_perror("failed to setsockopt"); + close(sk[0]); + close(sk[1]); + return -1; + } - if (setsockopt(sk[1], SOL_SOCKET, SO_PASSCRED, &one, sizeof(1)) < 0) { - pr_perror("failed to setsockopt"); - return -1; + if (setsockopt(sk[1], SOL_SOCKET, SO_PASSCRED, &one, sizeof(1)) < 0) { + pr_perror("failed to setsockopt"); + close(sk[0]); + close(sk[1]); + return -1; + } } *pid = fork(); @@ -1506,7 +1530,7 @@ static int start_usernsd(void) if (!(root_ns_mask & CLONE_NEWUSER)) return 0; - sk = start_unix_cred_daemon(&usernsd_pid, usernsd); + sk = start_unix_cred_daemon(&usernsd_pid, usernsd, true); if (sk < 0) { pr_err("failed to start usernsd\n"); return -1; diff --git a/criu/pagemap.c b/criu/pagemap.c index e2e93e03e..b362acc55 100644 --- a/criu/pagemap.c +++ b/criu/pagemap.c @@ -8,6 +8,7 @@ #include #include "types.h" +#include "atomic.h" #include "image.h" #include "cr_options.h" #include "servicefd.h" @@ -857,7 +858,8 @@ int probe_pages_o_direct(int fd) int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int pr_flags) { int flags, i_typ; - static unsigned ids = 1; + /* Shared across asyncd fill-daemon workers, which open page-reads concurrently. */ + static atomic_t ids = { 0 }; bool remote = pr_flags & PR_REMOTE; /* @@ -943,7 +945,7 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p pr->seek_pagemap = seek_pagemap; pr->reset = reset_pagemap; pr->io_complete = NULL; /* set up by the client if needed */ - pr->id = ids++; + pr->id = atomic_inc_return(&ids); pr->img_id = img_id; if (remote) diff --git a/criu/shmem.c b/criu/shmem.c index c0b175f50..e43147258 100644 --- a/criu/shmem.c +++ b/criu/shmem.c @@ -516,23 +516,18 @@ int restore_sysv_shmem_content(void *addr, unsigned long size, unsigned long shm return do_restore_shmem_content(addr, round_up(size, PAGE_SIZE), shmid); } -int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size) +int restore_shmem_fd_content(int fd, unsigned long shmid, unsigned long size) { - void *addr = NULL; - int ret = 1; + void *addr = MAP_FAILED; + int exit_code = -1; if (size == 0) return 0; - if (ftruncate(fd, size) < 0) { - pr_perror("Can't resize shmem 0x%lx size=%ld", shmid, size); - goto out; - } - addr = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); if (addr == MAP_FAILED) { - pr_perror("Can't mmap shmem 0x%lx size=%ld", shmid, size); - goto out; + pr_perror("Can't mmap shmem 0x%lx size=%lu", shmid, size); + goto err; } /* @@ -540,15 +535,24 @@ int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size) */ if (do_restore_shmem_content(addr, round_up(size, PAGE_SIZE), shmid) < 0) { pr_err("Can't restore shmem content\n"); - goto out; + goto err; } - ret = 0; - -out: - if (addr) + exit_code = 0; +err: + if (addr != MAP_FAILED) munmap(addr, size); - return ret; + return exit_code; +} + +int async_restore_shmem_content(void *arg, int fd, pid_t pid) +{ + struct async_restore_shmem_args *args = arg; + + if (restore_shmem_fd_content(fd, args->shmid, args->size)) + return -1; + + return 0; } struct open_map_file_args {