From 258be469fb6d5d91d5bb50a7cc6e77ea1ad75b2e Mon Sep 17 00:00:00 2001 From: Dan Feigin Date: Sun, 26 Apr 2026 14:29:26 +0300 Subject: [PATCH] shmem: batch page reads with PR_ASYNC Shared-memory restore currently issues page reads one pagemap entry at a time. Use the existing PR_ASYNC path to enqueue shmem and memfd-shmem reads, then flush each batch with one async-queue sync pass. Restore shmem, sysv-shmem and memfd-shmem content through a single flat do_restore_shmem_content() loop that owns its page_read, and map the memfd the same way as anonymous shmem so both share one restore path instead of an extra temporary mapping. Shmem segments are mapped via mmap() (page-aligned addresses) and the pagemap offsets are PAGE_SIZE multiples, so the batched reads are page-aligned and need no special O_DIRECT alignment handling. Assisted-by: Claude:claude-sonnet-4-6 Assisted-by: Claude:claude-opus-4-7 Assisted-by: Codex:gpt-5 Signed-off-by: Dan Feigin --- criu/pagemap.c | 46 +++++++++++++++++++++++++++++++++++++++++----- criu/shmem.c | 10 +++++++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/criu/pagemap.c b/criu/pagemap.c index dfc431ada..e2e93e03e 100644 --- a/criu/pagemap.c +++ b/criu/pagemap.c @@ -28,6 +28,9 @@ #define MAX_BUNCH_SIZE 256 +#define OFF_MAX (sizeof(off_t) == sizeof(long long) ? LLONG_MAX : sizeof(off_t) == sizeof(int) ? INT_MAX : -999999) +#define OFF_MIN (sizeof(off_t) == sizeof(long long) ? LLONG_MIN : sizeof(off_t) == sizeof(int) ? INT_MIN : -999999) + /* * One "job" for the preadv() syscall in pagemap.c */ @@ -529,12 +532,42 @@ static void advance_piov(struct page_read_iov *piov, ssize_t len) pr_debug("Advanced iov %zu bytes, %d->%d iovs, %zu tail\n", olen, onr, piov->nr, len); } +/* + * Drain (free without reading) all async entries in pr and its parent chain. + * Called on error paths to satisfy BUG_ON(!list_empty(&pr->async)) in + * close_page_read(). + */ +static void drain_async_queue(struct page_read *pr) +{ + struct page_read_iov *piov, *n; + + list_for_each_entry_safe(piov, n, &pr->async, l) { + list_del(&piov->l); + xfree(piov->to); + xfree(piov); + } + if (pr->parent) + drain_async_queue(pr->parent); +} + static int process_async_reads(struct page_read *pr) { int fd, ret = 0; struct page_read_iov *piov, *n; + off_t first_off = OFF_MAX, last_end = OFF_MIN; fd = img_raw_fd(pr->pi); + if (!pr->use_direct) { + list_for_each_entry(piov, &pr->async, l) { + first_off = min(piov->from, first_off); + last_end = max(piov->end, last_end); + } + if (last_end > first_off) { + if (posix_fadvise(fd, first_off, (off_t)(last_end - first_off), POSIX_FADV_WILLNEED) != 0) + pr_debug("posix_fadvise(WILLNEED) failed for async range\n"); + } + } + list_for_each_entry_safe(piov, n, &pr->async, l) { ssize_t ret; struct iovec *iovs = piov->to; @@ -549,26 +582,27 @@ static int process_async_reads(struct page_read *pr) * purposes let's try to force the advance_piov() * and re-read tail. */ - if (ret > 0 && piov->nr >= 2) { + if (ret >= 2 * PAGE_SIZE) { pr_debug("`- trim preadv %zu\n", ret); ret /= 2; + ret &= PAGE_MASK; } } if (ret < 0) { pr_err("Can't read async pr bytes (%zd / %ju read, %ju off, %d iovs)\n", ret, piov->end - piov->from, piov->from, piov->nr); - return -1; + goto err; } if (ret == 0 && piov->end != piov->from) { pr_err("Unexpected EOF reading pages: expected %ju more bytes at offset %ju\n", piov->end - piov->from, piov->from); - return -1; + goto err; } if (opts.auto_dedup && punch_hole(pr, piov->from, ret, false)) - return -1; + goto err; if (ret != piov->end - piov->from) { /* @@ -585,7 +619,6 @@ static int process_async_reads(struct page_read *pr) } BUG_ON(pr->io_complete); /* FIXME -- implement once needed */ - list_del(&piov->l); xfree(iovs); xfree(piov); @@ -595,6 +628,9 @@ static int process_async_reads(struct page_read *pr) ret = process_async_reads(pr->parent); return ret; +err: + drain_async_queue(pr); + return -1; } static void close_page_read(struct page_read *pr) diff --git a/criu/shmem.c b/criu/shmem.c index bc7aa3669..c0b175f50 100644 --- a/criu/shmem.c +++ b/criu/shmem.c @@ -17,6 +17,7 @@ #include "rst-malloc.h" #include "vma.h" #include "mem.h" +#include "pagemap.h" #include #include "bitops.h" #include "log.h" @@ -491,11 +492,18 @@ static int do_restore_shmem_content(void *addr, unsigned long size, unsigned lon if (vaddr + nr_pages * PAGE_SIZE > size) break; - pr.read_pages(&pr, vaddr, nr_pages, addr + vaddr, 0); + ret = pr.read_pages(&pr, vaddr, nr_pages, addr + vaddr, PR_ASYNC); + if (ret < 0) + goto err; } + ret = pr.sync(&pr); pr.close(&pr); return ret; +err: + pr.sync(&pr); /* drain async queue before close */ + pr.close(&pr); + return -1; } int restore_shmem_content(void *addr, struct shmem_info *si)