pagemap/mem: probe O_DIRECT for page reads

Native AIO page restore needs the pages image fd to support direct
I/O, but some filesystems accept O_DIRECT with fcntl() and reject
the first read with EINVAL.

Enable O_DIRECT after the image headers are read, probe with an
aligned read, and fall back to buffered sequential I/O when the
probe reports EINVAL. Cache the result in page_read so normal page
reads do not need a flag lookup per request.

The streamer fd (opts.stream) is excluded from the probe because
its read loop uses arbitrary user buffers that are not aligned for
O_DIRECT.

PAGE_SIZE is not a compile-time constant on aarch64 (sysconf-backed
runtime value to support 4K/16K/64K kernels), so the probe buffer
is allocated with posix_memalign() instead of a stack array with
__attribute__((aligned(PAGE_SIZE))).

The single-page COW comparison buffer in restore_priv_vma_content()
is likewise allocated with posix_memalign() so direct reads into it
are page-aligned; this lets the local read path drop its per-read
bounce buffer.

Assisted-by: Claude:claude-sonnet-4-6
Assisted-by: Claude:claude-opus-4-7
Assisted-by: Codex:gpt-5
Signed-off-by: Dan Feigin <dfeigin@nvidia.com>

criu/include/pagemap.h

@@ -61,6 +61,11 @@ struct page_read {
 	/* Whether or not disable image deduplication*/
 	bool disable_dedup;
 
+	/* Whether O_DIRECT is active on the pages fd. Set once during
+	 * open_page_read_at() after probing that direct reads work; never changes after
+	 * that. Cached here to avoid a fcntl(F_GETFL) syscall per page read. */
+	bool use_direct;
+
 	/* Private data of reader */
 	struct cr_img *pmi;
 	struct cr_img *pi;
@@ -108,6 +113,20 @@ struct task_restore_args;
 int pagemap_enqueue_iovec(struct page_read *pr, void *buf, unsigned long len, struct list_head *to);
 int pagemap_render_iovec(struct list_head *from, struct task_restore_args *ta);
 
+/*
+ * Try to enable O_DIRECT on a pages-image fd and verify with one
+ * aligned probe read.
+ *
+ *   1 - O_DIRECT enabled
+ *   0 - O_DIRECT disabled or rejected; fd left in usable buffered
+ *       state with POSIX_FADV_SEQUENTIAL hinted
+ *  -1 - hard error; caller releases the fd
+ *
+ * PAGE_SIZE is not a compile-time constant on aarch64, so the probe
+ * buffer comes from posix_memalign().
+ */
+int probe_pages_o_direct(int fd);
+
 /*
  * Create a shallow copy of page_read object.
  * The new object shares the pagemap structures with the original, but

criu/include/restorer.h

@@ -169,6 +169,7 @@ struct task_restore_args {
 	int vma_ios_fd;
 	struct restore_vma_io *vma_ios;
 	unsigned int vma_ios_n;
+	bool vma_ios_use_direct;	/* set from probe_pages_o_direct() in mem.c */
 
 	struct restore_posix_timer *posix_timers;
 	unsigned int posix_timers_n;

criu/mem.c

@@ -3,8 +3,10 @@
 #include <sys/mman.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <string.h>
 #include <sys/syscall.h>
 #include <sys/prctl.h>
+#include <stdlib.h>
 
 #include "types.h"
 #include "cr_options.h"
@@ -27,6 +29,7 @@
 #include "bitmap.h"
 #include "sk-packet.h"
 #include "files-reg.h"
+#include "pagemap.h"
 #include "pagemap-cache.h"
 #include "fault-injection.h"
 #include "prctl.h"
@@ -1130,6 +1133,7 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr)
 {
 	struct vma_area *vma;
 	int ret = 0;
+	int exit_code = -1;
 	struct list_head *vmas = &rsti(t)->vmas.h;
 	struct list_head *vma_io = &rsti(t)->vma_io;
 
@@ -1140,10 +1144,19 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr)
 	unsigned int nr_enqueued = 0;
 	unsigned int nr_lazy = 0;
 	unsigned long va;
+	void *buf = NULL;
+	int memerr;
 
 	vma = list_first_entry(vmas, struct vma_area, list);
 	rsti(t)->pages_img_id = pr->pages_img_id;
 
+	/* O_DIRECT may require the buffer to be aligned. */
+	memerr = posix_memalign(&buf, PAGE_SIZE, PAGE_SIZE);
+	if (memerr) {
+		pr_err("Can't allocate COW buffer: %s\n", strerror(memerr));
+		return -1;
+	}
+
 	/*
 	 * Read page contents.
 	 */
@@ -1169,7 +1182,6 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr)
 		}
 
 		for (i = 0; i < nr_pages; i++) {
-			unsigned char buf[PAGE_SIZE];
 			void *p;
 
 			/*
@@ -1205,7 +1217,7 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr)
 				}
 
 				if (pagemap_enqueue_iovec(pr, (void *)va, len, vma_io))
-					return -1;
+					goto out;
 
 				pr->skip_pages(pr, len);
 
@@ -1272,11 +1284,13 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr)
 
 err_read:
 	if (pr->sync(pr))
-		return -1;
+		goto out;
 
 	pr->close(pr);
-	if (ret < 0)
-		return ret;
+	if (ret < 0) {
+		exit_code = ret;
+		goto out;
+	}
 
 	/* Remove pages, which were not shared with a child */
 	list_for_each_entry(vma, vmas, list) {
@@ -1300,7 +1314,7 @@ err_read:
 			ret = madvise(addr + PAGE_SIZE * i, PAGE_SIZE, MADV_DONTNEED);
 			if (ret < 0) {
 				pr_perror("madvise failed");
-				return -1;
+				goto out;
 			}
 			i++;
 			nr_dropped++;
@@ -1317,11 +1331,14 @@ err_read:
 	pr_info("nr_enqueued:       %d\n", nr_enqueued);
 	pr_info("nr_lazy:           %d\n", nr_lazy);
 
-	return 0;
+	exit_code = 0;
+	goto out;
 
 err_addr:
 	pr_err("Page entry address %lx outside of VMA %lx-%lx\n", va, (long)vma->e->start, (long)vma->e->end);
-	return -1;
+out:
+	xfree(buf);
+	return exit_code;
 }
 
 static int maybe_disable_thp(struct pstree_item *t, struct page_read *pr)
@@ -1507,6 +1524,7 @@ static int prepare_vma_ios(struct pstree_item *t, struct task_restore_args *ta)
 		ta->vma_ios = NULL;
 		ta->vma_ios_n = 0;
 		ta->vma_ios_fd = -1;
+		ta->vma_ios_use_direct = false;
 		return 0;
 	}
 
@@ -1519,6 +1537,14 @@ static int prepare_vma_ios(struct pstree_item *t, struct task_restore_args *ta)
 		return -1;
 
 	ta->vma_ios_fd = img_raw_fd(pages);
+	if (ta->vma_ios_fd >= 0) {
+		int direct = probe_pages_o_direct(ta->vma_ios_fd);
+		if (direct < 0) {
+			close_image(pages);
+			return -1;
+		}
+		ta->vma_ios_use_direct = (direct == 1);
+	}
 	return pagemap_render_iovec(&rsti(t)->vma_io, ta);
 }
 

criu/pagemap.c

@@ -1,6 +1,8 @@
 #include <fcntl.h>
 #include <stdio.h>
 #include <unistd.h>
+#include <errno.h>
+#include <string.h>
 #include <linux/falloc.h>
 #include <sys/uio.h>
 #include <limits.h>
@@ -762,6 +764,60 @@ free_pagemaps:
 	return -1;
 }
 
+int probe_pages_o_direct(int fd)
+{
+	int fl, ret, memerr;
+	void *probe = NULL;
+	ssize_t probe_ret;
+
+	fl = fcntl(fd, F_GETFL);
+	if (fl < 0)
+		return 0;
+
+	ret = fcntl(fd, F_SETFL, fl | O_DIRECT);
+	if (ret < 0) {
+		pr_warn("Failed to set O_DIRECT on pages fd %d: %s\n", fd, strerror(errno));
+		return 0;
+	}
+
+	/*
+	 * PAGE_SIZE is not a compile-time constant on aarch64, so the
+	 * probe buffer is allocated via posix_memalign() instead of a
+	 * stack array with __attribute__((aligned)).
+	 */
+	memerr = posix_memalign(&probe, PAGE_SIZE, PAGE_SIZE);
+	if (memerr) {
+		pr_err("O_DIRECT probe alloc failed on pages fd %d: %s\n", fd, strerror(memerr));
+		return -1;
+	}
+
+	probe_ret = pread(fd, probe, PAGE_SIZE, 0);
+	xfree(probe);
+
+	if (probe_ret >= 0) {
+		pr_debug("O_DIRECT enabled on pages fd %d\n", fd);
+		return 1;
+	}
+
+	if (errno != EINVAL) {
+		pr_perror("O_DIRECT probe failed on pages fd %d", fd);
+		return -1;
+	}
+
+	pr_info("O_DIRECT rejected at read time on pages fd %d, using buffered I/O\n", fd);
+	if (fcntl(fd, F_SETFL, fl) < 0) {
+		pr_perror("Failed to clear O_DIRECT on pages fd %d", fd);
+		return -1;
+	}
+
+	/*
+	 * Hint the kernel that fallback buffered reads will mostly
+	 * advance through the pages file in offset order.
+	 */
+	posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+	return 0;
+}
+
 int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int pr_flags)
 {
 	int flags, i_typ;
@@ -802,6 +858,7 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p
 	pr->pmes = NULL;
 	pr->pieok = false;
 	pr->disable_dedup = false;
+	pr->use_direct = false;
 
 	pr->pmi = open_image_at(dfd, i_typ, O_RSTR, img_id);
 	if (!pr->pmi)
@@ -823,6 +880,20 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p
 		return -1;
 	}
 
+	{
+		int pfd = img_raw_fd(pr->pi);
+
+		if (pfd >= 0 && !opts.stream) {
+			int direct = probe_pages_o_direct(pfd);
+
+			if (direct < 0) {
+				close_page_read(pr);
+				return -1;
+			}
+			pr->use_direct = (direct == 1);
+		}
+	}
+
 	if (init_pagemaps(pr)) {
 		close_page_read(pr);
 		return -1;