diff --git a/criu/include/pagemap.h b/criu/include/pagemap.h index 4cbc87cc6..87d1536d4 100644 --- a/criu/include/pagemap.h +++ b/criu/include/pagemap.h @@ -61,6 +61,11 @@ struct page_read { /* Whether or not disable image deduplication*/ bool disable_dedup; + /* Whether O_DIRECT is active on the pages fd. Set once during + * open_page_read_at() after probing that direct reads work; never changes after + * that. Cached here to avoid a fcntl(F_GETFL) syscall per page read. */ + bool use_direct; + /* Private data of reader */ struct cr_img *pmi; struct cr_img *pi; @@ -108,6 +113,20 @@ struct task_restore_args; int pagemap_enqueue_iovec(struct page_read *pr, void *buf, unsigned long len, struct list_head *to); int pagemap_render_iovec(struct list_head *from, struct task_restore_args *ta); +/* + * Try to enable O_DIRECT on a pages-image fd and verify with one + * aligned probe read. + * + * 1 - O_DIRECT enabled + * 0 - O_DIRECT disabled or rejected; fd left in usable buffered + * state with POSIX_FADV_SEQUENTIAL hinted + * -1 - hard error; caller releases the fd + * + * PAGE_SIZE is not a compile-time constant on aarch64, so the probe + * buffer comes from posix_memalign(). + */ +int probe_pages_o_direct(int fd); + /* * Create a shallow copy of page_read object. * The new object shares the pagemap structures with the original, but diff --git a/criu/include/restorer.h b/criu/include/restorer.h index 5a141664c..103b69df5 100644 --- a/criu/include/restorer.h +++ b/criu/include/restorer.h @@ -169,6 +169,7 @@ struct task_restore_args { int vma_ios_fd; struct restore_vma_io *vma_ios; unsigned int vma_ios_n; + bool vma_ios_use_direct; /* set from probe_pages_o_direct() in mem.c */ struct restore_posix_timer *posix_timers; unsigned int posix_timers_n; diff --git a/criu/mem.c b/criu/mem.c index 2eb2580d4..d3b387595 100644 --- a/criu/mem.c +++ b/criu/mem.c @@ -3,8 +3,10 @@ #include #include #include +#include #include #include +#include #include "types.h" #include "cr_options.h" @@ -27,6 +29,7 @@ #include "bitmap.h" #include "sk-packet.h" #include "files-reg.h" +#include "pagemap.h" #include "pagemap-cache.h" #include "fault-injection.h" #include "prctl.h" @@ -1130,6 +1133,7 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr) { struct vma_area *vma; int ret = 0; + int exit_code = -1; struct list_head *vmas = &rsti(t)->vmas.h; struct list_head *vma_io = &rsti(t)->vma_io; @@ -1140,10 +1144,19 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr) unsigned int nr_enqueued = 0; unsigned int nr_lazy = 0; unsigned long va; + void *buf = NULL; + int memerr; vma = list_first_entry(vmas, struct vma_area, list); rsti(t)->pages_img_id = pr->pages_img_id; + /* O_DIRECT may require the buffer to be aligned. */ + memerr = posix_memalign(&buf, PAGE_SIZE, PAGE_SIZE); + if (memerr) { + pr_err("Can't allocate COW buffer: %s\n", strerror(memerr)); + return -1; + } + /* * Read page contents. */ @@ -1169,7 +1182,6 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr) } for (i = 0; i < nr_pages; i++) { - unsigned char buf[PAGE_SIZE]; void *p; /* @@ -1205,7 +1217,7 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr) } if (pagemap_enqueue_iovec(pr, (void *)va, len, vma_io)) - return -1; + goto out; pr->skip_pages(pr, len); @@ -1272,11 +1284,13 @@ static int restore_priv_vma_content(struct pstree_item *t, struct page_read *pr) err_read: if (pr->sync(pr)) - return -1; + goto out; pr->close(pr); - if (ret < 0) - return ret; + if (ret < 0) { + exit_code = ret; + goto out; + } /* Remove pages, which were not shared with a child */ list_for_each_entry(vma, vmas, list) { @@ -1300,7 +1314,7 @@ err_read: ret = madvise(addr + PAGE_SIZE * i, PAGE_SIZE, MADV_DONTNEED); if (ret < 0) { pr_perror("madvise failed"); - return -1; + goto out; } i++; nr_dropped++; @@ -1317,11 +1331,14 @@ err_read: pr_info("nr_enqueued: %d\n", nr_enqueued); pr_info("nr_lazy: %d\n", nr_lazy); - return 0; + exit_code = 0; + goto out; err_addr: pr_err("Page entry address %lx outside of VMA %lx-%lx\n", va, (long)vma->e->start, (long)vma->e->end); - return -1; +out: + xfree(buf); + return exit_code; } static int maybe_disable_thp(struct pstree_item *t, struct page_read *pr) @@ -1507,6 +1524,7 @@ static int prepare_vma_ios(struct pstree_item *t, struct task_restore_args *ta) ta->vma_ios = NULL; ta->vma_ios_n = 0; ta->vma_ios_fd = -1; + ta->vma_ios_use_direct = false; return 0; } @@ -1519,6 +1537,14 @@ static int prepare_vma_ios(struct pstree_item *t, struct task_restore_args *ta) return -1; ta->vma_ios_fd = img_raw_fd(pages); + if (ta->vma_ios_fd >= 0) { + int direct = probe_pages_o_direct(ta->vma_ios_fd); + if (direct < 0) { + close_image(pages); + return -1; + } + ta->vma_ios_use_direct = (direct == 1); + } return pagemap_render_iovec(&rsti(t)->vma_io, ta); } diff --git a/criu/pagemap.c b/criu/pagemap.c index efcce0837..dfc431ada 100644 --- a/criu/pagemap.c +++ b/criu/pagemap.c @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -762,6 +764,60 @@ free_pagemaps: return -1; } +int probe_pages_o_direct(int fd) +{ + int fl, ret, memerr; + void *probe = NULL; + ssize_t probe_ret; + + fl = fcntl(fd, F_GETFL); + if (fl < 0) + return 0; + + ret = fcntl(fd, F_SETFL, fl | O_DIRECT); + if (ret < 0) { + pr_warn("Failed to set O_DIRECT on pages fd %d: %s\n", fd, strerror(errno)); + return 0; + } + + /* + * PAGE_SIZE is not a compile-time constant on aarch64, so the + * probe buffer is allocated via posix_memalign() instead of a + * stack array with __attribute__((aligned)). + */ + memerr = posix_memalign(&probe, PAGE_SIZE, PAGE_SIZE); + if (memerr) { + pr_err("O_DIRECT probe alloc failed on pages fd %d: %s\n", fd, strerror(memerr)); + return -1; + } + + probe_ret = pread(fd, probe, PAGE_SIZE, 0); + xfree(probe); + + if (probe_ret >= 0) { + pr_debug("O_DIRECT enabled on pages fd %d\n", fd); + return 1; + } + + if (errno != EINVAL) { + pr_perror("O_DIRECT probe failed on pages fd %d", fd); + return -1; + } + + pr_info("O_DIRECT rejected at read time on pages fd %d, using buffered I/O\n", fd); + if (fcntl(fd, F_SETFL, fl) < 0) { + pr_perror("Failed to clear O_DIRECT on pages fd %d", fd); + return -1; + } + + /* + * Hint the kernel that fallback buffered reads will mostly + * advance through the pages file in offset order. + */ + posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); + return 0; +} + int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int pr_flags) { int flags, i_typ; @@ -802,6 +858,7 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p pr->pmes = NULL; pr->pieok = false; pr->disable_dedup = false; + pr->use_direct = false; pr->pmi = open_image_at(dfd, i_typ, O_RSTR, img_id); if (!pr->pmi) @@ -823,6 +880,20 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p return -1; } + { + int pfd = img_raw_fd(pr->pi); + + if (pfd >= 0 && !opts.stream) { + int direct = probe_pages_o_direct(pfd); + + if (direct < 0) { + close_page_read(pr); + return -1; + } + pr->use_direct = (direct == 1); + } + } + if (init_pagemaps(pr)) { close_page_read(pr); return -1;