diff --git a/criu/include/page-pipe.h b/criu/include/page-pipe.h index 10ae87321..76ec1fd5a 100644 --- a/criu/include/page-pipe.h +++ b/criu/include/page-pipe.h @@ -138,4 +138,14 @@ extern int page_pipe_split(struct page_pipe *pp, unsigned long addr, extern void page_pipe_destroy_ppb(struct page_pipe_buf *ppb); +struct pipe_read_dest { + int p[2]; + int sink_fd; +}; + +extern int pipe_read_dest_init(struct pipe_read_dest *prd); +extern int page_pipe_read(struct page_pipe *pp, struct pipe_read_dest *prd, + unsigned long addr, unsigned int *nr_pages, + unsigned int ppb_flags); + #endif /* __CR_PAGE_PIPE_H__ */ diff --git a/criu/page-pipe.c b/criu/page-pipe.c index 4ebd0cbe8..4b4b3fc2a 100644 --- a/criu/page-pipe.c +++ b/criu/page-pipe.c @@ -474,6 +474,79 @@ int page_pipe_split(struct page_pipe *pp, unsigned long addr, return 0; } +int pipe_read_dest_init(struct pipe_read_dest *prd) +{ + int ret; + + if (pipe(prd->p)) { + pr_perror("Cannot create pipe for reading from page-pipe"); + return -1; + } + + ret = fcntl(prd->p[0], F_SETPIPE_SZ, PIPE_MAX_SIZE * PAGE_SIZE); + if (ret < 0) + return -1; + + prd->sink_fd = open("/dev/null", O_WRONLY); + if (prd->sink_fd < 0) { + pr_perror("Cannot open sink for reading from page-pipe"); + return -1; + } + + ret = fcntl(prd->p[0], F_GETPIPE_SZ, 0); + pr_debug("Created tee pipe size %d\n", ret); + + return 0; +} + +int page_pipe_read(struct page_pipe *pp, struct pipe_read_dest *prd, + unsigned long addr, unsigned int *nr_pages, + unsigned int ppb_flags) +{ + struct page_pipe_buf *ppb; + struct iovec *iov = NULL; + unsigned long skip = 0, len; + int ret; + + /* + * Get ppb that contains addr and count length of data between + * the beginning of the pipe and addr. If no ppb is found, the + * requested page is mapped to zero pfn + */ + ppb = get_ppb(pp, addr, &iov, &skip); + if (!ppb) { + *nr_pages = 0; + return 0; + } + + if (!(ppb->flags & ppb_flags)) { + pr_err("PPB flags mismatch: %x %x\n", ppb_flags, ppb->flags); + return false; + } + + /* clamp the request if it passes the end of iovec */ + len = min((unsigned long)iov->iov_base + iov->iov_len - addr, + (unsigned long)(*nr_pages) * PAGE_SIZE); + *nr_pages = len / PAGE_SIZE; + + /* we should tee() the requested lenth + the beginning of the pipe */ + len += skip; + + ret = tee(ppb->p[0], prd->p[1], len, 0); + if (ret != len) { + pr_perror("tee: %d", ret); + return -1; + } + + ret = splice(prd->p[0], NULL, prd->sink_fd, NULL, skip, 0); + if (ret != skip) { + pr_perror("splice: %d", ret); + return -1; + } + + return 0; +} + void page_pipe_destroy_ppb(struct page_pipe_buf *ppb) { list_del(&ppb->l); diff --git a/criu/page-xfer.c b/criu/page-xfer.c index e282536f2..603f77b53 100644 --- a/criu/page-xfer.c +++ b/criu/page-xfer.c @@ -577,10 +577,19 @@ static struct page_xfer_job cxfer = { .dst_id = ~0, }; +static struct pipe_read_dest pipe_read_dest = { + .sink_fd = -1, +}; + static void page_server_close(void) { if (cxfer.dst_id != ~0) cxfer.loc_xfer.close(&cxfer.loc_xfer); + if (pipe_read_dest.sink_fd != -1) { + close(pipe_read_dest.sink_fd); + close(pipe_read_dest.p[0]); + close(pipe_read_dest.p[1]); + } } static int page_server_open(int sk, struct page_server_iov *pi) @@ -684,43 +693,18 @@ static int page_server_add(int sk, struct page_server_iov *pi, u32 flags) return 0; } -static bool can_send_pages(struct page_pipe_buf *ppb, struct iovec *iov, - struct page_server_iov *pi) -{ - unsigned long len = pi->nr_pages * PAGE_SIZE; - - if (!(ppb->flags & PPB_LAZY)) { - pr_err("Requested pages are not lazy\n"); - return false; - } - - if (iov->iov_len != len) { - pr_err("IOV len %zu does not match requested %lu\n", - iov->iov_len, len); - return false; - } - - if(pi->vaddr != encode_pointer(iov->iov_base)) { - pr_err("IOV start %p does not match requested addr %"PRIx64"\n", - iov->iov_base, pi->vaddr); - return false; - } - - return true; -} - static int page_server_get_pages(int sk, struct page_server_iov *pi) { struct pstree_item *item; struct page_pipe *pp; - struct page_pipe_buf *ppb; - struct iovec *iov; + unsigned long len; int ret; item = pstree_item_by_virt(pi->dst_id); pp = dmpi(item)->mem_pp; - ret = page_pipe_split(pp, pi->vaddr, &pi->nr_pages); + ret = page_pipe_read(pp, &pipe_read_dest, pi->vaddr, + &pi->nr_pages, PPB_LAZY); if (ret) return ret; @@ -730,23 +714,17 @@ static int page_server_get_pages(int sk, struct page_server_iov *pi) return send_psi(sk, PS_IOV_ZERO, 0, 0, 0); } - ppb = list_first_entry(&pp->bufs, struct page_pipe_buf, l); - iov = &ppb->iov[0]; - - if (!can_send_pages(ppb, iov, pi)) - return -1; + len = pi->nr_pages * PAGE_SIZE; if (send_psi(sk, PS_IOV_ADD, pi->nr_pages, pi->vaddr, pi->dst_id)) return -1; - ret = splice(ppb->p[0], NULL, sk, NULL, iov->iov_len, SPLICE_F_MOVE); - if (ret != iov->iov_len) + ret = splice(pipe_read_dest.p[0], NULL, sk, NULL, len, SPLICE_F_MOVE); + if (ret != len) return -1; tcp_nodelay(sk, true); - page_pipe_destroy_ppb(ppb); - return 0; } @@ -754,8 +732,9 @@ static int page_server_serve(int sk) { int ret = -1; bool flushed = false; + bool receiving_pages = !opts.lazy_pages; - if (!opts.lazy_pages) { + if (receiving_pages) { /* * This socket only accepts data except one thing -- it * writes back the has_parent bit from time to time, so @@ -772,6 +751,7 @@ static int page_server_serve(int sk) cxfer.pipe_size = fcntl(cxfer.p[0], F_GETPIPE_SZ, 0); pr_debug("Created xfer pipe size %u\n", cxfer.pipe_size); } else { + pipe_read_dest_init(&pipe_read_dest); tcp_cork(sk, true); } @@ -831,7 +811,6 @@ static int page_server_serve(int sk) break; } case PS_IOV_GET: - flushed = true; ret = page_server_get_pages(sk, &pi); break; default: @@ -844,7 +823,7 @@ static int page_server_serve(int sk) break; } - if (!ret && !flushed) { + if (receiving_pages && !ret && !flushed) { pr_err("The data were not flushed\n"); ret = -1; }