From 1ab35f2e5f2d30bb0345ccf8be3c393041d22660 Mon Sep 17 00:00:00 2001 From: Radostin Stoyanov Date: Sun, 15 Mar 2026 17:44:20 +0000 Subject: [PATCH] pagemap: Read compressed page images Existing pagemap readers derive each pages-image offset from a fixed PAGE_SIZE payload per present page. Packed compressed blocks make that calculation invalid and must be reconstructed before reaching restore destinations. Track block and intra-region cursors while advancing, skipping, and reading pagemap entries. Validate block counts, decoded sizes, totals, and aligned-payload offsets before I/O. Read per-page and region images locally, preserve ordinary parent entries, and support sequential per-page image streams. Carry block metadata with queued iovecs, coalesce only compatible regions, read each packed batch once, and rebuild its destinations block by block. Allow exactly IOV_MAX destinations and roll back iovec changes if appending metadata fails. Decompress complete regions directly into a contiguous destination and use scratch space for partial or split regions. Copy the metadata into restorer memory for the later PIE decode path. Keep compressed images on buffered I/O, avoid hole punching packed payloads, handle short reads, and keep compressed local reads out of PIE until that path can decode them. Assisted-by: Codex:GPT-5 Signed-off-by: Radostin Stoyanov --- criu/include/pagemap.h | 25 + criu/include/restorer.h | 14 + criu/pagemap.c | 1241 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 1262 insertions(+), 18 deletions(-) diff --git a/criu/include/pagemap.h b/criu/include/pagemap.h index 6d57e912c..1abda34d2 100644 --- a/criu/include/pagemap.h +++ b/criu/include/pagemap.h @@ -98,6 +98,23 @@ struct page_read { /* Current offset in pages file */ off_t pi_off; + /* Alignment bytes a sequential image-streamer reader must discard. */ + size_t stream_padding; + + /* + * Index into pe->compressed_size[] for the current pagemap + * entry. Tracks which compressed block (page in per-page mode, + * region in region mode) we are on when reading or skipping + * compressed pages. Reset to 0 on advance(). + */ + size_t compressed_size_index; + + /* + * In region mode: pages already consumed (read or skipped) from + * the current block. Always 0 in per-page mode. Reset to 0 on + * advance() and whenever the reader crosses a block boundary. + */ + unsigned int region_block_offset; /* Record consequent neighbour iov-ecs to punch together */ struct iovec bunch; @@ -201,4 +218,12 @@ static inline bool pagemap_payload_aligned(PagemapEntry *pe) return !!(pe->flags & PE_PAYLOAD_ALIGNED); } +/* Keep the mask as wide as off_t so offsets above 4 GiB remain intact. */ +static inline off_t pagemap_page_align_offset(off_t offset) +{ + off_t mask = ~((off_t)PAGE_SIZE - 1); + + return (offset + (off_t)PAGE_SIZE - 1) & mask; +} + #endif /* __CR_PAGE_READ_H__ */ diff --git a/criu/include/restorer.h b/criu/include/restorer.h index 205e26be6..44a2f3c1e 100644 --- a/criu/include/restorer.h +++ b/criu/include/restorer.h @@ -138,6 +138,20 @@ typedef long (*thread_restore_fcall_t)(struct thread_restore_args *args); struct restore_vma_io { int nr_iovs; loff_t off; + uint32_t *compressed_size; + uint64_t total_compressed_size; + int n_compressed_size; + /* + * Region compression metadata. region_pages == 0 means per-page + * compression and block_pages is unused. When region_pages > 0, + * compressed_size[] holds n_compressed_size (== n_blocks) entries + * and block_pages is a parallel uint16_t-per-block array giving + * each block's page count (the last block of any pagemap entry + * spanning this iov may be shorter than region_pages). + */ + uint32_t region_pages; + int n_pages; + uint16_t *block_pages; struct iovec iovs[0]; }; diff --git a/criu/pagemap.c b/criu/pagemap.c index d2180704b..09be3db08 100644 --- a/criu/pagemap.c +++ b/criu/pagemap.c @@ -16,6 +16,7 @@ #include "restorer.h" #include "rst-malloc.h" #include "page-xfer.h" +#include "compression.h" #include "fault-injection.h" #include "xmalloc.h" @@ -28,6 +29,7 @@ #endif #define MAX_BUNCH_SIZE 256 +#define PAGE_PADDING_CHUNK 4096 #define OFF_MAX (sizeof(off_t) == sizeof(long long) ? LLONG_MAX : sizeof(off_t) == sizeof(int) ? INT_MAX : -999999) #define OFF_MIN (sizeof(off_t) == sizeof(long long) ? LLONG_MIN : sizeof(off_t) == sizeof(int) ? INT_MIN : -999999) @@ -37,10 +39,38 @@ */ struct page_read_iov { off_t from; /* offset in pi file where to start reading from */ - off_t end; /* the end of the read == sum to.iov_len -s */ + off_t end; /* exclusive end offset in the pages image */ struct iovec *to; /* destination iovs */ unsigned int nr; /* their number */ + size_t n_compressed_size; /* Number of compressed blocks (pages or regions) */ + uint32_t *compressed_size; /* Per-block compressed sizes */ + uint64_t total_compressed_size; /* Sum of all compressed sizes */ + /* + * Region size in pages when this iov holds region-compressed data. + * 0 means per-page compression (or no compression). + * Iovs with different region_pages cannot be coalesced into one + * async batch. + */ + unsigned int region_pages; + /* + * Per-block page count when region_pages > 0. Most blocks have + * exactly region_pages pages; the last block of any pagemap + * entry that spans this piov may be shorter if the entry's + * nr_pages is not a multiple of region_pages. + */ + uint16_t *block_pages; + + /* + * RM_PRIVATE offsets of the compressed_size[] and block_pages[] + * copies that pagemap_render_iovec() places in restorer memory. + * Stored as offsets (not pointers) because struct restore_vma_io + * references them via RST_MEM_FIXUP_PPTR, which adds the remapped + * arena base to the stored value. + */ + unsigned long rio_cs_off; + unsigned long rio_bp_off; + struct list_head l; }; @@ -103,9 +133,21 @@ int dedup_one_iovec(struct page_read *pr, unsigned long off, unsigned long len) return -1; piov_end = pr->pe->vaddr + pagemap_len(pr->pe); if (!pagemap_in_parent(pr->pe)) { - ret = punch_hole(pr, pr->pi_off, min(piov_end, iov_end) - off, false); - if (ret == -1) - return ret; + /* + * Skip hole-punching for compressed entries. + * fallocate(PUNCH_HOLE) zeroes whole filesystem + * blocks (typically 4K). Compressed pages are + * smaller than a block and packed contiguously, + * so punching one would destroy adjacent pages + * that share the same block. + */ + if (!pr->pe->n_compressed_size) { + unsigned long punch_len = min(piov_end, iov_end) - off; + + ret = punch_hole(pr, pr->pi_off, punch_len, false); + if (ret == -1) + return ret; + } } prp = pr->parent; @@ -135,17 +177,121 @@ static int advance(struct page_read *pr) pr->pe = pr->pmes[pr->curr_pme]; pr->cvaddr = pr->pe->vaddr; + pr->stream_padding = 0; + if (pagemap_present(pr->pe) && pagemap_payload_aligned(pr->pe)) { + off_t aligned; + + /* init_pagemaps() rejects this; retain a checked guard here so + * malformed state can never trigger signed off_t overflow. */ + if (pr->pi_off < 0 || pr->pi_off > OFF_MAX - (PAGE_SIZE - 1)) { + pr_err("Pages image offset %jd cannot be page-aligned\n", + (intmax_t)pr->pi_off); + return 0; + } + aligned = pagemap_page_align_offset(pr->pi_off); + + pr->stream_padding = aligned - pr->pi_off; + pr->pi_off = aligned; + } + + pr->compressed_size_index = 0; + pr->region_block_offset = 0; return 1; } +/* + * Return the page count of the current compressed block (in region mode). + * The last block of an entry may be shorter than region_pages when the + * entry's nr_pages is not a multiple of region_pages. + */ +static unsigned int current_block_pages(struct page_read *pr) +{ + unsigned int region_pages = pr->pe->region_pages; + uint64_t pages_before = (uint64_t)pr->compressed_size_index * region_pages; + uint64_t pages_left = pr->pe->nr_pages - pages_before; + + return pages_left < region_pages ? (unsigned int)pages_left : region_pages; +} + +/* + * Advance the file offset (pi_off) by @len bytes without reading + * the page data. Used by seek_pagemap() to skip over pages that + * are not needed for the current read request. + */ static void skip_pagemap_pages(struct page_read *pr, unsigned long len) { if (!len) return; - if (pagemap_present(pr->pe)) - pr->pi_off += len; + if (pagemap_present(pr->pe)) { + if (!pr->pe->n_compressed_size) { + pr->pi_off += len; + } else if (pr->compressed_size_index == 0 && + pr->region_block_offset == 0 && + pr->pe->has_total_compressed_size && + (uint64_t)(len / PAGE_SIZE) == pr->pe->nr_pages) { + /* + * Fast path: skipping a whole compressed entry from + * its start. total_compressed_size equals the sum of + * compressed_size[] for both per-page and region mode, + * so advance pi_off by it in a single step instead of + * walking every block. Mark all blocks consumed so the + * reader state matches the block-by-block paths below. + */ + pr->pi_off += pr->pe->total_compressed_size; + pr->compressed_size_index = pr->pe->n_compressed_size; + } else if (pr->pe->has_region_pages && pr->pe->region_pages) { + /* + * Region mode: each compressed_size[] entry covers + * up to region_pages pages. Walk forward, advancing + * the in-block cursor and crossing block boundaries + * by adding the block's compressed size to pi_off. + */ + unsigned long nr = len / PAGE_SIZE; + + while (nr > 0) { + unsigned int bp = current_block_pages(pr); + unsigned int avail = bp - pr->region_block_offset; + unsigned int take = avail; + + if (nr < avail) + take = (unsigned int)nr; + + if (pr->compressed_size_index >= pr->pe->n_compressed_size) { + pr_err("skip_pagemap_pages: index out of bounds: " + "%zu >= %zu\n", + pr->compressed_size_index, + pr->pe->n_compressed_size); + BUG(); + } + + pr->region_block_offset += take; + nr -= take; + + if (pr->region_block_offset == bp) { + pr->pi_off += pr->pe->compressed_size[pr->compressed_size_index]; + pr->compressed_size_index++; + pr->region_block_offset = 0; + } + } + } else { + /* Per-page compressed: variable size per page. */ + unsigned long nr = len / PAGE_SIZE; + unsigned long i; + + if (pr->compressed_size_index + nr > pr->pe->n_compressed_size) { + pr_err("skip_pagemap_pages: index out of bounds: %zu + %lu > %zu\n", + pr->compressed_size_index, nr, + pr->pe->n_compressed_size); + BUG(); + } + + for (i = 0; i < nr; i++) + pr->pi_off += pr->pe->compressed_size[pr->compressed_size_index + i]; + pr->compressed_size_index += nr; + } + } pr->cvaddr += len; } @@ -276,6 +422,97 @@ static int read_local_page(struct page_read *pr, unsigned long vaddr, unsigned l return 0; } +static int piov_add_compressed_blocks(struct page_read_iov *piov, + struct page_read *pr, + unsigned long nr_pages) +{ + unsigned int region_pages = 0; + size_t first = piov->n_compressed_size; + unsigned long n_blocks; + uint32_t *new_cs; + uint64_t added = 0; + size_t new_n; + unsigned long i; + + if (pr->pe->has_region_pages && pr->pe->region_pages) + region_pages = pr->pe->region_pages; + + if (region_pages) { + uint64_t pages_consumed = (uint64_t)pr->compressed_size_index * region_pages; + uint64_t end_page = pages_consumed + nr_pages; + + /* + * Region mode allows two alignment cases: nr_pages is + * a multiple of region_pages, or the read finishes + * exactly at the entry's end (last region may be + * shorter than region_pages). + */ + if (pages_consumed % region_pages != 0 || + (nr_pages % region_pages != 0 && + end_page != pr->pe->nr_pages)) { + if (first) + pr_err("Region-mode async append not aligned: idx=%zu nr=%lu region=%u nr_pages=%" PRIu64 "\n", + pr->compressed_size_index, nr_pages, + region_pages, pr->pe->nr_pages); + else + pr_err("Region-mode async enqueue not aligned: idx=%zu nr=%lu region=%u nr_pages=%" PRIu64 "\n", + pr->compressed_size_index, nr_pages, + region_pages, pr->pe->nr_pages); + return -1; + } + n_blocks = (nr_pages + region_pages - 1) / region_pages; + } else { + n_blocks = nr_pages; + } + + if (pr->compressed_size_index + n_blocks > + pr->pe->n_compressed_size) { + pr_err("Compressed size index out of bounds: %zu + %lu > %zu\n", + pr->compressed_size_index, n_blocks, + pr->pe->n_compressed_size); + return -1; + } + + new_n = first + n_blocks; + new_cs = xrealloc(piov->compressed_size, new_n * sizeof(*piov->compressed_size)); + if (!new_cs) + return -1; + piov->compressed_size = new_cs; + + for (i = 0; i < n_blocks; i++) { + uint32_t cs = pr->pe->compressed_size[pr->compressed_size_index + i]; + + piov->compressed_size[first + i] = cs; + added += cs; + } + + if (region_pages) { + uint64_t pages_consumed_at_start = (uint64_t)pr->compressed_size_index * region_pages; + uint16_t *new_bp = xrealloc(piov->block_pages, new_n * sizeof(*piov->block_pages)); + + if (!new_bp) + return -1; + piov->block_pages = new_bp; + + for (i = 0; i < n_blocks; i++) { + uint64_t pages_so_far = pages_consumed_at_start + i * region_pages; + uint64_t pages_left = pr->pe->nr_pages - pages_so_far; + uint16_t block_pages = (uint16_t)region_pages; + + if (pages_left < region_pages) + block_pages = (uint16_t)pages_left; + piov->block_pages[first + i] = block_pages; + } + } + + piov->total_compressed_size += added; + piov->n_compressed_size = new_n; + piov->region_pages = region_pages; + piov->end += added; + + return 0; +} + static int enqueue_async_iov(struct page_read *pr, void *buf, unsigned long len, struct list_head *to) { struct page_read_iov *pr_iov; @@ -286,7 +523,7 @@ static int enqueue_async_iov(struct page_read *pr, void *buf, unsigned long len, return -1; pr_iov->from = pr->pi_off; - pr_iov->end = pr->pi_off + len; + pr_iov->end = pr_iov->from; iov = xzalloc(sizeof(*iov)); if (!iov) { @@ -300,6 +537,23 @@ static int enqueue_async_iov(struct page_read *pr, void *buf, unsigned long len, pr_iov->to = iov; pr_iov->nr = 1; + /* + * For uncompressed entries, the end offset is simply + * start + len. For compressed entries, copy the per-block + * sizes (one per page in per-page mode, one per region in + * region mode) so that process_async_reads() knows how much + * compressed data to read from the image. + */ + if (!pr->pe || !pr->pe->n_compressed_size) { + pr_iov->end += len; + } else if (piov_add_compressed_blocks(pr_iov, pr, len / PAGE_SIZE)) { + xfree(pr_iov->compressed_size); + xfree(pr_iov->block_pages); + xfree(iov); + xfree(pr_iov); + return -1; + } + list_add_tail(&pr_iov->l, to); return 0; @@ -309,6 +563,47 @@ int pagemap_render_iovec(struct list_head *from, struct task_restore_args *ta) { struct page_read_iov *piov; + /* + * Pass 1: copy each entry's compressed metadata (compressed_size[] + * and, for region mode, block_pages[]) into restorer memory and + * remember their RM_PRIVATE offsets. + * + * These must be allocated before the contiguous rio array (pass 2) + * for two reasons: the restorer strides over the rio array with + * RIO_SIZE(), so nothing may be interleaved between rios; and + * rst_mem_alloc() can move the arena, so a rio pointer must never be + * held across a subsequent allocation. Offsets are stored (not + * pointers) because struct restore_vma_io references them through + * RST_MEM_FIXUP_PPTR, which adds the remapped base to the value. + */ + list_for_each_entry(piov, from, l) { + size_t sz; + void *p; + + piov->rio_cs_off = 0; + piov->rio_bp_off = 0; + + if (!piov->n_compressed_size) + continue; + + sz = piov->n_compressed_size * sizeof(uint32_t); + piov->rio_cs_off = rst_mem_align_cpos(RM_PRIVATE); + p = rst_mem_alloc(sz, RM_PRIVATE); + if (!p) + return -1; + memcpy(p, piov->compressed_size, sz); + + if (piov->region_pages) { + sz = piov->n_compressed_size * sizeof(uint16_t); + piov->rio_bp_off = rst_mem_align_cpos(RM_PRIVATE); + p = rst_mem_alloc(sz, RM_PRIVATE); + if (!p) + return -1; + memcpy(p, piov->block_pages, sz); + } + } + + /* Pass 2: build the contiguous rio array. */ ta->vma_ios = (struct restore_vma_io *)rst_mem_align_cpos(RM_PRIVATE); ta->vma_ios_n = 0; @@ -322,6 +617,50 @@ int pagemap_render_iovec(struct list_head *from, struct task_restore_args *ta) rio->nr_iovs = piov->nr; rio->off = piov->from; + + /* Ordinary reads do not carry compressed block metadata. */ + if (!piov->n_compressed_size) { + rio->compressed_size = NULL; + rio->n_compressed_size = 0; + rio->total_compressed_size = 0; + rio->region_pages = 0; + rio->n_pages = 0; + rio->block_pages = NULL; + } else { + /* Use the block metadata copied in pass 1. */ + rio->compressed_size = (uint32_t *)piov->rio_cs_off; + rio->n_compressed_size = piov->n_compressed_size; + rio->total_compressed_size = piov->total_compressed_size; + rio->region_pages = piov->region_pages; + + /* The final region may contain fewer pages. */ + if (piov->region_pages) { + uint64_t pages_total = 0; + size_t i; + + rio->block_pages = (uint16_t *)piov->rio_bp_off; + for (i = 0; i < piov->n_compressed_size; i++) + pages_total += piov->block_pages[i]; + + if (pages_total > INT_MAX) { + pr_err("restore_vma_io exceeds int page count limit: %llu\n", + (unsigned long long)pages_total); + return -1; + } + rio->n_pages = (int)pages_total; + } else { + /* Per-page mode uses one page per block. */ + rio->block_pages = NULL; + + if (piov->n_compressed_size > (size_t)INT_MAX) { + pr_err("restore_vma_io exceeds int page count limit: %zu\n", + piov->n_compressed_size); + return -1; + } + rio->n_pages = piov->n_compressed_size; + } + } + memcpy(rio->iovs, piov->to, piov->nr * sizeof(struct iovec)); ta->vma_ios_n++; @@ -334,6 +673,12 @@ int pagemap_enqueue_iovec(struct page_read *pr, void *buf, unsigned long len, st { struct page_read_iov *cur_async = NULL; struct iovec *iov; + bool extended_iov = false; + bool added_iov = false; + unsigned int new_region_pages = 0; + + if (pr->pe && pr->pe->has_region_pages && pr->pe->region_pages) + new_region_pages = pr->pe->region_pages; if (!list_empty(to)) cur_async = list_entry(to->prev, struct page_read_iov, l); @@ -347,6 +692,13 @@ int pagemap_enqueue_iovec(struct page_read *pr, void *buf, unsigned long len, st if (!cur_async || pr->pi_off != cur_async->end) return enqueue_async_iov(pr, buf, len, to); + /* + * Don't merge piovs with different region modes: each piov is + * decompressed with a single algorithm. + */ + if (cur_async->region_pages != new_region_pages) + return enqueue_async_iov(pr, buf, len, to); + /* * This read is pure continuation of the previous one. Let's * just add another IOV (or extend one of the existing). @@ -355,11 +707,12 @@ int pagemap_enqueue_iovec(struct page_read *pr, void *buf, unsigned long len, st if (iov->iov_base + iov->iov_len == buf) { /* Extendable */ iov->iov_len += len; + extended_iov = true; } else { /* Need one more target iovec */ unsigned int n_iovs = cur_async->nr + 1; - if (n_iovs >= IOV_MAX) + if (n_iovs > IOV_MAX) return enqueue_async_iov(pr, buf, len, to); iov = xrealloc(cur_async->to, n_iovs * sizeof(*iov)); @@ -373,11 +726,25 @@ int pagemap_enqueue_iovec(struct page_read *pr, void *buf, unsigned long len, st iov->iov_len = len; cur_async->nr = n_iovs; + added_iov = true; } - cur_async->end += len; + /* Extend the end offset. For compressed entries, append + * per-block sizes and advance by compressed bytes. */ + if (!pr->pe || !pr->pe->n_compressed_size) { + cur_async->end += len; + } else if (piov_add_compressed_blocks(cur_async, pr, len / PAGE_SIZE)) { + goto rollback_iov; + } return 0; + +rollback_iov: + if (extended_iov) + cur_async->to[cur_async->nr - 1].iov_len -= len; + else if (added_iov) + cur_async->nr--; + return -1; } static int maybe_read_page_local(struct page_read *pr, unsigned long vaddr, unsigned long nr, void *buf, unsigned flags) @@ -404,6 +771,340 @@ static int maybe_read_page_local(struct page_read *pr, unsigned long vaddr, unsi return ret; } +static int pread_full(int fd, void *buf, size_t count, off_t offset) +{ + size_t rd = 0; + + while (rd < count) { + ssize_t ret = pread(fd, (char *)buf + rd, count - rd, offset + rd); + if (ret < 0) { + if (errno == EINTR) + continue; + pr_perror("Short pread %zd/%zu at offset %lld", + ret, count, (long long)(offset + rd)); + return -1; + } + if (ret == 0) { + pr_err("Short pread %zu/%zu at offset %lld\n", + rd, count, (long long)(offset + rd)); + return -1; + } + rd += ret; + } + return 0; +} + +/* + * Advance compressed offset tracking by @nr pages without + * reading any data. Used when enqueuing async compressed reads. + * + * Caller has already verified that @nr aligns to region boundaries + * (start at block 0 offset, end on block boundary or entry end) and + * that pr->region_block_offset is 0 in region mode. + */ +static void skip_compressed_offsets(struct page_read *pr, unsigned long nr) +{ + unsigned int region_pages = 0; + unsigned long n_blocks, i; + + if (pr->pe && pr->pe->has_region_pages && pr->pe->region_pages) + region_pages = pr->pe->region_pages; + + /* + * Whole-entry skip from its start: advance by total_compressed_size + * in one step (same fast path as skip_pagemap_pages()) instead of + * summing every block. The caller guarantees region_block_offset is + * 0 here, so compressed_size_index == 0 means we are at the start. + */ + if (pr->compressed_size_index == 0 && pr->pe->has_total_compressed_size && + (uint64_t)nr == pr->pe->nr_pages) { + pr->pi_off += pr->pe->total_compressed_size; + pr->compressed_size_index = pr->pe->n_compressed_size; + return; + } + + if (region_pages) + n_blocks = (nr + region_pages - 1) / region_pages; + else + n_blocks = nr; + + if (pr->compressed_size_index + n_blocks > pr->pe->n_compressed_size) { + pr_err("skip_compressed_offsets: index out of bounds: %zu + %lu > %zu\n", + pr->compressed_size_index, n_blocks, pr->pe->n_compressed_size); + BUG(); + } + + for (i = 0; i < n_blocks; i++) + pr->pi_off += pr->pe->compressed_size[pr->compressed_size_index + i]; + pr->compressed_size_index += n_blocks; +} + +/* + * Synchronous compressed page read: decompress pages one-by-one + * from the image file directly into @buf. + */ +static int read_compressed_pages(struct page_read *pr, int fd, + unsigned long nr, void *buf) +{ + size_t curr = 0; + off_t compressed_offset = 0; + char compressed_buf[PAGE_COMPRESSED_SIZE_BOUND]; + unsigned long i; + + for (i = 0; i < nr; i++) { + size_t idx = pr->compressed_size_index + i; + uint32_t cs = pr->pe->compressed_size[idx]; + + if (cs > PAGE_SIZE) { + pr_err("Invalid compressed size %u for page %zu\n", + cs, idx); + return -1; + } + + if (cs == 0) { + /* Zero page, nothing stored in the image */ + memset(buf + curr, 0, PAGE_SIZE); + } else if (cs == PAGE_SIZE) { + /* Stored raw, no decompression needed */ + if (pread_full(fd, buf + curr, PAGE_SIZE, + pr->pi_off + compressed_offset)) + return -1; + } else { + if (pread_full(fd, compressed_buf, cs, + pr->pi_off + compressed_offset)) + return -1; + + if (decompress_data(compressed_buf, cs, PAGE_SIZE, + buf + curr)) { + pr_err("Decompression failed for i=%zu compressed_size=%u, curr: %zu pi_off: %lld\n", + idx, cs, curr, (long long)(pr->pi_off + compressed_offset)); + return -1; + } + } + + curr += PAGE_SIZE; + compressed_offset += cs; + } + + pr->pi_off += compressed_offset; + pr->compressed_size_index += nr; + + return 0; +} + +/* + * Region-mode synchronous compressed read. Each compressed_size[] + * entry covers up to region_pages pages. Reads may start mid-region + * (when pr->region_block_offset > 0) and may end mid-region. + * + * For each block touched, read its compressed bytes, decompress into + * a heap scratch buffer once, then memcpy the requested page slice + * out. Crossing a block boundary advances pr->compressed_size_index + * and pr->pi_off; staying within a block bumps region_block_offset. + */ +static int read_compressed_pages_region(struct page_read *pr, int fd, + unsigned long nr, void *buf) +{ + unsigned int region_pages = pr->pe->region_pages; + size_t region_bytes_max = (size_t)region_pages * PAGE_SIZE; + unsigned long pages_done = 0; + int rc = -1; + char *scratch; + + scratch = xmalloc(region_bytes_max); + if (!scratch) + return -1; + + while (pages_done < nr) { + size_t idx = pr->compressed_size_index; + uint32_t cs; + unsigned int this_region; + size_t this_bytes; + unsigned int off = pr->region_block_offset; + unsigned int avail; + unsigned int take; + size_t out_bytes; + + if (idx >= pr->pe->n_compressed_size) { + pr_err("region read: index %zu >= n_compressed %zu\n", + idx, pr->pe->n_compressed_size); + goto out; + } + + cs = pr->pe->compressed_size[idx]; + this_region = current_block_pages(pr); + this_bytes = (size_t)this_region * PAGE_SIZE; + + if ((size_t)cs > this_bytes) { + pr_err("Invalid region compressed size %u (region=%u idx=%zu)\n", + cs, this_region, idx); + goto out; + } + + avail = this_region - off; + take = avail; + if (nr - pages_done < avail) + take = (unsigned int)(nr - pages_done); + out_bytes = (size_t)take * PAGE_SIZE; + + if (cs == 0) { + memset((char *)buf + pages_done * PAGE_SIZE, 0, out_bytes); + } else if ((size_t)cs == this_bytes) { + /* Stored raw: pread directly the requested slice. */ + if (pread_full(fd, + (char *)buf + pages_done * PAGE_SIZE, + out_bytes, + pr->pi_off + (off_t)off * PAGE_SIZE)) + goto out; + } else if (off == 0 && take == this_region) { + /* Whole region in one go: decompress straight into the dest. */ + if (pread_full(fd, scratch, cs, pr->pi_off)) + goto out; + if (decompress_region(scratch, cs, this_region, + (char *)buf + pages_done * PAGE_SIZE)) { + pr_err("Region decompression failed (idx=%zu cs=%u region=%u)\n", + idx, cs, this_region); + goto out; + } + } else { + /* Partial slice: decompress into scratch, then memcpy. */ + char *region_dec = xmalloc(this_bytes); + + if (!region_dec) + goto out; + if (pread_full(fd, scratch, cs, pr->pi_off)) { + xfree(region_dec); + goto out; + } + if (decompress_region(scratch, cs, this_region, region_dec)) { + pr_err("Region decompression failed (idx=%zu cs=%u region=%u)\n", + idx, cs, this_region); + xfree(region_dec); + goto out; + } + memcpy((char *)buf + pages_done * PAGE_SIZE, + region_dec + (size_t)off * PAGE_SIZE, + out_bytes); + xfree(region_dec); + } + + pages_done += take; + pr->region_block_offset = off + take; + + if (pr->region_block_offset == this_region) { + pr->pi_off += cs; + pr->compressed_size_index++; + pr->region_block_offset = 0; + } + } + + rc = 0; +out: + xfree(scratch); + return rc; +} + +static int maybe_read_page_local_compressed(struct page_read *pr, unsigned long vaddr, unsigned long nr, void *buf, unsigned flags) +{ + int fd, ret; + unsigned long len = nr * PAGE_SIZE; + unsigned int region_pages = 0; + + if (pr->pe->has_region_pages && pr->pe->region_pages) + region_pages = pr->pe->region_pages; + + /* + * If this pagemap entry has no compressed_size array, it + * was stored uncompressed (e.g. shared memory pagemaps or + * entries from a non-compressed parent). Fall back to the + * normal uncompressed reader. + */ + if (!pr->pe->n_compressed_size) + return maybe_read_page_local(pr, vaddr, nr, buf, flags); + + /* + * region_pages comes from the image and sizes the region scratch + * buffers and a uint16_t block_pages[]; reject a value the dump + * side would never write before it drives a huge allocation or + * truncates silently. + */ + if (region_pages > MAX_REGION_PAGES) { + pr_err("Invalid region_pages %u in pagemap (max %lu)\n", + region_pages, MAX_REGION_PAGES); + return -1; + } + + if (!region_pages && + pr->compressed_size_index + nr > pr->pe->n_compressed_size) { + pr_err("Compressed size index out of bounds: %zu + %lu > %zu\n", + pr->compressed_size_index, nr, + pr->pe->n_compressed_size); + return -1; + } + + /* + * For PR_ASYNC (without PR_ASAP), enqueue the request. + * The compressed_size metadata is captured by + * pagemap_enqueue_iovec(); actual decompression happens + * later in process_async_reads(). This batches many + * small reads into fewer large pread() calls. + * + * Region mode async requires the request to align to region + * boundaries (start at offset 0 in a block; finish on a block + * boundary or at the entry's end). Mid-region async requests + * fall through to the sync path. + */ + if ((flags & (PR_ASYNC | PR_ASAP)) == PR_ASYNC) { + bool can_async = true; + + if (region_pages) { + uint64_t pages_consumed = (uint64_t)pr->compressed_size_index * region_pages; + uint64_t end_page = pages_consumed + nr; + + can_async = (pr->region_block_offset == 0) && + (nr % region_pages == 0 || + end_page == pr->pe->nr_pages); + } + + if (can_async) { + ret = pagemap_enqueue_iovec(pr, buf, len, &pr->async); + if (ret) + return ret; + skip_compressed_offsets(pr, nr); + return 0; + } + /* Mid-region or unaligned: fall back to sync. */ + } + + fd = img_raw_fd(pr->pi); + if (fd < 0) { + pr_err("Failed getting raw image fd\n"); + return -1; + } + + /* + * Flush any pending async requests not to break the + * linear reading from the pages.img file. + */ + if (pr->sync(pr)) + return -1; + + pr_debug("\tpr%lu-%u Read page from self %lx/%" PRIx64 "\n", + pr->img_id, pr->id, pr->cvaddr, pr->pi_off); + + if (region_pages) + ret = read_compressed_pages_region(pr, fd, nr, buf); + else + ret = read_compressed_pages(pr, fd, nr, buf); + if (ret) + return ret; + + if (pr->io_complete) + ret = pr->io_complete(pr, vaddr, nr); + + return ret; +} + /* * We cannot use maybe_read_page_local() for streaming images as it uses * pread(), seeking in the file. Instead, we use this custom page reader. @@ -420,6 +1121,23 @@ static int maybe_read_page_img_streamer(struct page_read *pr, unsigned long vadd pr_err("Getting raw FD failed\n"); return -1; } + while (pr->stream_padding) { + char discard[PAGE_PADDING_CHUNK]; + size_t chunk = min(pr->stream_padding, sizeof(discard)); + + ret = read(fd, discard, chunk); + if (ret == 0) { + pr_err("Reached EOF while skipping page-image alignment\n"); + return -1; + } + if (ret < 0) { + if (errno == EINTR) + continue; + pr_perror("Can't skip page-image alignment"); + return -1; + } + pr->stream_padding -= ret; + } pr_debug("\tpr%lu-%u Read page from self %lx/%" PRIx64 "\n", pr->img_id, pr->id, pr->cvaddr, pr->pi_off); @@ -451,6 +1169,146 @@ static int maybe_read_page_img_streamer(struct page_read *pr, unsigned long vadd return ret; } +/* + * Streaming + compressed reader: reads compressed data sequentially + * from the pipe (no seeking), decompresses page-by-page. + */ +static int maybe_read_page_img_streamer_compressed(struct page_read *pr, unsigned long vaddr, unsigned long nr, void *buf, unsigned flags) +{ + int fd; + ssize_t ret = 0; + size_t curr = 0; + off_t compressed_offset = 0; + char compressed_buf[PAGE_COMPRESSED_SIZE_BOUND]; + unsigned long i; + + /* + * Fall back to uncompressed streamer for entries without compressed data. + * Region mode is not supported in streaming restore (checked in config). + */ + if (!pr->pe->n_compressed_size) + return maybe_read_page_img_streamer(pr, vaddr, nr, buf, flags); + + if (pr->pe->has_region_pages && pr->pe->region_pages) { + pr_err("Streaming restore does not support region-mode compression\n"); + return -1; + } + + fd = img_raw_fd(pr->pi); + if (fd < 0) { + pr_err("Getting raw FD failed\n"); + return -1; + } + while (pr->stream_padding) { + char discard[PAGE_PADDING_CHUNK]; + size_t chunk = min(pr->stream_padding, sizeof(discard)); + + ret = read(fd, discard, chunk); + if (ret == 0) { + pr_err("Reached EOF while skipping compressed page-image alignment\n"); + return -1; + } + if (ret < 0) { + if (errno == EINTR) + continue; + pr_perror("Can't skip compressed page-image alignment"); + return -1; + } + pr->stream_padding -= ret; + } + + pr_debug("\tpr%lu-%u Read page from self %lx/%" PRIx64 "\n", + pr->img_id, pr->id, pr->cvaddr, pr->pi_off); + + BUG_ON(pr->cvaddr != vaddr); + + if (pr->compressed_size_index + nr > pr->pe->n_compressed_size) { + pr_err("Compressed size index out of bounds: %zu + %lu > %zu\n", + pr->compressed_size_index, nr, + pr->pe->n_compressed_size); + return -1; + } + + for (i = 0; i < nr; i++) { + size_t idx = pr->compressed_size_index + i; + uint32_t cs = pr->pe->compressed_size[idx]; + size_t rd = 0; + + if (cs > PAGE_COMPRESSED_SIZE_BOUND) { + pr_err("Invalid compressed size %u for page %zu\n", + cs, idx); + return -1; + } + + if (cs == 0) { + /* Zero page, nothing stored in the image */ + memset(buf + curr, 0, PAGE_SIZE); + curr += PAGE_SIZE; + continue; + } + + if (cs == PAGE_SIZE) { + /* Stored raw, read directly into output */ + while (rd < PAGE_SIZE) { + ret = read(fd, buf + curr + rd, PAGE_SIZE - rd); + if (ret == 0) { + pr_err("Reached EOF reading raw page\n"); + return -1; + } else if (ret < 0) { + if (errno == EINTR) + continue; + pr_perror("Can't read raw page"); + return -1; + } + rd += ret; + } + } else { + while (rd < cs) { + /* + * cs is bounded by PAGE_COMPRESSED_SIZE_BOUND + * (validated above), so cs - rd fits in the + * stack buffer. Re-state the bound explicitly + * so the compiler's fortified-read analysis + * does not warn about a possible overflow into + * compressed_buf. + */ + size_t to_read = cs - rd; + + if (to_read > sizeof(compressed_buf) - rd) + to_read = sizeof(compressed_buf) - rd; + + ret = read(fd, compressed_buf + rd, to_read); + if (ret == 0) { + pr_err("Reached EOF reading compressed page\n"); + return -1; + } else if (ret < 0) { + if (errno == EINTR) + continue; + pr_perror("Can't read compressed page"); + return -1; + } + rd += ret; + } + + if (decompress_data(compressed_buf, cs, PAGE_SIZE, buf + curr)) { + pr_err("Decompression failed for page %zu\n", idx); + return -1; + } + } + + curr += PAGE_SIZE; + compressed_offset += cs; + } + + if (pr->io_complete) + ret = pr->io_complete(pr, vaddr, nr); + + pr->pi_off += compressed_offset; + pr->compressed_size_index += nr; + + return ret; +} + static int read_page_complete(unsigned long img_id, unsigned long vaddr, unsigned long int nr_pages, void *priv) { int ret = 0; @@ -544,6 +1402,8 @@ static void drain_async_queue(struct page_read *pr) list_for_each_entry_safe(piov, n, &pr->async, l) { list_del(&piov->l); + xfree(piov->compressed_size); + xfree(piov->block_pages); xfree(piov->to); xfree(piov); } @@ -551,6 +1411,199 @@ static void drain_async_queue(struct page_read *pr) drain_async_queue(pr->parent); } +/* + * Compressed async read: read all compressed + * data in one pread(), then decompress + * block-by-block into the destination iovecs. + * In per-page mode each block is one page; in + * region mode each block is up to region_pages + * pages. + */ +static int process_compressed_async_read(struct page_read *pr, int fd, + struct page_read_iov *piov) +{ + char *comp_buf; + off_t comp_off; + size_t total = piov->total_compressed_size; + int iov_idx = 0; + size_t iov_off = 0; + char *region_scratch = NULL; + size_t region_scratch_cap = 0; + int ret = -1; + + (void)pr; + + /* + * An all-zero batch has total == 0: there is nothing + * to read from the image and every block is rebuilt by + * memset below, so leave comp_buf NULL. xmalloc(0) may + * return NULL, which must not be treated as an error. + * Route a genuine allocation failure through the err + * path so the async queue is drained. + */ + comp_buf = NULL; + if (total) { + comp_buf = xmalloc(total); + if (!comp_buf) + goto out; + + if (pread_full(fd, comp_buf, total, piov->from)) + goto out; + } + + if (piov->region_pages) { + region_scratch_cap = (size_t)piov->region_pages * PAGE_SIZE; + region_scratch = xmalloc(region_scratch_cap); + if (!region_scratch) + goto out; + } + + comp_off = 0; + for (int i = 0; i < piov->n_compressed_size; i++) { + uint32_t cs = piov->compressed_size[i]; + unsigned int block_pages; + size_t block_bytes; + size_t bound; + size_t out_left; + char *src; + + if (piov->region_pages) { + block_pages = piov->block_pages[i]; + bound = REGION_COMPRESSED_SIZE_BOUND(block_pages); + } else { + block_pages = 1; + bound = PAGE_COMPRESSED_SIZE_BOUND; + } + block_bytes = (size_t)block_pages * PAGE_SIZE; + + if (cs > bound) { + pr_err("Async: invalid compressed size %u for block %d\n", + cs, i); + goto out; + } + + /* + * Decompress this block to a contiguous + * source buffer. + * + * Region mode raw blocks (cs == block_bytes) + * skip the staging memcpy and copy directly + * out of comp_buf. Zero blocks (cs == 0) are + * handled by memset on the destination. + * + * Compressed blocks normally decompress into + * region_scratch and we then memcpy into the + * iovecs. As an optimisation, when the whole + * block fits inside a single destination iovec + * (the common case after a contiguous VMA), we + * decompress directly into the iovec and skip + * the staging buffer. + */ + if (piov->region_pages) { + if (cs == 0) { + src = NULL; /* memset on dst */ + } else if (cs == block_bytes) { + src = comp_buf + comp_off; + } else { + /* Locate destination iovec for this block */ + int peek_iov = iov_idx; + size_t peek_off = iov_off; + + while (peek_iov < piov->nr && peek_off >= piov->to[peek_iov].iov_len) { + peek_off -= piov->to[peek_iov].iov_len; + peek_iov++; + } + + if (peek_iov < piov->nr && + peek_off + block_bytes <= + piov->to[peek_iov].iov_len) { + /* Block fits — decompress directly into iovec */ + char *direct = (char *)piov->to[peek_iov].iov_base + peek_off; + + if (decompress_region(comp_buf + comp_off, + cs, block_pages, + direct)) { + pr_err("Async region decompress failed at block %d\n", + i); + goto out; + } + /* Advance iovec cursor and skip the copy loop. */ + iov_off += block_bytes; + comp_off += cs; + continue; + } + + if (decompress_region(comp_buf + comp_off, + cs, block_pages, + region_scratch)) { + pr_err("Async region decompress failed at block %d\n", + i); + goto out; + } + src = region_scratch; + } + } else { + src = NULL; /* handled below */ + } + + /* + * Walk the destination iovec array, + * copying block_bytes worth of decompressed + * data, splitting across iovecs as needed. + */ + out_left = block_bytes; + while (out_left > 0) { + char *dst; + size_t this_chunk; + + while (iov_idx < piov->nr && iov_off >= piov->to[iov_idx].iov_len) { + iov_off -= piov->to[iov_idx].iov_len; + iov_idx++; + } + if (iov_idx >= piov->nr) { + pr_err("Async: ran out of iovs at block %d\n", i); + goto out; + } + + dst = (char *)piov->to[iov_idx].iov_base + iov_off; + this_chunk = piov->to[iov_idx].iov_len - iov_off; + if (this_chunk > out_left) + this_chunk = out_left; + + if (piov->region_pages) { + if (src) { + memcpy(dst, src, this_chunk); + src += this_chunk; + } else { + /* zero block */ + memset(dst, 0, this_chunk); + } + } else { + /* Per-page blocks fit in one page-aligned iovec slice. */ + if (cs == 0) + memset(dst, 0, this_chunk); + else if (cs == PAGE_SIZE) + memcpy(dst, comp_buf + comp_off, this_chunk); + else if (decompress_data(comp_buf + comp_off, cs, PAGE_SIZE, dst)) { + pr_err("Async decompress failed for page %d\n", i); + goto out; + } + } + + iov_off += this_chunk; + out_left -= this_chunk; + } + + comp_off += cs; + } + + ret = 0; +out: + xfree(comp_buf); + xfree(region_scratch); + return ret; +} + static int process_async_reads(struct page_read *pr) { int fd, ret = 0; @@ -575,6 +1628,15 @@ static int process_async_reads(struct page_read *pr) pr_debug("Read piov iovs %d, from %ju, len %ju, first %p:%zu\n", piov->nr, piov->from, piov->end - piov->from, piov->to->iov_base, piov->to->iov_len); + + if (piov->n_compressed_size) { + if (process_compressed_async_read(pr, fd, piov)) { + ret = -1; + goto err; + } + goto next; + } + more: ret = preadv(fd, piov->to, piov->nr, piov->from); if (fault_injected(FI_PARTIAL_PAGES)) { @@ -619,8 +1681,11 @@ static int process_async_reads(struct page_read *pr) goto more; } + next: BUG_ON(pr->io_complete); /* FIXME -- implement once needed */ list_del(&piov->l); + xfree(piov->compressed_size); + xfree(piov->block_pages); xfree(iovs); xfree(piov); } @@ -666,6 +1731,8 @@ static void reset_pagemap(struct page_read *pr) { pr->cvaddr = 0; pr->pi_off = 0; + pr->compressed_size_index = 0; + pr->region_block_offset = 0; pr->curr_pme = -1; pr->pe = NULL; @@ -732,6 +1799,117 @@ static void init_compat_pagemap_entry(PagemapEntry *pe) pe->nr_pages = pe->compat_nr_pages; } +static bool pagemap_entry_has_compressed(PagemapEntry *pe) +{ + return pagemap_present(pe) && pe->n_compressed_size; +} + +static int validate_compressed_pagemap_entry(PagemapEntry *pe) +{ + uint64_t expected_blocks, sum = 0; + unsigned int region_pages = 0; + size_t i; + + if (pe->has_region_pages && pe->region_pages) + region_pages = pe->region_pages; + + if (!pe->n_compressed_size) + return 0; + +#ifndef CONFIG_LZ4 + pr_err("Pagemap contains compressed pages but CRIU was built without LZ4 support (CONFIG_LZ4)\n"); + return -1; +#endif + + if (!pagemap_present(pe)) { + pr_err("Compressed metadata on non-present pagemap entry %#" PRIx64 "\n", + pe->vaddr); + return -1; + } + + if (!pe->nr_pages) { + pr_err("Compressed pagemap entry %#" PRIx64 " has no pages\n", pe->vaddr); + return -1; + } + + if (region_pages) { + if (region_pages > MAX_REGION_PAGES) { + pr_err("Compressed pagemap entry %#" PRIx64 " has invalid region_pages %u\n", + pe->vaddr, region_pages); + return -1; + } + expected_blocks = pe->nr_pages / region_pages; + if (pe->nr_pages % region_pages) + expected_blocks++; + } else { + expected_blocks = pe->nr_pages; + } + + if (expected_blocks > SIZE_MAX || + pe->n_compressed_size != (size_t)expected_blocks) { + pr_err("Compressed pagemap entry %#" PRIx64 " has %zu blocks, expected %" PRIu64 "\n", + pe->vaddr, pe->n_compressed_size, expected_blocks); + return -1; + } + + for (i = 0; i < pe->n_compressed_size; i++) { + uint32_t cs = pe->compressed_size[i]; + size_t block_bytes; + + if (region_pages) { + uint64_t pages_done = (uint64_t)i * region_pages; + uint64_t pages_left = pe->nr_pages - pages_done; + unsigned int block_pages = region_pages; + + if (pages_left < region_pages) + block_pages = (unsigned int)pages_left; + + block_bytes = (size_t)block_pages * PAGE_SIZE; + } else { + block_bytes = PAGE_SIZE; + } + + /* + * The writer stores a block raw once LZ4 no longer saves space, so + * an on-image block can never be larger than its decoded contents. + * Enforce that before any reader uses compressed_size as an I/O size. + */ + if ((size_t)cs > block_bytes) { + pr_err("Compressed pagemap entry %#" PRIx64 + " block %zu size %u exceeds decoded size %zu\n", + pe->vaddr, i, cs, block_bytes); + return -1; + } + + if (sum > UINT64_MAX - cs) { + pr_err("Compressed pagemap entry %#" PRIx64 " size sum overflows\n", + pe->vaddr); + return -1; + } + sum += cs; + } + + if (pe->has_total_compressed_size && pe->total_compressed_size != sum) { + pr_err("Compressed pagemap entry %#" PRIx64 " total size %" PRIu64 " != sum %" PRIu64 "\n", + pe->vaddr, pe->total_compressed_size, sum); + return -1; + } + + return 0; +} + +static bool page_read_has_compressed_entries(struct page_read *pr) +{ + int i; + + for (i = 0; i < pr->nr_pmes; i++) { + if (pagemap_entry_has_compressed(pr->pmes[i])) + return true; + } + + return false; +} + /* * The pagemap entry size is at least 8 bytes for small mappings with * low address and may get to 18 bytes or even more for large mappings @@ -772,15 +1950,19 @@ static int init_pagemaps(struct page_read *pr) pr->curr_pme = -1; while (1) { + PagemapEntry *pe; int ret = pb_read_one_eof(pr->pmi, &pr->pmes[pr->nr_pmes], PB_PAGEMAP); + if (ret < 0) goto free_pagemaps; if (ret == 0) break; - init_compat_pagemap_entry(pr->pmes[pr->nr_pmes]); + pe = pr->pmes[pr->nr_pmes++]; + init_compat_pagemap_entry(pe); + if (validate_compressed_pagemap_entry(pe)) + goto free_pagemaps; - pr->nr_pmes++; if (pr->nr_pmes >= nr_pmes) { PagemapEntry **new; nr_pmes += nr_realloc; @@ -891,6 +2073,8 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p pr->parent = NULL; pr->cvaddr = 0; pr->pi_off = 0; + pr->compressed_size_index = 0; + pr->region_block_offset = 0; pr->bunch.iov_len = 0; pr->bunch.iov_base = NULL; pr->pmes = NULL; @@ -918,10 +2102,24 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p return -1; } + if (init_pagemaps(pr)) { + close_page_read(pr); + return -1; + } + { int pfd = img_raw_fd(pr->pi); - if (pfd >= 0 && !opts.stream && opts.image_io_mode == IMAGE_IO_DIRECT) { + /* + * O_DIRECT requires reads to be aligned in offset and + * length. Compressed pages are stored as variable-length + * blocks packed contiguously, so reads are unaligned and + * O_DIRECT would fail with EINVAL. Use buffered I/O for any + * reader whose own pagemap contains compressed entries; parent + * images may differ from the top inventory's compression mode. + */ + if (pfd >= 0 && !opts.stream && opts.image_io_mode == IMAGE_IO_DIRECT && + !page_read_has_compressed_entries(pr)) { int direct = probe_pages_o_direct(pfd); if (direct < 0) { @@ -932,10 +2130,14 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p } } - if (init_pagemaps(pr)) { - close_page_read(pr); - return -1; - } + /* + * Hint the kernel to use aggressive readahead on the pages + * image. pread() does not advance the file offset, so the + * kernel's sequential-access heuristic may not trigger + * without this hint. + */ + if (img_raw_fd(pr->pi) >= 0) + posix_fadvise(img_raw_fd(pr->pi), 0, 0, POSIX_FADV_SEQUENTIAL); pr->read_pages = read_pagemap_page; pr->advance = advance; @@ -950,11 +2152,14 @@ int open_page_read_at(int dfd, unsigned long img_id, struct page_read *pr, int p if (remote) pr->maybe_read_page = maybe_read_page_remote; + else if (opts.stream && page_read_has_compressed_entries(pr)) + pr->maybe_read_page = maybe_read_page_img_streamer_compressed; else if (opts.stream) pr->maybe_read_page = maybe_read_page_img_streamer; else { - pr->maybe_read_page = maybe_read_page_local; - if (!pr->parent && !opts.lazy_pages) + pr->maybe_read_page = maybe_read_page_local_compressed; + if (!pr->parent && !opts.lazy_pages && + !page_read_has_compressed_entries(pr)) pr->pieok = true; }