mirror of
https://github.com/checkpoint-restore/criu.git
synced 2026-07-17 16:47:50 +00:00
crit: Convert memory page compression offline
CRIT can edit image metadata but cannot convert the matching page payloads as one consistent image set. Changing either side alone can leave pagemap, inventory, and payload data incompatible. Add compress and decompress commands for checkpoint directories. Match CRIU's block encoding, raw threshold, alignment, image version, parent compatibility, and validation rules. Keep hugetlb and external-plugin ranges raw because CRIU cannot premap them generically, and add the Python LZ4 binding to supported dependencies. Stage every output before replacing any image. Preserve ownership, permissions, timestamps, xattrs, ACLs, and security metadata; create exclusive hard-link backups; synchronize directory changes; and defer terminating signals until the transaction has a definite result. Roll back the complete image set after a failure and retain the recoverable source if rollback or cleanup also fails. Reject symlinks, non-regular images, unsupported versions, inconsistent compression metadata, unknown modes, and truncated payloads. Recheck source identities before replacement so a changed pathname is not overwritten. Cover page and region images, parent chains, exceptional mappings, metadata preservation, signals, races, and rollback failures. Assisted-by: Codex:GPT-5 Assisted-by: Claude:claude-fable-5 Signed-off-by: Radostin Stoyanov <rstoyanov@fedoraproject.org>
This commit is contained in:
Radostin Stoyanov
parent
94447379a9
commit
47f14e9469
14 changed files with 3201 additions and 7 deletions
|
|
@ -18,6 +18,10 @@ SYNOPSIS
|
|||
|
||||
*crit* 'show' [-h] in
|
||||
|
||||
*crit* 'compress' [-h] [--in-place] [--acceleration N] dir
|
||||
|
||||
*crit* 'decompress' [-h] [--in-place] dir
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
*crit* is a feature-rich replacement for existing *criu* show.
|
||||
|
|
@ -43,12 +47,46 @@ Positional Arguments
|
|||
*show*::
|
||||
convert *criu* image from binary to human-readable JSON
|
||||
|
||||
*compress*::
|
||||
compress memory page images in a checkpoint directory using CRIU's
|
||||
per-page LZ4 format. The command rewrites *pages-*.img*,
|
||||
*pagemap-*.img*, and *inventory.img*. Pages that are zero-filled or
|
||||
do not meet CRIU's compression threshold are stored without an LZ4
|
||||
payload. By default, the original files are preserved with a *.bak*
|
||||
suffix; use *--in-place* to skip these backup files. The inventory
|
||||
is updated to record compression metadata and the compressed-image
|
||||
version. Existing *.bak* files are never overwritten.
|
||||
|
||||
*decompress*::
|
||||
decompress memory page images in a checkpoint directory back to
|
||||
uncompressed page payloads. The command removes compression metadata
|
||||
from the pagemap and inventory images. When the directory has no parent
|
||||
reference, the command also restores the normal image version. It retains
|
||||
the compressed-image version for incremental checkpoints because a parent
|
||||
may still contain compressed payloads. By default, rewritten files are
|
||||
preserved with a *.bak* suffix; use *--in-place* to skip these backup
|
||||
files. Existing *.bak* files are never overwritten.
|
||||
|
||||
Both transformations preserve the ownership, permissions, ACLs, security
|
||||
labels, and other extended attributes of rewritten images. All output is
|
||||
staged and synchronized before it is installed. If a write, rename, *SIGHUP*,
|
||||
*SIGINT*, or *SIGTERM* interrupts the commit, CRIT restores the complete
|
||||
original image set instead of leaving a partially transformed set.
|
||||
|
||||
Optional Arguments
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
*-h*, *--help*::
|
||||
Print some help and exit
|
||||
|
||||
*--in-place*::
|
||||
Rewrite files without creating *.bak* backup files. This option is
|
||||
accepted by *compress* and *decompress*.
|
||||
|
||||
*--acceleration* 'N'::
|
||||
Set the LZ4 acceleration level used by *compress*. Higher values
|
||||
trade compression ratio for speed.
|
||||
|
||||
SEE ALSO
|
||||
--------
|
||||
criu(8)
|
||||
|
|
|
|||
1
Makefile
1
Makefile
|
|
@ -463,6 +463,7 @@ ruff:
|
|||
test/others/criu-ns/run.py \
|
||||
crit/*.py \
|
||||
crit/crit/*.py \
|
||||
test/others/crit/*.py \
|
||||
scripts/uninstall_module.py \
|
||||
coredump/ coredump/coredump \
|
||||
scripts/github-indent-warnings.py \
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ apk add --no-cache \
|
|||
protobuf-c-dev \
|
||||
protobuf-dev \
|
||||
py3-importlib-metadata \
|
||||
py3-lz4 \
|
||||
py3-pip \
|
||||
py3-protobuf \
|
||||
py3-yaml \
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ fi
|
|||
protobuf-c-compiler \
|
||||
protobuf-compiler \
|
||||
python3-importlib-metadata \
|
||||
python3-lz4 \
|
||||
python3-pip \
|
||||
python3-protobuf \
|
||||
python3-yaml \
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ dnf install -y \
|
|||
protobuf-devel \
|
||||
python-devel \
|
||||
python3-importlib-metadata \
|
||||
python3-lz4 \
|
||||
python3-protobuf \
|
||||
python3-pyyaml \
|
||||
python3-setuptools \
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ pacman -Syu --noconfirm \
|
|||
protobuf \
|
||||
protobuf-c \
|
||||
python-importlib-metadata \
|
||||
python-lz4 \
|
||||
python-pip \
|
||||
python-protobuf \
|
||||
python-yaml \
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -15,6 +15,9 @@ requires-python = ">=3.6"
|
|||
[project.scripts]
|
||||
crit = "crit.__main__:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
compression = ["lz4"]
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["crit"]
|
||||
|
||||
|
|
|
|||
14
test/others/crit/.gitignore
vendored
14
test/others/crit/.gitignore
vendored
|
|
@ -3,3 +3,17 @@
|
|||
*.txt
|
||||
stats-*
|
||||
*.json
|
||||
comp/
|
||||
uncomp/
|
||||
cdc/
|
||||
dcd/
|
||||
sparse-compress/
|
||||
sparse-decompress/
|
||||
threshold-compress/
|
||||
threshold-parent/
|
||||
parent-placeholder/
|
||||
truncated-decompress/
|
||||
region/
|
||||
malformed-*/
|
||||
transaction-compress/
|
||||
transaction-decompress/
|
||||
|
|
|
|||
|
|
@ -4,3 +4,6 @@ run: clean
|
|||
|
||||
clean:
|
||||
rm -f *.img *.log *.txt stats-* *.json
|
||||
rm -rf comp/ uncomp/ cdc/ dcd/ region/ sparse-compress/ sparse-decompress/
|
||||
rm -rf threshold-compress/ threshold-parent/ parent-placeholder/ truncated-decompress/
|
||||
rm -rf malformed-*/ transaction-compress/ transaction-decompress/
|
||||
|
|
|
|||
451
test/others/crit/compression_image_helpers.py
Normal file
451
test/others/crit/compression_image_helpers.py
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import importlib
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
|
||||
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
|
||||
PAGE_COMPRESSION_THRESHOLD = PAGE_SIZE * 7 // 8
|
||||
PE_PARENT = 1 << 0
|
||||
PE_PRESENT = 1 << 2
|
||||
PE_PAYLOAD_ALIGNED = 1 << 3
|
||||
|
||||
|
||||
def pycriu_images():
|
||||
return importlib.import_module("pycriu.images")
|
||||
|
||||
|
||||
def lz4_block():
|
||||
return importlib.import_module("lz4.block")
|
||||
|
||||
|
||||
def assert_inventory_version(directory, expected):
|
||||
images = pycriu_images()
|
||||
with open(os.path.join(directory, "inventory.img"), "rb") as image_file:
|
||||
inventory = images.load(image_file)
|
||||
|
||||
actual = inventory["entries"][0]["img_version"]
|
||||
if actual != expected:
|
||||
print(
|
||||
"FAIL: inventory version is %d, expected %d" % (actual, expected)
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def make_sparse_pagemap_image(directory, compressed):
|
||||
images = pycriu_images()
|
||||
with open(os.path.join(directory, "inventory.img"), "wb") as image_file:
|
||||
inventory = {"magic": "INVENTORY", "entries": [{"img_version": 2}]}
|
||||
if compressed:
|
||||
inventory["entries"][0]["img_version"] = 3
|
||||
inventory["entries"][0]["compress"] = 1
|
||||
images.dump(inventory, image_file)
|
||||
|
||||
if compressed:
|
||||
# One raw present page, one non-present hole, one legacy parent entry
|
||||
# without flags, and one zero present page with no payload.
|
||||
pages = bytes([0x41]) * PAGE_SIZE
|
||||
entries = [
|
||||
{"pages_id": 1},
|
||||
{
|
||||
"vaddr": 0x100000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
"compressed_size": [PAGE_SIZE],
|
||||
"total_compressed_size": PAGE_SIZE,
|
||||
},
|
||||
{
|
||||
"vaddr": 0x101000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": 0,
|
||||
},
|
||||
{
|
||||
"vaddr": 0x102000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"in_parent": True,
|
||||
},
|
||||
{
|
||||
"vaddr": 0x103000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
"compressed_size": [0],
|
||||
"total_compressed_size": 0,
|
||||
},
|
||||
]
|
||||
else:
|
||||
# Two present pages are adjacent in pages.img, while the pagemap has a
|
||||
# non-present hole and a legacy parent reference between them.
|
||||
pages = bytes([0x41]) * PAGE_SIZE + bytes([0x42]) * PAGE_SIZE
|
||||
entries = [
|
||||
{"pages_id": 1},
|
||||
{
|
||||
"vaddr": 0x100000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
},
|
||||
{
|
||||
"vaddr": 0x101000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": 0,
|
||||
},
|
||||
{
|
||||
"vaddr": 0x102000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"in_parent": True,
|
||||
},
|
||||
{
|
||||
"vaddr": 0x103000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
},
|
||||
]
|
||||
|
||||
with open(os.path.join(directory, "pages-1.img"), "wb") as image_file:
|
||||
image_file.write(pages)
|
||||
|
||||
with open(os.path.join(directory, "pagemap-1.img"), "wb") as image_file:
|
||||
images.dump({"magic": "PAGEMAP", "entries": entries}, image_file)
|
||||
return 0
|
||||
|
||||
|
||||
def make_threshold_pagemap_image(directory):
|
||||
images = pycriu_images()
|
||||
block = lz4_block()
|
||||
rng = random.Random(0)
|
||||
random_page = bytes(rng.randrange(256) for _ in range(PAGE_SIZE))
|
||||
page = None
|
||||
|
||||
# Find a deterministic page on the raw side of CRIU's 7/8 threshold. A
|
||||
# fixed-size compressible island is brittle across LZ4 releases because an
|
||||
# incompressible block may legitimately grow by a few different bytes.
|
||||
for zero_bytes in range(1, PAGE_SIZE + 1):
|
||||
trial_page = random_page[:-zero_bytes] + b"\0" * zero_bytes
|
||||
compressed_size = len(
|
||||
block.compress(trial_page, store_size=False, acceleration=1)
|
||||
)
|
||||
if PAGE_COMPRESSION_THRESHOLD <= compressed_size < PAGE_SIZE:
|
||||
page = trial_page
|
||||
break
|
||||
|
||||
if page is None:
|
||||
print("FAIL: unable to construct a page at the raw fallback threshold")
|
||||
return 1
|
||||
|
||||
with open(os.path.join(directory, "inventory.img"), "wb") as image_file:
|
||||
images.dump(
|
||||
{"magic": "INVENTORY", "entries": [{"img_version": 2}]},
|
||||
image_file,
|
||||
)
|
||||
|
||||
with open(os.path.join(directory, "pages-1.img"), "wb") as image_file:
|
||||
image_file.write(page)
|
||||
|
||||
with open(os.path.join(directory, "pagemap-1.img"), "wb") as image_file:
|
||||
images.dump(
|
||||
{
|
||||
"magic": "PAGEMAP",
|
||||
"entries": [
|
||||
{"pages_id": 1},
|
||||
{
|
||||
"vaddr": 0x100000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
},
|
||||
],
|
||||
},
|
||||
image_file,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def make_truncated_uncompressed_entry_image(directory):
|
||||
images = pycriu_images()
|
||||
with open(os.path.join(directory, "inventory.img"), "wb") as image_file:
|
||||
images.dump(
|
||||
{
|
||||
"magic": "INVENTORY",
|
||||
"entries": [{"img_version": 3, "compress": 1}],
|
||||
},
|
||||
image_file,
|
||||
)
|
||||
|
||||
with open(os.path.join(directory, "pages-1.img"), "wb") as image_file:
|
||||
image_file.write(bytes([0x41]) * PAGE_SIZE)
|
||||
|
||||
with open(os.path.join(directory, "pagemap-1.img"), "wb") as image_file:
|
||||
images.dump(
|
||||
{
|
||||
"magic": "PAGEMAP",
|
||||
"entries": [
|
||||
{"pages_id": 1},
|
||||
{
|
||||
"vaddr": 0x100000,
|
||||
"compat_nr_pages": 2,
|
||||
"nr_pages": 2,
|
||||
"flags": PE_PRESENT,
|
||||
},
|
||||
],
|
||||
},
|
||||
image_file,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def assert_pagemap_entry_uncompressed(directory):
|
||||
images = pycriu_images()
|
||||
with open(os.path.join(directory, "pagemap-1.img"), "rb") as image_file:
|
||||
pagemap = images.load(image_file)
|
||||
|
||||
entry = pagemap["entries"][1]
|
||||
for field in ("compressed_size", "total_compressed_size", "region_pages"):
|
||||
if field in entry:
|
||||
print("FAIL: raw-only entry retained %s" % field)
|
||||
return 1
|
||||
if not int(entry.get("flags", 0)) & PE_PAYLOAD_ALIGNED:
|
||||
print("FAIL: raw-only entry lacks PE_PAYLOAD_ALIGNED")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def assert_region_pagemap(directory):
|
||||
images = pycriu_images()
|
||||
found_region = False
|
||||
found_short_region = False
|
||||
for path in glob.glob(os.path.join(directory, "pagemap-*.img")):
|
||||
with open(path, "rb") as image_file:
|
||||
pagemap = images.load(image_file)
|
||||
for entry in pagemap["entries"][1:]:
|
||||
region_pages = entry.get("region_pages", 0)
|
||||
if region_pages > 1:
|
||||
found_region = True
|
||||
if entry.get("nr_pages", entry["compat_nr_pages"]) % region_pages:
|
||||
found_short_region = True
|
||||
if found_region and found_short_region:
|
||||
break
|
||||
if not found_region:
|
||||
print("FAIL: no region-compressed pagemap entry found")
|
||||
return 1
|
||||
if not found_short_region:
|
||||
print("FAIL: no short final compression region found")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def write_pair(directory, images, pages_id, entry, payload):
|
||||
with open(
|
||||
os.path.join(directory, "pages-%d.img" % pages_id), "wb"
|
||||
) as image_file:
|
||||
image_file.write(payload)
|
||||
with open(
|
||||
os.path.join(directory, "pagemap-%d.img" % pages_id), "wb"
|
||||
) as image_file:
|
||||
images.dump(
|
||||
{
|
||||
"magic": "PAGEMAP",
|
||||
"entries": [{"pages_id": pages_id}, entry],
|
||||
},
|
||||
image_file,
|
||||
)
|
||||
|
||||
|
||||
def make_malformed_compressed_image(directory, kind):
|
||||
images = pycriu_images()
|
||||
with open(os.path.join(directory, "inventory.img"), "wb") as image_file:
|
||||
images.dump(
|
||||
{
|
||||
"magic": "INVENTORY",
|
||||
"entries": [{"img_version": 3, "compress": 1}],
|
||||
},
|
||||
image_file,
|
||||
)
|
||||
|
||||
base = {
|
||||
"vaddr": 0x100000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
}
|
||||
|
||||
if kind == "oversize":
|
||||
base["compressed_size"] = [PAGE_SIZE + 1]
|
||||
base["total_compressed_size"] = PAGE_SIZE + 1
|
||||
write_pair(directory, images, 1, base, b"A" * (PAGE_SIZE + 1))
|
||||
elif kind == "total":
|
||||
base["compressed_size"] = [PAGE_SIZE]
|
||||
base["total_compressed_size"] = PAGE_SIZE - 1
|
||||
write_pair(directory, images, 1, base, b"A" * PAGE_SIZE)
|
||||
elif kind == "count":
|
||||
base["compat_nr_pages"] = 2
|
||||
base["nr_pages"] = 2
|
||||
base["compressed_size"] = [PAGE_SIZE]
|
||||
base["total_compressed_size"] = PAGE_SIZE
|
||||
write_pair(directory, images, 1, base, b"A" * PAGE_SIZE)
|
||||
elif kind == "flags":
|
||||
base["flags"] = PE_PRESENT | PE_PARENT
|
||||
base["compressed_size"] = [PAGE_SIZE]
|
||||
base["total_compressed_size"] = PAGE_SIZE
|
||||
write_pair(directory, images, 1, base, b"A" * PAGE_SIZE)
|
||||
elif kind == "aligned-nonpresent":
|
||||
base["flags"] = PE_PAYLOAD_ALIGNED
|
||||
write_pair(directory, images, 1, base, b"")
|
||||
elif kind == "region-limit":
|
||||
base["region_pages"] = 4 * 1024 * 1024 // PAGE_SIZE + 1
|
||||
base["compressed_size"] = [PAGE_SIZE]
|
||||
base["total_compressed_size"] = PAGE_SIZE
|
||||
write_pair(directory, images, 1, base, b"A" * PAGE_SIZE)
|
||||
elif kind == "region-count":
|
||||
base["compat_nr_pages"] = 17
|
||||
base["nr_pages"] = 17
|
||||
base["region_pages"] = 16
|
||||
base["compressed_size"] = [PAGE_SIZE]
|
||||
base["total_compressed_size"] = PAGE_SIZE
|
||||
write_pair(directory, images, 1, base, b"A" * PAGE_SIZE)
|
||||
elif kind == "region-final-oversize":
|
||||
base["compat_nr_pages"] = 17
|
||||
base["nr_pages"] = 17
|
||||
base["region_pages"] = 16
|
||||
base["compressed_size"] = [0, PAGE_SIZE + 1]
|
||||
base["total_compressed_size"] = PAGE_SIZE + 1
|
||||
write_pair(directory, images, 1, base, b"A" * (PAGE_SIZE + 1))
|
||||
elif kind == "transaction":
|
||||
first = dict(base)
|
||||
first["compressed_size"] = [PAGE_SIZE]
|
||||
first["total_compressed_size"] = PAGE_SIZE
|
||||
write_pair(directory, images, 1, first, b"A" * PAGE_SIZE)
|
||||
|
||||
second = dict(base)
|
||||
second["vaddr"] = 0x200000
|
||||
second["compressed_size"] = [8]
|
||||
second["total_compressed_size"] = 8
|
||||
write_pair(directory, images, 2, second, b"not-lz4!")
|
||||
else:
|
||||
raise ValueError("unknown malformed-image kind %s" % kind)
|
||||
return 0
|
||||
|
||||
|
||||
def make_transactional_compress_image(directory):
|
||||
images = pycriu_images()
|
||||
with open(os.path.join(directory, "inventory.img"), "wb") as image_file:
|
||||
images.dump(
|
||||
{"magic": "INVENTORY", "entries": [{"img_version": 2}]},
|
||||
image_file,
|
||||
)
|
||||
|
||||
for pages_id, size in ((1, PAGE_SIZE), (2, PAGE_SIZE - 1)):
|
||||
with open(
|
||||
os.path.join(directory, "pages-%d.img" % pages_id), "wb"
|
||||
) as image_file:
|
||||
image_file.write(bytes([0x40 + pages_id]) * size)
|
||||
with open(
|
||||
os.path.join(directory, "pagemap-%d.img" % pages_id), "wb"
|
||||
) as image_file:
|
||||
images.dump(
|
||||
{
|
||||
"magic": "PAGEMAP",
|
||||
"entries": [
|
||||
{"pages_id": pages_id},
|
||||
{
|
||||
"vaddr": pages_id * 0x100000,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
},
|
||||
],
|
||||
},
|
||||
image_file,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def expected_sparse_checksum(kind):
|
||||
if kind == "raw-zero":
|
||||
contents = bytes([0x41]) * PAGE_SIZE + bytes(PAGE_SIZE)
|
||||
elif kind == "two-raw":
|
||||
contents = bytes([0x41]) * PAGE_SIZE + bytes([0x42]) * PAGE_SIZE
|
||||
else:
|
||||
raise ValueError("unknown checksum kind %s" % kind)
|
||||
print(hashlib.md5(contents).hexdigest())
|
||||
return 0
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
inventory_parser = subparsers.add_parser("assert-inventory-version")
|
||||
inventory_parser.add_argument("directory")
|
||||
inventory_parser.add_argument("expected", type=int)
|
||||
|
||||
sparse_parser = subparsers.add_parser("make-sparse-pagemap")
|
||||
sparse_parser.add_argument("directory")
|
||||
sparse_parser.add_argument("compressed", type=int, choices=(0, 1))
|
||||
|
||||
threshold_parser = subparsers.add_parser("make-threshold-pagemap")
|
||||
threshold_parser.add_argument("directory")
|
||||
|
||||
truncated_parser = subparsers.add_parser("make-truncated-uncompressed")
|
||||
truncated_parser.add_argument("directory")
|
||||
|
||||
raw_parser = subparsers.add_parser("assert-entry-uncompressed")
|
||||
raw_parser.add_argument("directory")
|
||||
|
||||
region_parser = subparsers.add_parser("assert-region-pagemap")
|
||||
region_parser.add_argument("directory")
|
||||
|
||||
malformed_parser = subparsers.add_parser("make-malformed-compressed")
|
||||
malformed_parser.add_argument("directory")
|
||||
malformed_parser.add_argument("kind")
|
||||
|
||||
transaction_parser = subparsers.add_parser("make-transactional-compress")
|
||||
transaction_parser.add_argument("directory")
|
||||
|
||||
checksum_parser = subparsers.add_parser("expected-sparse-checksum")
|
||||
checksum_parser.add_argument("kind", choices=("raw-zero", "two-raw"))
|
||||
|
||||
subparsers.add_parser("has-lz4")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if args.command == "assert-inventory-version":
|
||||
return assert_inventory_version(args.directory, args.expected)
|
||||
if args.command == "make-sparse-pagemap":
|
||||
return make_sparse_pagemap_image(args.directory, bool(args.compressed))
|
||||
if args.command == "make-threshold-pagemap":
|
||||
return make_threshold_pagemap_image(args.directory)
|
||||
if args.command == "make-truncated-uncompressed":
|
||||
return make_truncated_uncompressed_entry_image(args.directory)
|
||||
if args.command == "assert-entry-uncompressed":
|
||||
return assert_pagemap_entry_uncompressed(args.directory)
|
||||
if args.command == "assert-region-pagemap":
|
||||
return assert_region_pagemap(args.directory)
|
||||
if args.command == "make-malformed-compressed":
|
||||
return make_malformed_compressed_image(args.directory, args.kind)
|
||||
if args.command == "make-transactional-compress":
|
||||
return make_transactional_compress_image(args.directory)
|
||||
if args.command == "expected-sparse-checksum":
|
||||
return expected_sparse_checksum(args.kind)
|
||||
if args.command == "has-lz4":
|
||||
lz4_block()
|
||||
return 0
|
||||
raise AssertionError("unhandled command %s" % args.command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
419
test/others/crit/crit_compression_tests.py
Normal file
419
test/others/crit/crit_compression_tests.py
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import glob
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
|
||||
import lz4.block
|
||||
import pycriu
|
||||
from crit import __main__ as crit_main
|
||||
|
||||
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
|
||||
PE_PRESENT = 1 << 2
|
||||
MAP_SHARED = 1 << 0
|
||||
MAP_PRIVATE = 1 << 1
|
||||
MAP_ANON = 0x20
|
||||
MAP_HUGETLB = 0x40000
|
||||
VMA_AREA_REGULAR = 1 << 0
|
||||
VMA_ANON_SHARED = 1 << 8
|
||||
VMA_ANON_PRIVATE = 1 << 9
|
||||
VMA_EXT_PLUGIN = 1 << 27
|
||||
|
||||
|
||||
def write_image(directory, name, image):
|
||||
with open(os.path.join(directory, name), "wb") as image_file:
|
||||
pycriu.images.dump(image, image_file)
|
||||
|
||||
|
||||
def inventory(directory, compress=None, image_version=2):
|
||||
entry = {"img_version": image_version}
|
||||
if compress is not None:
|
||||
entry["compress"] = compress
|
||||
write_image(
|
||||
directory,
|
||||
"inventory.img",
|
||||
{"magic": "INVENTORY", "entries": [entry]},
|
||||
)
|
||||
|
||||
|
||||
def pagemap(directory, image_id, pages_id, entries, shared=False):
|
||||
prefix = "pagemap-shmem-" if shared else "pagemap-"
|
||||
write_image(
|
||||
directory,
|
||||
"%s%d.img" % (prefix, image_id),
|
||||
{
|
||||
"magic": "PAGEMAP",
|
||||
"entries": [{"pages_id": pages_id}] + entries,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def vma(start, page_count, flags, status, shmid=0, pgoff=0):
|
||||
return {
|
||||
"start": start,
|
||||
"end": start + page_count * PAGE_SIZE,
|
||||
"pgoff": pgoff,
|
||||
"shmid": shmid,
|
||||
"prot": 3,
|
||||
"flags": flags,
|
||||
"status": status,
|
||||
"fd": -1,
|
||||
}
|
||||
|
||||
|
||||
def mm(directory, task_id, vmas):
|
||||
entry = {
|
||||
"mm_start_code": 0,
|
||||
"mm_end_code": 0,
|
||||
"mm_start_data": 0,
|
||||
"mm_end_data": 0,
|
||||
"mm_start_stack": 0,
|
||||
"mm_start_brk": 0,
|
||||
"mm_brk": 0,
|
||||
"mm_arg_start": 0,
|
||||
"mm_arg_end": 0,
|
||||
"mm_env_start": 0,
|
||||
"mm_env_end": 0,
|
||||
"exe_file_id": 0,
|
||||
"vmas": vmas,
|
||||
}
|
||||
write_image(
|
||||
directory,
|
||||
"mm-%d.img" % task_id,
|
||||
{"magic": "MM", "entries": [entry]},
|
||||
)
|
||||
|
||||
|
||||
def load(directory, name):
|
||||
with open(os.path.join(directory, name), "rb") as image_file:
|
||||
return pycriu.images.load(image_file)
|
||||
|
||||
|
||||
def image_bytes(directory):
|
||||
contents = {}
|
||||
for path in glob.glob(os.path.join(directory, "*.img")):
|
||||
with open(path, "rb") as image_file:
|
||||
contents[path] = image_file.read()
|
||||
return contents
|
||||
|
||||
|
||||
def ordinary_checkpoint(directory, page, task_id=1):
|
||||
inventory(directory)
|
||||
with open(os.path.join(directory, "pages-1.img"), "wb") as pages:
|
||||
pages.write(page)
|
||||
base = 0x100000
|
||||
pagemap(
|
||||
directory,
|
||||
task_id,
|
||||
1,
|
||||
[
|
||||
{
|
||||
"vaddr": base,
|
||||
"compat_nr_pages": 1,
|
||||
"nr_pages": 1,
|
||||
"flags": PE_PRESENT,
|
||||
}
|
||||
],
|
||||
)
|
||||
mm(
|
||||
directory,
|
||||
task_id,
|
||||
[
|
||||
vma(
|
||||
base,
|
||||
1,
|
||||
MAP_PRIVATE | MAP_ANON,
|
||||
VMA_AREA_REGULAR | VMA_ANON_PRIVATE,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_sign_extended_ranges():
|
||||
# Native 32-bit x86 pagemaps sign-extend pointers while mm VMA bounds do
|
||||
# not.
|
||||
sign_extended_ranges = [(0x90000000, 0x90000000 + PAGE_SIZE)]
|
||||
assert crit_main._address_in_ranges(
|
||||
0xFFFFFFFF90000000,
|
||||
sign_extended_ranges,
|
||||
crit_main._range_starts(sign_extended_ranges),
|
||||
)
|
||||
|
||||
|
||||
def test_exceptional_mappings_and_timestamps():
|
||||
# Private hugetlb and external-plugin ranges use task virtual addresses;
|
||||
# shared hugetlb ranges use shmid and pgoff in pagemap-shmem images. Verify
|
||||
# both translations and ensure only the ordinary pages become LZ4 blocks.
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
inventory(directory)
|
||||
task_base = 0x100000
|
||||
task_pages = b"A" * PAGE_SIZE + b"B" * PAGE_SIZE + b"C" * PAGE_SIZE
|
||||
shared_pages = b"D" * PAGE_SIZE + b"E" * PAGE_SIZE
|
||||
with open(os.path.join(directory, "pages-1.img"), "wb") as pages:
|
||||
pages.write(task_pages)
|
||||
with open(os.path.join(directory, "pages-2.img"), "wb") as pages:
|
||||
pages.write(shared_pages)
|
||||
pagemap(
|
||||
directory,
|
||||
100,
|
||||
1,
|
||||
[
|
||||
{
|
||||
"vaddr": task_base,
|
||||
"compat_nr_pages": 3,
|
||||
"nr_pages": 3,
|
||||
"flags": PE_PRESENT,
|
||||
}
|
||||
],
|
||||
)
|
||||
pagemap(
|
||||
directory,
|
||||
77,
|
||||
2,
|
||||
[
|
||||
{
|
||||
"vaddr": 0,
|
||||
"compat_nr_pages": 2,
|
||||
"nr_pages": 2,
|
||||
"flags": PE_PRESENT,
|
||||
}
|
||||
],
|
||||
shared=True,
|
||||
)
|
||||
mm(
|
||||
directory,
|
||||
100,
|
||||
[
|
||||
vma(
|
||||
task_base,
|
||||
1,
|
||||
MAP_PRIVATE | MAP_ANON | MAP_HUGETLB,
|
||||
VMA_AREA_REGULAR | VMA_ANON_PRIVATE,
|
||||
),
|
||||
vma(
|
||||
task_base + PAGE_SIZE,
|
||||
1,
|
||||
MAP_PRIVATE | MAP_ANON,
|
||||
VMA_AREA_REGULAR | VMA_ANON_PRIVATE | VMA_EXT_PLUGIN,
|
||||
),
|
||||
vma(
|
||||
task_base + 2 * PAGE_SIZE,
|
||||
1,
|
||||
MAP_PRIVATE | MAP_ANON,
|
||||
VMA_AREA_REGULAR | VMA_ANON_PRIVATE,
|
||||
),
|
||||
vma(
|
||||
0x200000,
|
||||
1,
|
||||
MAP_SHARED | MAP_ANON | MAP_HUGETLB,
|
||||
VMA_AREA_REGULAR | VMA_ANON_SHARED,
|
||||
shmid=77,
|
||||
),
|
||||
vma(
|
||||
0x300000,
|
||||
1,
|
||||
MAP_SHARED | MAP_ANON,
|
||||
VMA_AREA_REGULAR | VMA_ANON_SHARED,
|
||||
shmid=77,
|
||||
pgoff=PAGE_SIZE,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Make source atime older than mtime so an ordinary read would change
|
||||
# it.
|
||||
original_times = {}
|
||||
for index, path in enumerate(
|
||||
sorted(glob.glob(os.path.join(directory, "*.img")))
|
||||
):
|
||||
times = (946684800123456789 + index, 978307200987654321 + index)
|
||||
os.utime(path, ns=times)
|
||||
before = os.stat(path)
|
||||
original_times[path] = (before.st_atime_ns, before.st_mtime_ns)
|
||||
|
||||
assert (
|
||||
crit_main.compress_cmd(
|
||||
{"dir": directory, "in_place": True, "acceleration": 1}
|
||||
)
|
||||
== 0
|
||||
)
|
||||
for path, times in original_times.items():
|
||||
after = os.stat(path)
|
||||
assert (after.st_atime_ns, after.st_mtime_ns) == times, path
|
||||
|
||||
task_pm = load(directory, "pagemap-100.img")["entries"][1]
|
||||
shared_pm = load(directory, "pagemap-shmem-77.img")["entries"][1]
|
||||
assert task_pm["compressed_size"][:2] == [PAGE_SIZE, PAGE_SIZE]
|
||||
assert 0 < task_pm["compressed_size"][2] < PAGE_SIZE
|
||||
assert shared_pm["compressed_size"][0] == PAGE_SIZE
|
||||
assert 0 < shared_pm["compressed_size"][1] < PAGE_SIZE
|
||||
|
||||
decompressed_times = {}
|
||||
for index, path in enumerate(
|
||||
sorted(glob.glob(os.path.join(directory, "*.img")))
|
||||
):
|
||||
times = (1009843200123456789 + index, 1041379200987654321 + index)
|
||||
os.utime(path, ns=times)
|
||||
before = os.stat(path)
|
||||
decompressed_times[path] = (before.st_atime_ns, before.st_mtime_ns)
|
||||
|
||||
assert (
|
||||
crit_main.decompress_cmd({"dir": directory, "in_place": True}) == 0
|
||||
)
|
||||
for path, times in decompressed_times.items():
|
||||
after = os.stat(path)
|
||||
assert (after.st_atime_ns, after.st_mtime_ns) == times, path
|
||||
with open(os.path.join(directory, "pages-1.img"), "rb") as pages:
|
||||
assert pages.read() == task_pages
|
||||
with open(os.path.join(directory, "pages-2.img"), "rb") as pages:
|
||||
assert pages.read() == shared_pages
|
||||
|
||||
|
||||
def test_acceleration():
|
||||
# This page is encoded differently by fast-mode acceleration 1 and 100.
|
||||
# Comparing CRIT output to python-lz4 catches accidentally passing an
|
||||
# ignored acceleration argument to the library's default compression mode.
|
||||
rng = random.Random(1238)
|
||||
tokens = [bytes(rng.randrange(256) for _ in range(4)) for _ in range(128)]
|
||||
page = b"".join(
|
||||
tokens[(index * 17 + index // 7) % len(tokens)]
|
||||
for index in range(PAGE_SIZE // 4)
|
||||
)
|
||||
expected = {
|
||||
acceleration: lz4.block.compress(
|
||||
page,
|
||||
mode="fast",
|
||||
store_size=False,
|
||||
acceleration=acceleration,
|
||||
)
|
||||
for acceleration in (1, 100)
|
||||
}
|
||||
assert expected[1] != expected[100]
|
||||
|
||||
for acceleration in (1, 100):
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
ordinary_checkpoint(directory, page)
|
||||
assert (
|
||||
crit_main.compress_cmd(
|
||||
{
|
||||
"dir": directory,
|
||||
"in_place": True,
|
||||
"acceleration": acceleration,
|
||||
}
|
||||
)
|
||||
== 0
|
||||
)
|
||||
pagemap_entry = load(directory, "pagemap-1.img")["entries"][1]
|
||||
encoded = expected[acceleration]
|
||||
if len(encoded) >= crit_main.PAGE_COMPRESSION_THRESHOLD:
|
||||
assert "compressed_size" not in pagemap_entry
|
||||
with open(os.path.join(directory, "pages-1.img"), "rb") as pages:
|
||||
assert pages.read() == page
|
||||
else:
|
||||
assert pagemap_entry["compressed_size"] == [len(encoded)]
|
||||
with open(os.path.join(directory, "pages-1.img"), "rb") as pages:
|
||||
assert pages.read() == encoded
|
||||
|
||||
|
||||
def test_unknown_compression_mode():
|
||||
# Unknown enum values must be rejected before the "already compressed"
|
||||
# no-op and before any temporary or replacement image is created.
|
||||
for command in ("compress", "decompress"):
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
ordinary_checkpoint(directory, b"F" * PAGE_SIZE)
|
||||
inventory_image = load(directory, "inventory.img")
|
||||
inventory_image["entries"][0]["compress"] = 0xFFFFFFFF
|
||||
write_image(directory, "inventory.img", inventory_image)
|
||||
before = image_bytes(directory)
|
||||
assert run_transform(command, directory) == 1
|
||||
assert image_bytes(directory) == before
|
||||
assert not glob.glob(os.path.join(directory, ".*.crit-*"))
|
||||
|
||||
|
||||
def run_transform(command, directory):
|
||||
if command == "compress":
|
||||
return crit_main.compress_cmd(
|
||||
{"dir": directory, "in_place": True, "acceleration": 1}
|
||||
)
|
||||
return crit_main.decompress_cmd({"dir": directory, "in_place": True})
|
||||
|
||||
|
||||
def test_unsupported_image_version():
|
||||
for image_version in (1, 4):
|
||||
for command in ("compress", "decompress"):
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
ordinary_checkpoint(directory, b"V" * PAGE_SIZE)
|
||||
inventory_image = load(directory, "inventory.img")
|
||||
inventory_image["entries"][0]["img_version"] = image_version
|
||||
write_image(directory, "inventory.img", inventory_image)
|
||||
before = image_bytes(directory)
|
||||
|
||||
assert run_transform(command, directory) == 1
|
||||
assert image_bytes(directory) == before
|
||||
assert not glob.glob(os.path.join(directory, ".*.crit-*"))
|
||||
|
||||
|
||||
def test_compression_requires_v1_2_inventory():
|
||||
for command in ("compress", "decompress"):
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
ordinary_checkpoint(directory, b"C" * PAGE_SIZE)
|
||||
inventory_image = load(directory, "inventory.img")
|
||||
inventory_image["entries"][0]["compress"] = 1
|
||||
write_image(directory, "inventory.img", inventory_image)
|
||||
before = image_bytes(directory)
|
||||
|
||||
assert run_transform(command, directory) == 1
|
||||
assert image_bytes(directory) == before
|
||||
assert not glob.glob(os.path.join(directory, ".*.crit-*"))
|
||||
|
||||
|
||||
def test_uncompressed_v1_2_inventory_is_valid():
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
ordinary_checkpoint(directory, b"U" * PAGE_SIZE)
|
||||
inventory_image = load(directory, "inventory.img")
|
||||
inventory_image["entries"][0]["img_version"] = 3
|
||||
write_image(directory, "inventory.img", inventory_image)
|
||||
before = image_bytes(directory)
|
||||
|
||||
assert run_transform("decompress", directory) == 0
|
||||
assert image_bytes(directory) == before
|
||||
|
||||
|
||||
def test_symlink_rejection():
|
||||
# Replacing an image symlink would sever it while transforming its target's
|
||||
# bytes with the symlink's metadata. Reject non-regular image paths instead.
|
||||
with tempfile.TemporaryDirectory(dir=".") as directory:
|
||||
ordinary_checkpoint(directory, b"G" * PAGE_SIZE)
|
||||
inventory_path = os.path.join(directory, "inventory.img")
|
||||
inventory_target = os.path.join(directory, "inventory.real")
|
||||
os.replace(inventory_path, inventory_target)
|
||||
os.symlink("inventory.real", inventory_path)
|
||||
with open(inventory_target, "rb") as inventory_file:
|
||||
before = inventory_file.read()
|
||||
assert (
|
||||
crit_main.compress_cmd(
|
||||
{"dir": directory, "in_place": True, "acceleration": 1}
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert os.path.islink(inventory_path)
|
||||
with open(inventory_target, "rb") as inventory_file:
|
||||
assert inventory_file.read() == before
|
||||
assert not glob.glob(os.path.join(directory, ".*.crit-*"))
|
||||
|
||||
|
||||
def main():
|
||||
test_sign_extended_ranges()
|
||||
test_exceptional_mappings_and_timestamps()
|
||||
test_acceleration()
|
||||
test_unknown_compression_mode()
|
||||
test_unsupported_image_version()
|
||||
test_compression_requires_v1_2_inventory()
|
||||
test_uncompressed_v1_2_inventory_is_valid()
|
||||
test_symlink_rejection()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
480
test/others/crit/crit_transaction_tests.py
Normal file
480
test/others/crit/crit_transaction_tests.py
Normal file
|
|
@ -0,0 +1,480 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Crash and race scenarios for CRIT's transactional image replacement.
|
||||
|
||||
crit compress/decompress rewrite pages-*.img, pagemap-*.img, and
|
||||
inventory.img as one transaction (_commit_staged() and friends in
|
||||
crit.__main__): replacements are staged next to the originals, the
|
||||
originals are retained through hard links, and any failure or
|
||||
termination signal before the whole set is durable must put every
|
||||
original image back. Each test sabotages one step of that commit and
|
||||
asserts the directory ends in one of the two legal states: the complete
|
||||
old image set or the complete new one.
|
||||
|
||||
Faults are injected with intercept(), which wraps a function in the os
|
||||
module for the duration of a test. crit resolves os functions at call
|
||||
time through the same module object this test imports, so the wrapper
|
||||
sees exactly the calls made by the code under test -- and also this
|
||||
module's own os calls inside the window, which is why it must always
|
||||
forward to the real function saved before patching.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import glob
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock as mock
|
||||
from contextlib import contextmanager, redirect_stderr
|
||||
|
||||
from crit import __main__ as crit_main
|
||||
|
||||
# Transaction files created next to the images: staged replacements
|
||||
# (".NAME.crit-XXXX") and rollback hard links (".NAME.crit-rollback-XXXX").
|
||||
# The name patterns come from _staged_file() and _make_rollback_link().
|
||||
TRANSACTION_GLOB = ".*.crit-*"
|
||||
ROLLBACK_GLOB = ".*.crit-rollback-*"
|
||||
|
||||
TERMINATION_SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM)
|
||||
|
||||
|
||||
def write(path, data):
|
||||
with open(path, "wb") as output:
|
||||
output.write(data)
|
||||
|
||||
|
||||
def read(path):
|
||||
with open(path, "rb") as source:
|
||||
return source.read()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def intercept(name, when, before=None, after=None):
|
||||
"""Wrap os.<name> so faults fire on calls matching when(*args).
|
||||
|
||||
For a matching call, before(*args) runs first (raise from it to
|
||||
suppress the real call), then the real function, then after(*args).
|
||||
Calls not matching when() pass straight through.
|
||||
"""
|
||||
real = getattr(os, name)
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
if not when(*args, **kwargs):
|
||||
return real(*args, **kwargs)
|
||||
if before is not None:
|
||||
before(*args, **kwargs)
|
||||
result = real(*args, **kwargs)
|
||||
if after is not None:
|
||||
after(*args, **kwargs)
|
||||
return result
|
||||
|
||||
with mock.patch.object(crit_main.os, name, new=wrapper):
|
||||
yield
|
||||
|
||||
|
||||
def raise_(exc):
|
||||
"""intercept() hook: fail the matching call with exc."""
|
||||
def effect(*args, **kwargs):
|
||||
raise exc
|
||||
return effect
|
||||
|
||||
|
||||
def send_signal(signum):
|
||||
"""intercept() hook: deliver signum to this process."""
|
||||
def effect(*args, **kwargs):
|
||||
os.kill(os.getpid(), signum)
|
||||
return effect
|
||||
|
||||
|
||||
class TransactionTestCase(unittest.TestCase):
|
||||
"""Shared fixture: a directory of original images and staged updates."""
|
||||
|
||||
def make_images(self, count=1):
|
||||
"""Create count originals ("old-N") with staged updates ("new-N").
|
||||
|
||||
Returns (directory, staged) where staged is the [(path, tmp_path)]
|
||||
list _commit_staged() consumes.
|
||||
"""
|
||||
directory = tempfile.mkdtemp(dir=".")
|
||||
self.addCleanup(shutil.rmtree, directory, ignore_errors=True)
|
||||
staged = []
|
||||
for index in range(count):
|
||||
path = os.path.join(directory, "image-%d" % index)
|
||||
staged_path = path + ".new"
|
||||
write(path, b"old-%d" % index)
|
||||
write(staged_path, b"new-%d" % index)
|
||||
staged.append((path, staged_path))
|
||||
return directory, staged
|
||||
|
||||
def assertOriginalsIntact(self, staged):
|
||||
for index, (path, _) in enumerate(staged):
|
||||
self.assertEqual(read(path), b"old-%d" % index)
|
||||
|
||||
def assertCommitted(self, staged):
|
||||
for index, (path, _) in enumerate(staged):
|
||||
self.assertEqual(read(path), b"new-%d" % index)
|
||||
|
||||
def assertNoTransactionFiles(self, directory):
|
||||
self.assertEqual(glob.glob(os.path.join(directory,
|
||||
TRANSACTION_GLOB)), [])
|
||||
|
||||
|
||||
class CommitRollbackTests(TransactionTestCase):
|
||||
"""A failed commit must restore the complete original image set."""
|
||||
|
||||
def test_failed_rename_restores_originals(self):
|
||||
"""A rename failure on a later image rolls back earlier renames."""
|
||||
directory, staged = self.make_images(count=2)
|
||||
second_staged = staged[1][1]
|
||||
|
||||
with intercept("replace",
|
||||
when=lambda src, dst: src == second_staged,
|
||||
before=raise_(OSError(
|
||||
"injected second-image rename failure"))):
|
||||
with self.assertRaisesRegex(crit_main.CritTransformError,
|
||||
"original images were restored"):
|
||||
crit_main._commit_staged(staged, True)
|
||||
|
||||
self.assertOriginalsIntact(staged)
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
def test_termination_signal_rolls_back(self):
|
||||
"""A signal between renames rolls back and is named in the error."""
|
||||
for signum in TERMINATION_SIGNALS:
|
||||
if signal.getsignal(signum) == signal.SIG_IGN:
|
||||
# _commit_staged() leaves ignored signals ignored, so an
|
||||
# injected kill would not interrupt anything.
|
||||
continue
|
||||
with self.subTest(signal=signal.Signals(signum).name):
|
||||
directory, staged = self.make_images(count=2)
|
||||
second_staged = staged[1][1]
|
||||
|
||||
with intercept("replace",
|
||||
when=lambda src, dst, marker=second_staged:
|
||||
src == marker,
|
||||
after=send_signal(signum)):
|
||||
with self.assertRaisesRegex(
|
||||
crit_main.CritTransformError,
|
||||
signal.Signals(signum).name):
|
||||
crit_main._commit_staged(staged, True)
|
||||
|
||||
self.assertOriginalsIntact(staged)
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
def test_failed_rollback_preserves_recovery_link(self):
|
||||
"""If rollback also fails, the original bytes survive in a named
|
||||
recovery link and the error reports the double failure."""
|
||||
directory, staged = self.make_images(count=1)
|
||||
staged_path = staged[0][1]
|
||||
|
||||
with intercept("replace",
|
||||
when=lambda src, dst: "crit-rollback" in src,
|
||||
before=raise_(OSError("injected rollback failure"))), \
|
||||
intercept("replace",
|
||||
when=lambda src, dst: src == staged_path,
|
||||
after=raise_(OSError("injected failure after rename"))):
|
||||
with self.assertRaisesRegex(crit_main.CritTransformError,
|
||||
"rollback also failed"):
|
||||
crit_main._commit_staged(staged, True)
|
||||
|
||||
recovery = glob.glob(os.path.join(directory, ROLLBACK_GLOB))
|
||||
self.assertEqual(len(recovery), 1)
|
||||
self.assertEqual(read(recovery[0]), b"old-0")
|
||||
|
||||
|
||||
class ConcurrentReplacementTests(TransactionTestCase):
|
||||
"""A transform must not overwrite an image replaced after staging."""
|
||||
|
||||
def test_replaced_source_is_preserved(self):
|
||||
for in_place in (False, True):
|
||||
with self.subTest(in_place=in_place):
|
||||
directory, staged = self.make_images(count=1)
|
||||
path, staged_path = staged[0]
|
||||
image_metadata = {
|
||||
path: crit_main._capture_file_metadata(path),
|
||||
}
|
||||
replacement = path + ".concurrent"
|
||||
write(replacement, b"concurrent")
|
||||
os.replace(replacement, path)
|
||||
|
||||
with self.assertRaisesRegex(
|
||||
crit_main.CritTransformError,
|
||||
"image changed while preparing to replace"):
|
||||
crit_main._commit_staged(
|
||||
staged, in_place, image_metadata=image_metadata)
|
||||
|
||||
self.assertEqual(read(path), b"concurrent")
|
||||
self.assertFalse(os.path.exists(staged_path))
|
||||
self.assertFalse(os.path.exists(path + ".bak"))
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
def test_replacement_while_commit_links_original_is_preserved(self):
|
||||
for in_place in (False, True):
|
||||
with self.subTest(in_place=in_place):
|
||||
directory, staged = self.make_images(count=1)
|
||||
path, staged_path = staged[0]
|
||||
image_metadata = {
|
||||
path: crit_main._capture_file_metadata(path),
|
||||
}
|
||||
|
||||
def replace_source(_source, _link):
|
||||
replacement = path + ".concurrent"
|
||||
write(replacement, b"concurrent")
|
||||
os.replace(replacement, path)
|
||||
|
||||
with intercept(
|
||||
"link", when=lambda src, _dst: src == path,
|
||||
after=replace_source):
|
||||
with self.assertRaisesRegex(
|
||||
crit_main.CritTransformError,
|
||||
"image changed while preparing to replace"):
|
||||
crit_main._commit_staged(
|
||||
staged, in_place,
|
||||
image_metadata=image_metadata)
|
||||
|
||||
self.assertEqual(read(path), b"concurrent")
|
||||
self.assertFalse(os.path.exists(staged_path))
|
||||
self.assertFalse(os.path.exists(path + ".bak"))
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
def test_later_replacement_rolls_back_only_earlier_images(self):
|
||||
for in_place in (False, True):
|
||||
with self.subTest(in_place=in_place):
|
||||
directory, staged = self.make_images(count=2)
|
||||
first_path, first_staged = staged[0]
|
||||
second_path, second_staged = staged[1]
|
||||
image_metadata = {
|
||||
path: crit_main._capture_file_metadata(path)
|
||||
for path, _ in staged
|
||||
}
|
||||
|
||||
def replace_second(_source, _destination):
|
||||
replacement = second_path + ".concurrent"
|
||||
write(replacement, b"concurrent")
|
||||
os.replace(replacement, second_path)
|
||||
|
||||
with intercept(
|
||||
"replace",
|
||||
when=lambda src, _dst: src == first_staged,
|
||||
after=replace_second):
|
||||
with self.assertRaisesRegex(
|
||||
crit_main.CritTransformError,
|
||||
"image changed while preparing to replace"):
|
||||
crit_main._commit_staged(
|
||||
staged, in_place,
|
||||
image_metadata=image_metadata)
|
||||
|
||||
self.assertEqual(read(first_path), b"old-0")
|
||||
self.assertEqual(read(second_path), b"concurrent")
|
||||
self.assertFalse(os.path.exists(first_staged))
|
||||
self.assertFalse(os.path.exists(second_staged))
|
||||
self.assertFalse(os.path.exists(first_path + ".bak"))
|
||||
self.assertFalse(os.path.exists(second_path + ".bak"))
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
|
||||
@unittest.skipIf(signal.getsignal(signal.SIGTERM) == signal.SIG_IGN,
|
||||
"SIGTERM is ignored in this environment")
|
||||
class SignalHandlingTests(TransactionTestCase):
|
||||
"""Termination signals must never leave a half-replaced image set."""
|
||||
|
||||
def test_repeated_signal_does_not_interrupt_cleanup(self):
|
||||
"""The first signal raises for rollback; a repeat during that
|
||||
unwinding is swallowed so cleanup can finish."""
|
||||
with crit_main._commit_signal_handlers():
|
||||
with self.assertRaisesRegex(crit_main.CritTransformError,
|
||||
"SIGTERM"):
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
# Unwinding has started; a second signal must not raise again.
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
def test_signal_after_commit_is_ignored(self):
|
||||
"""Once the replacement set is durable, a late signal must not
|
||||
make a committed command look failed."""
|
||||
_, staged = self.make_images(count=1)
|
||||
|
||||
with crit_main._commit_signal_handlers() as signal_state:
|
||||
crit_main._commit_staged(staged, True, signal_state)
|
||||
self.assertTrue(signal_state["committed"])
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
self.assertCommitted(staged)
|
||||
|
||||
def test_signal_during_post_commit_cleanup(self):
|
||||
"""A signal while removing the now-unneeded rollback links stays
|
||||
deferred: it is too late to roll back, so the command succeeds
|
||||
with the new image set installed and no leftover links."""
|
||||
directory, staged = self.make_images(count=1)
|
||||
|
||||
with crit_main._commit_signal_handlers() as signal_state:
|
||||
def during_rollback_cleanup(target):
|
||||
return (signal_state["committed"] and
|
||||
"crit-rollback" in target)
|
||||
|
||||
with intercept("unlink", when=during_rollback_cleanup,
|
||||
after=send_signal(signal.SIGTERM)):
|
||||
crit_main._commit_staged(staged, True, signal_state)
|
||||
|
||||
self.assertCommitted(staged)
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
|
||||
class MetadataTests(TransactionTestCase):
|
||||
"""Replacing an image's inode must not lose its metadata."""
|
||||
|
||||
def test_replacement_preserves_mode_owner_and_times(self):
|
||||
"""The replacement keeps the original's complete Unix ownership
|
||||
and permission tuple, including when CRIT runs as root on an
|
||||
unprivileged user's checkpoint."""
|
||||
directory, _ = self.make_images(count=0)
|
||||
path = os.path.join(directory, "owned")
|
||||
write(path, b"old")
|
||||
# Owner-only, but distinct from mkstemp's 0600 default so a
|
||||
# replacement whose mode was never restored cannot pass.
|
||||
os.chmod(path, 0o400)
|
||||
os.utime(path, ns=(946684800123456789, 978307200987654321))
|
||||
if os.geteuid() == 0:
|
||||
os.chown(path, 65534, 65534)
|
||||
# Compare against a stat snapshot rather than the raw utime values
|
||||
# so filesystems with coarser timestamp granularity still pass.
|
||||
before = os.stat(path)
|
||||
|
||||
with crit_main._staged_file(path) as (output, staged_path):
|
||||
output.write(b"new")
|
||||
crit_main._commit_staged([(path, staged_path)], True)
|
||||
|
||||
after = os.stat(path)
|
||||
self.assertEqual(stat.S_IMODE(after.st_mode),
|
||||
stat.S_IMODE(before.st_mode))
|
||||
self.assertEqual((after.st_uid, after.st_gid),
|
||||
(before.st_uid, before.st_gid))
|
||||
self.assertEqual((after.st_atime_ns, after.st_mtime_ns),
|
||||
(before.st_atime_ns, before.st_mtime_ns))
|
||||
|
||||
def test_replacement_preserves_xattrs(self):
|
||||
"""Replacement keeps ACL/security metadata carried as xattrs."""
|
||||
directory, _ = self.make_images(count=0)
|
||||
path = os.path.join(directory, "labelled")
|
||||
write(path, b"old")
|
||||
# user.* is available to unprivileged users on the filesystems
|
||||
# normally used by the test; skip only when unsupported.
|
||||
try:
|
||||
os.setxattr(path, b"user.crit-test", b"preserved")
|
||||
except OSError as exc:
|
||||
if exc.errno not in (errno.ENOTSUP, errno.EOPNOTSUPP):
|
||||
raise
|
||||
self.skipTest("filesystem does not support user xattrs")
|
||||
|
||||
with crit_main._staged_file(path) as (output, staged_path):
|
||||
output.write(b"new")
|
||||
crit_main._commit_staged([(path, staged_path)], True)
|
||||
|
||||
self.assertEqual(os.getxattr(path, b"user.crit-test"), b"preserved")
|
||||
|
||||
def test_metadata_capture_without_xattr_support(self):
|
||||
"""Filesystems without xattr support must still permit a transform
|
||||
when there is no extended metadata to preserve."""
|
||||
directory, _ = self.make_images(count=0)
|
||||
path = os.path.join(directory, "no-xattrs")
|
||||
write(path, b"contents")
|
||||
|
||||
unsupported = OSError(errno.ENOTSUP, "xattrs unsupported")
|
||||
with mock.patch.object(crit_main.os, "listxattr",
|
||||
side_effect=unsupported):
|
||||
metadata, xattrs = crit_main._capture_file_metadata(path)
|
||||
|
||||
self.assertTrue(stat.S_ISREG(metadata.st_mode))
|
||||
self.assertEqual(xattrs, [])
|
||||
|
||||
def test_noatime_denied_fallback_restores_atime(self):
|
||||
"""O_NOATIME can be denied when the caller may read but does not
|
||||
own an image; the fallback read restores the captured atime."""
|
||||
directory, _ = self.make_images(count=0)
|
||||
path = os.path.join(directory, "source")
|
||||
write(path, b"contents")
|
||||
os.utime(path, ns=(946684800123456789, 978307200987654321))
|
||||
source_metadata = crit_main._capture_file_metadata(path)
|
||||
before = os.stat(path)
|
||||
noatime = getattr(os, "O_NOATIME", 0)
|
||||
|
||||
with intercept("open",
|
||||
when=lambda target, flags, *args, **kwargs:
|
||||
flags & noatime,
|
||||
before=raise_(OSError(errno.EPERM,
|
||||
"injected O_NOATIME denial"))):
|
||||
with crit_main._source_file(path, source_metadata) as source:
|
||||
self.assertEqual(source.read(), b"contents")
|
||||
|
||||
after = os.stat(path)
|
||||
self.assertEqual((after.st_atime_ns, after.st_mtime_ns),
|
||||
(before.st_atime_ns, before.st_mtime_ns))
|
||||
|
||||
|
||||
class CleanupDiagnosticsTests(TransactionTestCase):
|
||||
"""Cleanup failures must not mask the error that triggered cleanup."""
|
||||
|
||||
def test_cleanup_failure_does_not_mask_primary_error(self):
|
||||
directory, staged = self.make_images(count=1)
|
||||
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr), mock.patch.object(
|
||||
crit_main, "_remove_file",
|
||||
side_effect=OSError(errno.EIO, "injected cleanup failure")):
|
||||
result = crit_main._command_failed(
|
||||
"compress",
|
||||
ValueError("primary transformation failure"),
|
||||
staged)
|
||||
|
||||
self.assertEqual(result, 1)
|
||||
diagnostic = stderr.getvalue()
|
||||
self.assertIn("primary transformation failure", diagnostic)
|
||||
self.assertIn("injected cleanup failure", diagnostic)
|
||||
|
||||
|
||||
class BackupCollisionTests(TransactionTestCase):
|
||||
"""A prior .bak file is user data and must never be overwritten."""
|
||||
|
||||
def test_existing_backup_is_preserved(self):
|
||||
"""The complete operation is refused before the first rename."""
|
||||
directory, staged = self.make_images(count=1)
|
||||
path, staged_path = staged[0]
|
||||
backup = path + ".bak"
|
||||
write(backup, b"existing backup")
|
||||
|
||||
with self.assertRaisesRegex(crit_main.CritTransformError,
|
||||
"existing backup"):
|
||||
crit_main._commit_staged(staged, False)
|
||||
|
||||
self.assertOriginalsIntact(staged)
|
||||
self.assertEqual(read(backup), b"existing backup")
|
||||
self.assertFalse(os.path.exists(staged_path))
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
def test_concurrent_backup_creation_wins(self):
|
||||
"""A backup created by another process after this transaction
|
||||
starts must win atomically (link() failing with EEXIST)."""
|
||||
directory, staged = self.make_images(count=1)
|
||||
path, staged_path = staged[0]
|
||||
backup = path + ".bak"
|
||||
|
||||
def racer(source, destination):
|
||||
write(destination, b"concurrent backup")
|
||||
|
||||
with intercept("link",
|
||||
when=lambda src, dst: dst.endswith(".bak"),
|
||||
before=racer):
|
||||
with self.assertRaisesRegex(crit_main.CritTransformError,
|
||||
"existing backup"):
|
||||
crit_main._commit_staged(staged, False)
|
||||
|
||||
self.assertOriginalsIntact(staged)
|
||||
self.assertEqual(read(backup), b"concurrent backup")
|
||||
self.assertFalse(os.path.exists(staged_path))
|
||||
self.assertNoTransactionFiles(directory)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,19 +1,34 @@
|
|||
#!/bin/bash
|
||||
# shellcheck disable=SC2002
|
||||
|
||||
set -x
|
||||
set -ex
|
||||
|
||||
# shellcheck source=test/others/env.sh
|
||||
source ../env.sh
|
||||
|
||||
images_list=()
|
||||
RESTORED_PID=""
|
||||
|
||||
function cleanup_restored_process {
|
||||
if [ -z "$RESTORED_PID" ]; then
|
||||
return
|
||||
fi
|
||||
if kill -0 "$RESTORED_PID" 2>/dev/null; then
|
||||
kill -KILL "$RESTORED_PID" 2>/dev/null || true
|
||||
fi
|
||||
RESTORED_PID=""
|
||||
}
|
||||
|
||||
trap cleanup_restored_process EXIT
|
||||
|
||||
function gen_imgs {
|
||||
PID=$(../loop)
|
||||
if ! $CRIU dump -v4 -o dump.log -D ./ -t "$PID"; then
|
||||
if ! $CRIU dump --no-default-config -v4 -o dump.log -D ./ -t "$PID"; then
|
||||
echo "Failed to checkpoint process $PID"
|
||||
cat dump.log
|
||||
kill -9 "$PID"
|
||||
if [ -f dump.log ]; then
|
||||
cat dump.log
|
||||
fi
|
||||
kill -9 "$PID" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -101,8 +116,549 @@ function run_test2 {
|
|||
${CRIT} x ./ rss || exit 1
|
||||
}
|
||||
|
||||
ZDTM_DIR="${BASE_DIR}/test/zdtm/static"
|
||||
CRIT_TEST_DIR="${BASE_DIR}/test/others/crit"
|
||||
COMPRESSION_IMAGE_HELPER="${CRIT_TEST_DIR}/compression_image_helpers.py"
|
||||
CRIT_TRANSACTION_TESTS="${CRIT_TEST_DIR}/crit_transaction_tests.py"
|
||||
CRIT_COMPRESSION_TESTS="${CRIT_TEST_DIR}/crit_compression_tests.py"
|
||||
|
||||
function dump_test_process {
|
||||
# Dump compress_pages02 (covers 8 mapping types: anonymous,
|
||||
# zero-filled, shared anonymous, file-backed private/shared,
|
||||
# memfd, read-only, PROT_NONE guard) into the given directory.
|
||||
# Prints the PID on success.
|
||||
local dir=$1; shift
|
||||
local pid
|
||||
|
||||
if ! make -C "$ZDTM_DIR" compress_pages02 > /dev/null 2>&1; then
|
||||
echo "FAIL: failed to build compress_pages02"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Clean stale marker files and run from the test directory
|
||||
# so zdtm pidfile rename works (same filesystem).
|
||||
rm -f "$dir"/test.out* "$dir"/test.pid "$dir"/test.file
|
||||
|
||||
if ! (cd "$dir" && "$ZDTM_DIR/compress_pages02" \
|
||||
--pidfile=test.pid \
|
||||
--outfile=test.out \
|
||||
--filename=test.file); then
|
||||
echo "FAIL: failed to start compress_pages02"
|
||||
cat "$dir/test.out" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for pidfile
|
||||
local i=0
|
||||
while [ $i -lt 50 ]; do
|
||||
if [ -f "$dir/test.pid" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
pid=$(cat "$dir/test.pid" 2>/dev/null)
|
||||
if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
|
||||
echo "FAIL: test process didn't start"
|
||||
cat "$dir/test.out" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! $CRIU dump --no-default-config -v4 -o dump.log -D "$dir" \
|
||||
-t "$pid" --shell-job "$@"; then
|
||||
echo "FAIL: dump into $dir"
|
||||
if [ -f "$dir/dump.log" ]; then
|
||||
cat "$dir/dump.log"
|
||||
fi
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "$pid"
|
||||
}
|
||||
|
||||
function restore_and_verify {
|
||||
# Restore from directory, send SIGTERM so compress_pages02
|
||||
# verifies its own memory, then check the test output for PASS.
|
||||
local dir=$1
|
||||
local i
|
||||
local pid
|
||||
local verified=0
|
||||
|
||||
if ! $CRIU restore --no-default-config -v4 -o restore.log -D "$dir" \
|
||||
--shell-job -d; then
|
||||
echo "FAIL: restore from $dir"
|
||||
cat "$dir/restore.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get the root PID from the pstree image
|
||||
pid=$($CRIT decode -i "$dir/pstree.img" 2>/dev/null |
|
||||
grep -o '"pid": [0-9]*' | head -1 | grep -o '[0-9]*')
|
||||
|
||||
if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
|
||||
echo "FAIL: restored process not running from $dir (pid=$pid)"
|
||||
exit 1
|
||||
fi
|
||||
RESTORED_PID="$pid"
|
||||
|
||||
# Send SIGTERM so compress_pages02 verifies all memory
|
||||
# regions (zero, pattern, shared, file, memfd, readonly,
|
||||
# guard) then writes PASS/FAIL to its output file.
|
||||
if ! kill -TERM "$pid" 2>/dev/null; then
|
||||
echo "FAIL: unable to signal restored process $pid from $dir"
|
||||
exit 1
|
||||
fi
|
||||
for ((i = 0; i < 100; i++)); do
|
||||
if grep -q PASS "$dir/test.out" 2>/dev/null; then
|
||||
verified=1
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
if [ "$verified" -ne 1 ]; then
|
||||
echo "FAIL: memory verification failed after restore from $dir"
|
||||
cat "$dir/test.out" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The workload normally exits immediately after publishing PASS. Bound the
|
||||
# wait so a broken test cannot leak into subsequent CRIT cases.
|
||||
for ((i = 0; i < 50; i++)); do
|
||||
kill -0 "$pid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
RESTORED_PID=""
|
||||
}
|
||||
|
||||
function pages_checksum {
|
||||
# Print md5sum of all pages-*.img files in a directory.
|
||||
# This captures the raw page content for comparison.
|
||||
cat "$1"/pages-*.img | md5sum | awk '{print $1}'
|
||||
}
|
||||
|
||||
function assert_inventory_version {
|
||||
local dir=$1
|
||||
local expected=$2
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
assert-inventory-version "$dir" "$expected"
|
||||
}
|
||||
|
||||
function make_sparse_pagemap_image {
|
||||
local dir=$1
|
||||
local compressed=$2
|
||||
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
make-sparse-pagemap "$dir" "$compressed"
|
||||
}
|
||||
|
||||
function make_threshold_pagemap_image {
|
||||
local dir=$1
|
||||
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
make-threshold-pagemap "$dir"
|
||||
}
|
||||
|
||||
function make_truncated_uncompressed_entry_image {
|
||||
local dir=$1
|
||||
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
make-truncated-uncompressed "$dir"
|
||||
}
|
||||
|
||||
function assert_sparse_pagemap_payload {
|
||||
local dir=$1
|
||||
local expected_size=$2
|
||||
local expected_md5=$3
|
||||
local actual_size
|
||||
local actual_md5
|
||||
|
||||
actual_size=$(stat -c %s "$dir/pages-1.img")
|
||||
if [ "$actual_size" != "$expected_size" ]; then
|
||||
echo "FAIL: $dir/pages-1.img size is $actual_size, expected $expected_size"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
actual_md5=$(md5sum "$dir/pages-1.img" | awk '{print $1}')
|
||||
if [ "$actual_md5" != "$expected_md5" ]; then
|
||||
echo "FAIL: $dir/pages-1.img checksum is $actual_md5, expected $expected_md5"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function assert_pagemap_entry_uncompressed {
|
||||
local dir=$1
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
assert-entry-uncompressed "$dir"
|
||||
}
|
||||
|
||||
function assert_region_pagemap {
|
||||
local dir=$1
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
assert-region-pagemap "$dir"
|
||||
}
|
||||
|
||||
function make_malformed_compressed_image {
|
||||
local dir=$1
|
||||
local kind=$2
|
||||
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
make-malformed-compressed "$dir" "$kind"
|
||||
}
|
||||
|
||||
function make_transactional_compress_image {
|
||||
local dir=$1
|
||||
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
|
||||
PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
make-transactional-compress "$dir"
|
||||
}
|
||||
|
||||
function image_set_checksum {
|
||||
(cd "$1" && sha256sum ./*.img) | sha256sum | awk '{print $1}'
|
||||
}
|
||||
|
||||
function assert_no_transaction_files {
|
||||
local dir=$1
|
||||
local file
|
||||
|
||||
file=$(find "$dir" -maxdepth 1 -name '.*.crit-*' -print -quit)
|
||||
if [ -n "$file" ]; then
|
||||
echo "FAIL: transaction file was not removed: $file"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function test_commit_transactions {
|
||||
PYTHONPATH="${BASE_DIR}/lib:${BASE_DIR}/crit" \
|
||||
python3 "$CRIT_TRANSACTION_TESTS" -v
|
||||
}
|
||||
|
||||
function test_compress_regressions {
|
||||
PYTHONPATH="${BASE_DIR}/lib:${BASE_DIR}/crit" \
|
||||
python3 "$CRIT_COMPRESSION_TESTS"
|
||||
}
|
||||
|
||||
function run_test_compress {
|
||||
local page_size
|
||||
page_size=$(getconf PAGESIZE)
|
||||
|
||||
echo "=== compress/decompress tests ==="
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 1: dump -c -> decompress -> restore
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 1: compressed dump -> decompress -> restore"
|
||||
dump_test_process comp/ -c > /dev/null
|
||||
|
||||
$CRIT decompress comp/ || exit 1
|
||||
assert_inventory_version comp/ 2
|
||||
|
||||
# Verify backup files exist
|
||||
ls comp/pages-*.img.bak > /dev/null 2>&1 || { echo "FAIL: no pages backup"; exit 1; }
|
||||
ls comp/pagemap-*.img.bak > /dev/null 2>&1 || { echo "FAIL: no pagemap backup"; exit 1; }
|
||||
ls comp/inventory.img.bak > /dev/null 2>&1 || { echo "FAIL: no inventory backup"; exit 1; }
|
||||
|
||||
restore_and_verify comp/
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 2: dump uncompressed -> compress -> restore
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 2: uncompressed dump -> compress -> restore"
|
||||
dump_test_process uncomp/ > /dev/null
|
||||
|
||||
$CRIT compress uncomp/ --in-place || exit 1
|
||||
assert_inventory_version uncomp/ 3
|
||||
|
||||
# Verify no backup files with --in-place
|
||||
if ls uncomp/*.bak > /dev/null 2>&1; then
|
||||
echo "FAIL: backup files created with --in-place"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
restore_and_verify uncomp/
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 3: compress already compressed, decompress already decompressed
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 3: compress already compressed, decompress already decompressed"
|
||||
$CRIT compress uncomp/ --in-place 2>&1 | grep -q "already compressed" || {
|
||||
echo "FAIL: compress should report already compressed"
|
||||
exit 1
|
||||
}
|
||||
$CRIT decompress comp/ 2>&1 | grep -q "already decompressed" || {
|
||||
echo "FAIL: decompress should report already decompressed"
|
||||
exit 1
|
||||
}
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 4: compress -> decompress -> compress produces same pages
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 4: compress -> decompress -> compress stability"
|
||||
rm -rf cdc/
|
||||
mkdir -p cdc/
|
||||
dump_test_process cdc/ > /dev/null
|
||||
|
||||
$CRIT compress cdc/ --in-place || exit 1
|
||||
local sum1
|
||||
sum1=$(pages_checksum cdc/)
|
||||
|
||||
$CRIT decompress cdc/ --in-place || exit 1
|
||||
assert_inventory_version cdc/ 2
|
||||
$CRIT compress cdc/ --in-place || exit 1
|
||||
assert_inventory_version cdc/ 3
|
||||
local sum2
|
||||
sum2=$(pages_checksum cdc/)
|
||||
|
||||
if [ "$sum1" != "$sum2" ]; then
|
||||
echo "FAIL: compress->decompress->compress changed pages data"
|
||||
echo " first: $sum1"
|
||||
echo " second: $sum2"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
restore_and_verify cdc/
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 5: decompress -> compress -> decompress produces same pages
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 5: decompress -> compress -> decompress stability"
|
||||
rm -rf dcd/
|
||||
mkdir -p dcd/
|
||||
dump_test_process dcd/ -c > /dev/null
|
||||
|
||||
$CRIT decompress dcd/ --in-place || exit 1
|
||||
assert_inventory_version dcd/ 2
|
||||
local sum3
|
||||
sum3=$(pages_checksum dcd/)
|
||||
|
||||
$CRIT compress dcd/ --in-place || exit 1
|
||||
assert_inventory_version dcd/ 3
|
||||
$CRIT decompress dcd/ --in-place || exit 1
|
||||
assert_inventory_version dcd/ 2
|
||||
local sum4
|
||||
sum4=$(pages_checksum dcd/)
|
||||
|
||||
if [ "$sum3" != "$sum4" ]; then
|
||||
echo "FAIL: decompress->compress->decompress changed pages data"
|
||||
echo " first: $sum3"
|
||||
echo " second: $sum4"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
restore_and_verify dcd/
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 6: CRIT must not consume pages payload for sparse
|
||||
# pagemap entries without PE_PRESENT, including legacy
|
||||
# in_parent entries with no flags field.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 6: sparse pagemap entries have no pages payload"
|
||||
local raw_md5
|
||||
local zero_md5
|
||||
raw_md5=$(PYTHONPATH="${BASE_DIR}/lib" \
|
||||
python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
expected-sparse-checksum raw-zero)
|
||||
zero_md5=$(PYTHONPATH="${BASE_DIR}/lib" \
|
||||
python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
expected-sparse-checksum two-raw)
|
||||
|
||||
make_sparse_pagemap_image sparse-compress/ 0
|
||||
$CRIT compress sparse-compress/ --in-place || exit 1
|
||||
assert_inventory_version sparse-compress/ 3
|
||||
$CRIT decompress sparse-compress/ --in-place || exit 1
|
||||
# Parent entries may still resolve to compressed parent images, so CRIT
|
||||
# must retain the V1_2 reader gate after transforming the local payload.
|
||||
assert_inventory_version sparse-compress/ 3
|
||||
assert_sparse_pagemap_payload sparse-compress/ "$((page_size * 2))" "$zero_md5"
|
||||
|
||||
make_sparse_pagemap_image sparse-decompress/ 1
|
||||
$CRIT decompress sparse-decompress/ --in-place || exit 1
|
||||
assert_inventory_version sparse-decompress/ 3
|
||||
assert_sparse_pagemap_payload sparse-decompress/ "$((page_size * 2))" "$raw_md5"
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 7: CRIT uses the same 7/8 raw fallback threshold as CRIU and
|
||||
# omits compression metadata when the complete entry stays raw.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 7: raw-only entries use ordinary pagemap representation"
|
||||
make_threshold_pagemap_image threshold-compress/
|
||||
local threshold_md5
|
||||
threshold_md5=$(md5sum threshold-compress/pages-1.img | awk '{print $1}')
|
||||
|
||||
$CRIT compress threshold-compress/ --in-place || exit 1
|
||||
assert_inventory_version threshold-compress/ 3
|
||||
assert_pagemap_entry_uncompressed threshold-compress/
|
||||
assert_sparse_pagemap_payload threshold-compress/ "$page_size" "$threshold_md5"
|
||||
|
||||
# A parent link means a parent image may still require compressed-image
|
||||
# readers even when every page in this directory is present and raw.
|
||||
cp -a threshold-compress/ threshold-parent/
|
||||
mkdir -p parent-placeholder/
|
||||
cp threshold-compress/inventory.img parent-placeholder/inventory.img
|
||||
ln -s ../parent-placeholder threshold-parent/parent
|
||||
$CRIT decompress threshold-parent/ --in-place || exit 1
|
||||
assert_inventory_version threshold-parent/ 3
|
||||
|
||||
$CRIT decompress threshold-compress/ --in-place || exit 1
|
||||
assert_inventory_version threshold-compress/ 2
|
||||
assert_sparse_pagemap_payload threshold-compress/ "$page_size" "$threshold_md5"
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 8: CRIT must fail explicitly on a truncated
|
||||
# uncompressed payload copied through by decompress.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 8: truncated uncompressed payload is rejected"
|
||||
make_truncated_uncompressed_entry_image truncated-decompress/
|
||||
if $CRIT decompress truncated-decompress/ --in-place \
|
||||
> truncated-decompress.log 2>&1; then
|
||||
echo "FAIL: decompress accepted truncated pages payload"
|
||||
cat truncated-decompress.log
|
||||
exit 1
|
||||
fi
|
||||
grep -q "describes.*payload bytes" truncated-decompress.log || {
|
||||
echo "FAIL: decompress did not report the payload-size mismatch"
|
||||
cat truncated-decompress.log
|
||||
exit 1
|
||||
}
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 9: region-compressed images are transformed and
|
||||
# restored, including a final short region where present.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 9: region-compressed dump -> decompress -> restore"
|
||||
rm -rf region/
|
||||
mkdir -p region/
|
||||
dump_test_process region/ "--compress-region=$((page_size * 16))" > /dev/null
|
||||
assert_region_pagemap region/
|
||||
$CRIT decompress region/ --in-place || exit 1
|
||||
assert_inventory_version region/ 2
|
||||
restore_and_verify region/
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 10: reject malformed compressed metadata before
|
||||
# attempting any unbounded read or allocation.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 10: malformed compressed metadata is rejected"
|
||||
local kind
|
||||
for kind in oversize total count flags aligned-nonpresent region-limit region-count \
|
||||
region-final-oversize; do
|
||||
local dir="malformed-$kind"
|
||||
local before
|
||||
local after
|
||||
make_malformed_compressed_image "$dir/" "$kind"
|
||||
before=$(image_set_checksum "$dir/")
|
||||
if $CRIT decompress "$dir/" --in-place \
|
||||
> "$dir.log" 2>&1; then
|
||||
echo "FAIL: decompress accepted malformed metadata: $kind"
|
||||
cat "$dir.log"
|
||||
exit 1
|
||||
fi
|
||||
after=$(image_set_checksum "$dir/")
|
||||
if [ "$before" != "$after" ]; then
|
||||
echo "FAIL: malformed $kind image was modified"
|
||||
exit 1
|
||||
fi
|
||||
assert_no_transaction_files "$dir/"
|
||||
done
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 11: a failure in a later pages image leaves every
|
||||
# original image untouched for both transformations.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 11: transformations are transactional"
|
||||
local before
|
||||
local after
|
||||
|
||||
make_malformed_compressed_image transaction-decompress/ transaction
|
||||
before=$(image_set_checksum transaction-decompress/)
|
||||
if $CRIT decompress transaction-decompress/ --in-place \
|
||||
> transaction-decompress.log 2>&1; then
|
||||
echo "FAIL: decompress accepted corrupt LZ4 payload"
|
||||
cat transaction-decompress.log
|
||||
exit 1
|
||||
fi
|
||||
after=$(image_set_checksum transaction-decompress/)
|
||||
if [ "$before" != "$after" ]; then
|
||||
echo "FAIL: failed decompression modified the image set"
|
||||
exit 1
|
||||
fi
|
||||
assert_no_transaction_files transaction-decompress/
|
||||
|
||||
make_transactional_compress_image transaction-compress/
|
||||
before=$(image_set_checksum transaction-compress/)
|
||||
if $CRIT compress transaction-compress/ --in-place \
|
||||
> transaction-compress.log 2>&1; then
|
||||
echo "FAIL: compress accepted a truncated pages image"
|
||||
cat transaction-compress.log
|
||||
exit 1
|
||||
fi
|
||||
after=$(image_set_checksum transaction-compress/)
|
||||
if [ "$before" != "$after" ]; then
|
||||
echo "FAIL: failed compression modified the image set"
|
||||
exit 1
|
||||
fi
|
||||
assert_no_transaction_files transaction-compress/
|
||||
test_commit_transactions
|
||||
echo " PASS"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Test 12: offline compression observes restore invariants,
|
||||
# honors acceleration, preserves timestamps, and validates enums.
|
||||
# -------------------------------------------------------
|
||||
echo " -- Test 12: CRIT compression regressions"
|
||||
test_compress_regressions
|
||||
echo " PASS"
|
||||
|
||||
echo "=== compress/decompress: ALL PASS ==="
|
||||
}
|
||||
|
||||
${CRIT} --version
|
||||
|
||||
gen_imgs
|
||||
run_test1
|
||||
run_test2
|
||||
|
||||
# Skip compress/decompress tests if lz4 or CRIU compression is unavailable
|
||||
if PYTHONPATH="${BASE_DIR}/lib" python3 "$COMPRESSION_IMAGE_HELPER" \
|
||||
has-lz4 2>/dev/null && \
|
||||
$CRIU check --no-default-config --feature compress 2>/dev/null; then
|
||||
mkdir -p comp/ uncomp/
|
||||
run_test_compress
|
||||
else
|
||||
echo "=== Skipping compress/decompress tests (lz4 or CRIU compression not available) ==="
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue