From 4d76d1acdcd69afcedf1335c810bef5f8e18d95e Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Sun, 15 Mar 2026 10:46:35 +0000 Subject: [PATCH] ci: shard alpine-test into parallel jobs to reduce CI time The alpine-test CI job runs all ~483 zdtm tests sequentially three times (normal, mntns-compat-mode, criu-config), followed by many non-shardable tests. This dominates overall CI wait time. With only 2 jobs running in parallel (GCC and CLANG) the alpine tests take around 30 minutes. Use the existing --test-shard-index and --test-shard-count flags already built into test/zdtm.py to split the zdtm test suite across four parallel runners (shards 0-3). A fifth shard runs all non-shardable tests (lazy pages, fault injection, test/others/*, rootless, compel, plugins, etc.) independently and in parallel with the zdtm shards. This increases parallelism from 2 to 10 jobs and reduces the alpine test wall-clock time from ~30 to ~10 minutes. Changes: - run-ci-tests.sh: Build SHARD_OPTS from ZDTM_SHARD_INDEX/COUNT env vars and pass them to zdtm.py. Extract all non-shardable tests into a run_non_shardable_tests() function. Dispatch based on shard index: 0-3 run zdtm slices, 4 runs non-shardable tests, unset runs everything sequentially (preserving existing behavior). Validate that ZDTM_SHARD_INDEX is set when ZDTM_SHARD_COUNT is set. - Makefile: Pass ZDTM_SHARD_INDEX and ZDTM_SHARD_COUNT into the container when set. Split long container run command across multiple lines for readability. - ci.yml: Add shard: [0, 1, 2, 3, 4] to the alpine-test matrix, producing 10 jobs (2 compilers x 5 shards). Job labels now show descriptive shard names (e.g. "zdtm 1/4", "non-zdtm") instead of raw indices. When sharding is not configured the script behaves identically to before, so other CI jobs (aarch64, compat, gcov, etc.) are unaffected. Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Adrian Reber --- .github/workflows/ci.yml | 21 ++- scripts/ci/Makefile | 11 +- scripts/ci/run-ci-tests.sh | 320 +++++++++++++++++++++---------------- 3 files changed, 210 insertions(+), 142 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eef95f329..d0563b5b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,16 +9,31 @@ concurrency: jobs: alpine-test: - name: Alpine Test + name: Alpine Test (${{ matrix.target }}, ${{ matrix.shard_name }}) strategy: matrix: os: [ubuntu-22.04] target: [GCC=1, CLANG=1] + shard: [0, 1, 2, 3, 4] + include: + - shard: 0 + shard_name: zdtm 1/4 + - shard: 1 + shard_name: zdtm 2/4 + - shard: 2 + shard_name: zdtm 3/4 + - shard: 3 + shard_name: zdtm 4/4 + - shard: 4 + shard_name: non-zdtm runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - name: Run Alpine ${{ matrix.target }} Test - run: sudo -E make -C scripts/ci alpine ${{ matrix.target }} + - name: Run Alpine ${{ matrix.target }} ${{ matrix.shard_name }} Test + run: > + sudo -E make -C scripts/ci alpine ${{ matrix.target }} + ZDTM_SHARD_INDEX=${{ matrix.shard }} + ZDTM_SHARD_COUNT=4 alpine-test-arm64: name: Alpine Test ARM64 diff --git a/scripts/ci/Makefile b/scripts/ci/Makefile index bfb2f3d23..7eeda1b12 100644 --- a/scripts/ci/Makefile +++ b/scripts/ci/Makefile @@ -46,10 +46,19 @@ ifeq ($(CONTAINER_RUNTIME),podman) endif export ZDTM_OPTS +export ZDTM_SHARD_INDEX +export ZDTM_SHARD_COUNT $(TARGETS): $(MAKE) -C ../build $@$(target-suffix) - $(CONTAINER_RUNTIME) run --env-file docker.env -v `pwd`/../../:/criu $(if $(ZDTM_OPTS),-e ZDTM_OPTS) $(CONTAINER_OPTS) criu-$@ scripts/ci/run-ci-tests.sh + $(CONTAINER_RUNTIME) run \ + --env-file docker.env \ + -v `pwd`/../../:/criu \ + $(if $(ZDTM_OPTS),-e ZDTM_OPTS) \ + $(if $(ZDTM_SHARD_INDEX),-e ZDTM_SHARD_INDEX) \ + $(if $(ZDTM_SHARD_COUNT),-e ZDTM_SHARD_COUNT) \ + $(CONTAINER_OPTS) \ + criu-$@ scripts/ci/run-ci-tests.sh fedora-asan: $(MAKE) -C ../build $@$(target-suffix) diff --git a/scripts/ci/run-ci-tests.sh b/scripts/ci/run-ci-tests.sh index 432a7b577..d55bb13b8 100755 --- a/scripts/ci/run-ci-tests.sh +++ b/scripts/ci/run-ci-tests.sh @@ -8,6 +8,16 @@ X86_64_PKGS=(gcc-multilib) # Convert from string to array. IFS=" " read -r -a ZDTM_OPTS <<< "$ZDTM_OPTS" +SHARD_OPTS=() +if [ -n "$ZDTM_SHARD_COUNT" ] && [ "$ZDTM_SHARD_COUNT" -gt 0 ]; then + if [ -z "$ZDTM_SHARD_INDEX" ]; then + echo "ERROR: ZDTM_SHARD_COUNT set but ZDTM_SHARD_INDEX is not" + exit 1 + fi + SHARD_OPTS=(--test-shard-index "$ZDTM_SHARD_INDEX" \ + --test-shard-count "$ZDTM_SHARD_COUNT") +fi + UNAME_M=$(uname -m) if [ "$UNAME_M" != "x86_64" ]; then @@ -220,158 +230,192 @@ if [ "${STREAM_TEST}" = "1" ]; then exit 0 fi -./test/zdtm.py run -a -p 2 --keep-going "${ZDTM_OPTS[@]}" -if criu/criu check --feature move_mount_set_group; then - ./test/zdtm.py run -a -p 2 --mntns-compat-mode --keep-going "${ZDTM_OPTS[@]}" -fi - -./test/zdtm.py run -a -p 2 --keep-going --criu-config "${ZDTM_OPTS[@]}" - -# Newer kernels are blocking access to userfaultfd: -# uffd: Set unprivileged_userfaultfd sysctl knob to 1 if kernel faults must be handled without obtaining CAP_SYS_PTRACE capability -if [ -e /proc/sys/vm/unprivileged_userfaultfd ]; then - echo 1 > /proc/sys/vm/unprivileged_userfaultfd -fi - -LAZY_EXCLUDE=(-x maps04 -x cmdlinenv00 -x maps007) - -LAZY_TESTS='.*(maps0|uffd-events|lazy-thp|futex|fork).*' -LAZY_OPTS=(-p 2 -T "$LAZY_TESTS" "${LAZY_EXCLUDE[@]}" "${ZDTM_OPTS[@]}") - -./test/zdtm.py run "${LAZY_OPTS[@]}" --lazy-pages -./test/zdtm.py run "${LAZY_OPTS[@]}" --remote-lazy-pages -./test/zdtm.py run "${LAZY_OPTS[@]}" --remote-lazy-pages --tls - -bash -x ./test/jenkins/criu-fault.sh -if [ "$UNAME_M" == "x86_64" ]; then - # This fails on aarch64 (aws-graviton2) with: - # 33: ERR: thread-bomb.c:49: pthread_attr_setstacksize(): 22 - bash -x ./test/jenkins/criu-fcg.sh -fi -bash -x ./test/jenkins/criu-inhfd.sh - -if [ -z "$SKIP_EXT_DEV_TEST" ]; then - make -C test/others/mnt-ext-dev/ run - if criu/criu check --feature move_mount_set_group; then - EXTRA_OPTS=--mntns-compat-mode make -C test/others/mnt-ext-dev/ run +run_non_shardable_tests() { + # Newer kernels are blocking access to userfaultfd: + # uffd: Set unprivileged_userfaultfd sysctl knob to 1 if kernel faults + # must be handled without obtaining CAP_SYS_PTRACE capability + if [ -e /proc/sys/vm/unprivileged_userfaultfd ]; then + echo 1 > /proc/sys/vm/unprivileged_userfaultfd fi -fi -make -C test/others/make/ run CC="$CC" -if [ -n "$CIRCLECI" ]; then - # GitHub Actions (and Cirrus CI) does not provide a real TTY and CRIU will fail with: - # Error (criu/tty.c:1014): tty: Don't have tty to inherit session from, aborting - make -C test/others/shell-job/ run -fi -make -C test/others/criu-ns/ run -make -C test/others/skip-file-rwx-check/ run -make -C test/others/rpc/ run + LAZY_EXCLUDE=(-x maps04 -x cmdlinenv00 -x maps007) -./test/zdtm.py run -t zdtm/static/env00 --sibling + LAZY_TESTS='.*(maps0|uffd-events|lazy-thp|futex|fork).*' + LAZY_OPTS=(-p 2 -T "$LAZY_TESTS" "${LAZY_EXCLUDE[@]}" "${ZDTM_OPTS[@]}") -./test/zdtm.py run -t zdtm/static/maps00 --preload-libfault -./test/zdtm.py run -t zdtm/static/maps02 --preload-libfault + ./test/zdtm.py run "${LAZY_OPTS[@]}" --lazy-pages + ./test/zdtm.py run "${LAZY_OPTS[@]}" --remote-lazy-pages + ./test/zdtm.py run "${LAZY_OPTS[@]}" --remote-lazy-pages --tls -./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --dedup -./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --noauto-dedup -./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --page-server -./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --page-server --dedup -./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --pre-dump-mode read + bash -x ./test/jenkins/criu-fault.sh + if [ "$UNAME_M" == "x86_64" ]; then + # This fails on aarch64 (aws-graviton2) with: + # 33: ERR: thread-bomb.c:49: pthread_attr_setstacksize(): 22 + bash -x ./test/jenkins/criu-fcg.sh + fi + bash -x ./test/jenkins/criu-inhfd.sh -./test/zdtm.py run -t zdtm/transition/pid_reuse --pre 2 # start time based pid reuse detection -./test/zdtm.py run -t zdtm/transition/pidfd_store_sk --rpc --pre 2 # pidfd based pid reuse detection + if [ -z "$SKIP_EXT_DEV_TEST" ]; then + make -C test/others/mnt-ext-dev/ run + if criu/criu check --feature move_mount_set_group; then + EXTRA_OPTS=--mntns-compat-mode make -C test/others/mnt-ext-dev/ run + fi + fi -./test/zdtm.py run -t zdtm/static/socket-tcp-local --norst + make -C test/others/make/ run CC="$CC" + if [ -n "$CIRCLECI" ]; then + # GitHub Actions (and Cirrus CI) does not provide a real TTY + # and CRIU will fail with: + # Error (criu/tty.c:1014): tty: Don't have tty to inherit + # session from, aborting + make -C test/others/shell-job/ run + fi + make -C test/others/criu-ns/ run + make -C test/others/skip-file-rwx-check/ run + make -C test/others/rpc/ run -ip net add test -./test/zdtm.py run -t zdtm/static/env00 -f h --join-ns + ./test/zdtm.py run -t zdtm/static/env00 --sibling -# RPC testing -./test/zdtm.py run -t zdtm/static/env00 --rpc # Basic -./test/zdtm.py run -t zdtm/static/env00 --rpc --pre 2 --page-server -./test/zdtm.py run -t zdtm/static/ptrace_sig -f h --rpc # Error handling (crfail test) + ./test/zdtm.py run -t zdtm/static/maps00 --preload-libfault + ./test/zdtm.py run -t zdtm/static/maps02 --preload-libfault -./test/zdtm.py run --empty-ns -T zdtm/static/socket-tcp*-local --iter 2 + ./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --dedup + ./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --noauto-dedup + ./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --page-server + ./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --page-server --dedup + ./test/zdtm.py run -t zdtm/transition/maps007 --pre 2 --pre-dump-mode read -./test/zdtm.py run -t zdtm/static/env00 -t zdtm/transition/fork -t zdtm/static/ghost_holes00 -t zdtm/static/socket-tcp -t zdtm/static/msgque -k always -./test/crit-recode.py + ./test/zdtm.py run -t zdtm/transition/pid_reuse --pre 2 # start time based pid reuse detection + ./test/zdtm.py run -t zdtm/transition/pidfd_store_sk --rpc --pre 2 # pidfd based pid reuse detection -# Rootless tests -# Check if cap_checkpoint_restore is supported and also if unshare -c is supported. -# -# Do not run this test in a container (see https://github.com/checkpoint-restore/criu/issues/2312). -# Before v6.8-rc1~215^2~6, the kernel currently did not show correct device and -# inode numbers in /proc/pid/maps for stackable file systems. -skip=0 -findmnt -no FSTYPE / | grep overlay && { - ./criu/criu check --feature overlayfs_maps || skip=1 + ./test/zdtm.py run -t zdtm/static/socket-tcp-local --norst + + ip net add test + ./test/zdtm.py run -t zdtm/static/env00 -f h --join-ns + + # RPC testing + ./test/zdtm.py run -t zdtm/static/env00 --rpc # Basic + ./test/zdtm.py run -t zdtm/static/env00 --rpc --pre 2 --page-server + ./test/zdtm.py run -t zdtm/static/ptrace_sig -f h --rpc # Error handling (crfail test) + + ./test/zdtm.py run --empty-ns -T zdtm/static/socket-tcp*-local --iter 2 + + ./test/zdtm.py run -t zdtm/static/env00 -t zdtm/transition/fork -t zdtm/static/ghost_holes00 -t zdtm/static/socket-tcp -t zdtm/static/msgque -k always + ./test/crit-recode.py + + # Rootless tests + # Check if cap_checkpoint_restore is supported and also if unshare -c + # is supported. + # + # Do not run this test in a container + # (see https://github.com/checkpoint-restore/criu/issues/2312). + # Before v6.8-rc1~215^2~6, the kernel currently did not show correct + # device and inode numbers in /proc/pid/maps for stackable file + # systems. + skip=0 + findmnt -no FSTYPE / | grep overlay && { + ./criu/criu check --feature overlayfs_maps || skip=1 + } + unshare -c /bin/true || skip=1 + capsh --supports=cap_checkpoint_restore || skip=1 + + if [ "$skip" == 0 ]; then + make -C test/zdtm/ cleanout + rm -rf test/dump + setcap cap_checkpoint_restore,cap_sys_ptrace+eip criu/criu + if [ -d /sys/fs/selinux ] && command -v getenforce &>/dev/null; then + # Note: selinux in Enforcing mode prevents us from + # calling clone3() or writing to ns_last_pid on + # restore; hence set to Permissive for the test and + # then set back. + selinuxmode=$(getenforce) + if [ "$selinuxmode" != "Disabled" ]; then + setenforce Permissive + fi + + fi + # Run it as non-root in a user namespace. Since + # CAP_CHECKPOINT_RESTORE behaves differently in non-user + # namespaces (e.g. no access to map_files) this tests that we + # can dump and restore under those conditions. Note that the + # "... && true" part is necessary; we need at least one + # statement after the tests so that bash can reap zombies in + # the user namespace, otherwise it will exec the last statement + # and get replaced and nobody will be left to reap our zombies. + sudo --user=#65534 --group=#65534 unshare -Ucfpm --mount-proc -- bash -c "./test/zdtm.py run -t zdtm/static/maps00 -f h --rootless && true" + if [ -d /sys/fs/selinux ] && command -v getenforce &>/dev/null; then + if [ "$selinuxmode" != "Disabled" ]; then + setenforce "$selinuxmode" + fi + fi + setcap -r criu/criu + else + echo "Skipping unprivileged mode tests" + fi + + # more crit testing + make -C test/others/crit run + + # coredump testing + make -C test/others/criu-coredump run + + # libcriu testing + make -C test/others/libcriu run + + # criu-service-client testing + make -C contrib/criu-service-client/test run + + # external namespace testing + make -C test/others/ns_ext run + + # config file parser and parameter testing + make -C test/others/config-file run + + # action script testing + make -C test/others/action-script run + + # Skip all further tests when running with GCOV=1 + # The one test which currently cannot handle GCOV testing is + # compel/test. Probably because the GCOV Makefile infrastructure + # does not exist in compel. + [ -n "$GCOV" ] && return 0 + + # compel testing + make -C compel/test + + # amdgpu and cuda plugin testing + make amdgpu_plugin + make -C plugins/amdgpu/ test_topology_remap + ./plugins/amdgpu/test_topology_remap + + ./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin cuda + ./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin amdgpu + ./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin amdgpu cuda + ./test/zdtm.py run -t zdtm/static/busyloop00 --criu-plugin inventory_test_enabled inventory_test_disabled + + ./test/zdtm.py run -t zdtm/static/sigpending -t zdtm/static/pthread00 --mocked-cuda-checkpoint --fault 138 } -unshare -c /bin/true || skip=1 -capsh --supports=cap_checkpoint_restore || skip=1 - -if [ "$skip" == 0 ]; then - make -C test/zdtm/ cleanout - rm -rf test/dump - setcap cap_checkpoint_restore,cap_sys_ptrace+eip criu/criu - if [ -d /sys/fs/selinux ] && command -v getenforce &>/dev/null; then - # Note: selinux in Enforcing mode prevents us from calling clone3() or writing to ns_last_pid on restore; hence set to Permissive for the test and then set back. - selinuxmode=$(getenforce) - if [ "$selinuxmode" != "Disabled" ]; then - setenforce Permissive - fi +# When sharding is enabled, shards 0..count-1 run sharded zdtm tests and +# shard "count" (the extra shard) runs only the non-shardable tests. +# When sharding is not enabled, run everything sequentially. +if [ -z "$ZDTM_SHARD_COUNT" ] || [ "$ZDTM_SHARD_COUNT" -eq 0 ]; then + # No sharding: run all zdtm tests followed by non-shardable tests + ./test/zdtm.py run -a -p 2 --keep-going "${ZDTM_OPTS[@]}" + if criu/criu check --feature move_mount_set_group; then + ./test/zdtm.py run -a -p 2 --mntns-compat-mode --keep-going "${ZDTM_OPTS[@]}" fi - # Run it as non-root in a user namespace. Since CAP_CHECKPOINT_RESTORE behaves differently in non-user namespaces (e.g. no access to map_files) this tests that we can dump and restore - # under those conditions. Note that the "... && true" part is necessary; we need at least one statement after the tests so that bash can reap zombies in the user namespace, - # otherwise it will exec the last statement and get replaced and nobody will be left to reap our zombies. - sudo --user=#65534 --group=#65534 unshare -Ucfpm --mount-proc -- bash -c "./test/zdtm.py run -t zdtm/static/maps00 -f h --rootless && true" - if [ -d /sys/fs/selinux ] && command -v getenforce &>/dev/null; then - if [ "$selinuxmode" != "Disabled" ]; then - setenforce "$selinuxmode" - fi - fi - setcap -r criu/criu + ./test/zdtm.py run -a -p 2 --keep-going --criu-config "${ZDTM_OPTS[@]}" + run_non_shardable_tests +elif [ "$ZDTM_SHARD_INDEX" -eq "$ZDTM_SHARD_COUNT" ]; then + # This is the extra non-shardable shard (index == count, e.g. shard 4 + # when count is 4). Only run non-shardable tests, skip zdtm shards. + run_non_shardable_tests else - echo "Skipping unprivileged mode tests" + # Shards 0..count-1: run only the sharded zdtm tests + ./test/zdtm.py run -a -p 2 --keep-going "${SHARD_OPTS[@]}" "${ZDTM_OPTS[@]}" + if criu/criu check --feature move_mount_set_group; then + ./test/zdtm.py run -a -p 2 --mntns-compat-mode --keep-going "${SHARD_OPTS[@]}" "${ZDTM_OPTS[@]}" + fi + ./test/zdtm.py run -a -p 2 --keep-going --criu-config "${SHARD_OPTS[@]}" "${ZDTM_OPTS[@]}" fi - -# more crit testing -make -C test/others/crit run - -# coredump testing -make -C test/others/criu-coredump run - -# libcriu testing -make -C test/others/libcriu run - -# criu-service-client testing -make -C contrib/criu-service-client/test run - -# external namespace testing -make -C test/others/ns_ext run - -# config file parser and parameter testing -make -C test/others/config-file run - -# action script testing -make -C test/others/action-script run - -# Skip all further tests when running with GCOV=1 -# The one test which currently cannot handle GCOV testing is compel/test -# Probably because the GCOV Makefile infrastructure does not exist in compel -[ -n "$GCOV" ] && exit 0 - -# compel testing -make -C compel/test - -# amdgpu and cuda plugin testing -make amdgpu_plugin -make -C plugins/amdgpu/ test_topology_remap -./plugins/amdgpu/test_topology_remap - -./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin cuda -./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin amdgpu -./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin amdgpu cuda -./test/zdtm.py run -t zdtm/static/busyloop00 --criu-plugin inventory_test_enabled inventory_test_disabled - -./test/zdtm.py run -t zdtm/static/sigpending -t zdtm/static/pthread00 --mocked-cuda-checkpoint --fault 138