Mirrors/criu
1
0
Fork 0
mirror of https://github.com/checkpoint-restore/criu.git synced 2026-07-17 16:47:50 +00:00
Code Issues Projects Releases Wiki Activity

compel: remove hardware breakpoint usage

Browse source 
Hardware breakpoints were originally intended to speed up the resume process
by stopping the process at a specific point in the pie code. However, it
turned out that they don't provide a significant speedup and, in some cases,
can even slow it down. This is especially critical for hosts with a large
number of CPUs.

Hardware Breakpoint Restore Performance Benchmark
==================================================

System: Linux 6.17.0-1007-gcp x86_64
CPU:    INTEL(R) XEON(R) PLATINUM 8581C CPU @ 2.10GHz
Virt:   google
CRIU:   Version: 4.2
Iterations per data point: 5

Threads         With BP (us) Without BP (us)     Diff (%)
--------        ------------ ---------------     --------
>>> Benchmarking with 1 thread(s)...
1                        391             326        19.9%
>>> Benchmarking with 10 thread(s)...
10                      1098             695        58.0%
>>> Benchmarking with 50 thread(s)...
50                      3772            2344        60.9%
>>> Benchmarking with 100 thread(s)...
100                     6740            4504        49.6%
>>> Benchmarking with 500 thread(s)...
500                    31382           19982        57.1%
>>> Benchmarking with 1000 thread(s)...
1000                   58617           40568        44.5%

Notes:
  'With BP'    = hardware breakpoints enabled (current default)
  'Without BP' = CRIU_FAULT=130 (FI_NO_BREAKPOINTS, uses PTRACE_SYSCALL)
  Positive diff% means breakpoints are slower

Signed-off-by: Andrei Vagin <avagin@google.com>
This commit is contained in:
Andrei Vagin 2026-03-09 00:19:15 +00:00 • committed by Andrei Vagin
parent 869d481cf2
commit 7daebbe469
34 changed files with 34 additions and 496 deletions
Download patch file Download diff file Expand all files Collapse all files

42
compel/arch/aarch64/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,42 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
#include <sys/types.h>
#include <stdbool.h>
struct hwbp_cap {
char arch;
char bp_count;
};
/* copied from `linux/arch/arm64/include/asm/hw_breakpoint.h` */
/* Lengths */
#define ARM_BREAKPOINT_LEN_1 0x1
#define ARM_BREAKPOINT_LEN_2 0x3
#define ARM_BREAKPOINT_LEN_3 0x7
#define ARM_BREAKPOINT_LEN_4 0xf
#define ARM_BREAKPOINT_LEN_5 0x1f
#define ARM_BREAKPOINT_LEN_6 0x3f
#define ARM_BREAKPOINT_LEN_7 0x7f
#define ARM_BREAKPOINT_LEN_8 0xff
/* Privilege Levels */
#define AARCH64_BREAKPOINT_EL1 1
#define AARCH64_BREAKPOINT_EL0 2
/* Breakpoint */
#define ARM_BREAKPOINT_EXECUTE 0
/* Watchpoints */
#define ARM_BREAKPOINT_LOAD 1
#define ARM_BREAKPOINT_STORE 2
#define AARCH64_ESR_ACCESS_MASK (1 << 6)
#define DISABLE_HBP 0
#define ENABLE_HBP 1
int ptrace_set_breakpoint(pid_t pid, void *addr);
int ptrace_flush_breakpoints(pid_t pid);
#endif

109
compel/arch/aarch64/src/lib/infect.c
View file

@ -12,7 +12,6 @@
#include "errno.h"
#include "infect.h"
#include "infect-priv.h"
#include "asm/breakpoints.h"
#include "asm/gcs-types.h"
#include <linux/prctl.h>
@ -255,114 +254,6 @@ unsigned long compel_task_size(void)
return task_size;
}
static struct hwbp_cap *ptrace_get_hwbp_cap(pid_t pid)
{
static struct hwbp_cap info;
static int available = -1;
if (available == -1) {
unsigned int val;
struct iovec iovec = {
.iov_base = &val,
.iov_len = sizeof(val),
};
if (ptrace(PTRACE_GETREGSET, pid, NT_ARM_HW_BREAK, &iovec) < 0)
available = 0;
else {
info.arch = (char)((val >> 8) & 0xff);
info.bp_count = (char)(val & 0xff);
available = (info.arch != 0);
}
}
return available == 1 ? &info : NULL;
}
int ptrace_set_breakpoint(pid_t pid, void *addr)
{
k_rtsigset_t block;
struct hwbp_cap *info = ptrace_get_hwbp_cap(pid);
struct user_hwdebug_state regs = {};
unsigned int ctrl = 0;
struct iovec iovec;
if (info == NULL || info->bp_count == 0)
return 0;
/*
* The struct is copied from `arch/arm64/include/asm/hw_breakpoint.h` in
* linux kernel:
* struct arch_hw_breakpoint_ctrl {
* __u32 __reserved : 19,
* len : 8,
* type : 2,
* privilege : 2,
* enabled : 1;
* };
*
* The part of `struct arch_hw_breakpoint_ctrl` bits meaning is defined
* in <<ARM Architecture Reference Manual for A-profile architecture>>,
* D13.3.2 DBGBCR<n>_EL1, Debug Breakpoint Control Registers.
*/
ctrl = ARM_BREAKPOINT_LEN_4;
ctrl = (ctrl << 2) | ARM_BREAKPOINT_EXECUTE;
ctrl = (ctrl << 2) | AARCH64_BREAKPOINT_EL0;
ctrl = (ctrl << 1) | ENABLE_HBP;
regs.dbg_regs[0].addr = (__u64)addr;
regs.dbg_regs[0].ctrl = ctrl;
iovec.iov_base = &regs;
iovec.iov_len = (offsetof(struct user_hwdebug_state, dbg_regs) + sizeof(regs.dbg_regs[0]));
if (ptrace(PTRACE_SETREGSET, pid, NT_ARM_HW_BREAK, &iovec))
return -1;
/*
* FIXME(issues/1429): SIGTRAP can't be blocked, otherwise its handler
* will be reset to the default one.
*/
ksigfillset(&block);
ksigdelset(&block, SIGTRAP);
if (ptrace(PTRACE_SETSIGMASK, pid, sizeof(k_rtsigset_t), &block)) {
pr_perror("Can't block signals for %d", pid);
return -1;
}
if (ptrace(PTRACE_CONT, pid, NULL, NULL) != 0) {
pr_perror("Unable to restart the stopped tracee process %d", pid);
return -1;
}
return 1;
}
int ptrace_flush_breakpoints(pid_t pid)
{
struct hwbp_cap *info = ptrace_get_hwbp_cap(pid);
struct user_hwdebug_state regs = {};
unsigned int ctrl = 0;
struct iovec iovec;
if (info == NULL || info->bp_count == 0)
return 0;
ctrl = ARM_BREAKPOINT_LEN_4;
ctrl = (ctrl << 2) | ARM_BREAKPOINT_EXECUTE;
ctrl = (ctrl << 2) | AARCH64_BREAKPOINT_EL0;
ctrl = (ctrl << 1) | DISABLE_HBP;
regs.dbg_regs[0].addr = 0ul;
regs.dbg_regs[0].ctrl = ctrl;
iovec.iov_base = &regs;
iovec.iov_len = (offsetof(struct user_hwdebug_state, dbg_regs) + sizeof(regs.dbg_regs[0]));
if (ptrace(PTRACE_SETREGSET, pid, NT_ARM_HW_BREAK, &iovec))
return -1;
return 0;
}
int inject_gcs_cap_token(struct parasite_ctl *ctl, pid_t pid, struct cr_user_gcs *gcs)
{
struct iovec gcs_iov = { .iov_base = gcs, .iov_len = sizeof(*gcs) };

15
compel/arch/arm/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,15 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
static inline int ptrace_set_breakpoint(pid_t pid, void *addr)
{
return 0;
}
static inline int ptrace_flush_breakpoints(pid_t pid)
{
return 0;
}
#endif

1
compel/arch/arm/src/lib/include/uapi/asm/infect-types.h
View file

@ -2,6 +2,7 @@
#define UAPI_COMPEL_ASM_TYPES_H__
#include <stdint.h>
#include <signal.h>
#include <sys/mman.h>
#define SIGMAX 64

6
compel/arch/loongarch64/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,6 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
extern int ptrace_set_breakpoint(pid_t pid, void *addr);
extern int ptrace_flush_breakpoints(pid_t pid);
#endif

2
compel/arch/loongarch64/src/lib/include/uapi/asm/infect-types.h
View file

@ -2,9 +2,11 @@
#define UAPI_COMPEL_ASM_TYPES_H__
#include <stdint.h>
#include <signal.h>
#define SIGMAX 64
#define SIGMAX_OLD 31
#define ARCH_SI_TRAP TRAP_BRKPT
/*
* From the Linux kernel header arch/loongarch/include/uapi/asm/ptrace.h

13
compel/arch/loongarch64/src/lib/infect.c
View file

@ -174,19 +174,6 @@ int arch_fetch_sas(struct parasite_ctl *ctl, struct rt_sigframe *s)
return err ? err : ret;
}
/*
* TODO: add feature
*/
int ptrace_set_breakpoint(pid_t pid, void *addr)
{
return 0;
}
int ptrace_flush_breakpoints(pid_t pid)
{
return 0;
}
/*
* Refer to Linux kernel arch/loongarch/include/asm/processor.h
*/

6
compel/arch/mips/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,6 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
extern int ptrace_set_breakpoint(pid_t pid, void *addr);
extern int ptrace_flush_breakpoints(pid_t pid);
#endif

1
compel/arch/mips/src/lib/include/uapi/asm/infect-types.h
View file

@ -8,6 +8,7 @@
#include <linux/types.h>
#define SIGMAX 64
#define SIGMAX_OLD 31
#define ARCH_SI_TRAP TRAP_BRKPT
/*
* Copied from the Linux kernel header arch/mips/include/asm/ptrace.h

10
compel/arch/mips/src/lib/infect.c
View file

@ -226,16 +226,6 @@ int arch_fetch_sas(struct parasite_ctl *ctl, struct rt_sigframe *s)
return err ? err : ret;
}
int ptrace_set_breakpoint(pid_t pid, void *addr)
{
return 0;
}
int ptrace_flush_breakpoints(pid_t pid)
{
return 0;
}
/*refer to kernel linux-3.10/arch/mips/include/asm/processor.h*/
#define TASK_SIZE32 0x7fff8000UL
#define TASK_SIZE64 0x10000000000UL

15
compel/arch/ppc64/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,15 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
static inline int ptrace_set_breakpoint(pid_t pid, void *addr)
{
return 0;
}
static inline int ptrace_flush_breakpoints(pid_t pid)
{
return 0;
}
#endif

15
compel/arch/riscv64/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,15 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
static inline int ptrace_set_breakpoint(pid_t pid, void *addr)
{
return 0;
}
static inline int ptrace_flush_breakpoints(pid_t pid)
{
return 0;
}
#endif

15
compel/arch/s390/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,15 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP TRAP_BRKPT
static inline int ptrace_set_breakpoint(pid_t pid, void *addr)
{
return 0;
}
static inline int ptrace_flush_breakpoints(pid_t pid)
{
return 0;
}
#endif

1
compel/arch/s390/src/lib/include/uapi/asm/infect-types.h
View file

@ -9,6 +9,7 @@
#define SIGMAX 64
#define SIGMAX_OLD 31
#define ARCH_SI_TRAP TRAP_BRKPT
/*
* Definitions from /usr/include/asm/ptrace.h:

6
compel/arch/x86/src/lib/include/uapi/asm/breakpoints.h
View file

@ -1,6 +0,0 @@
#ifndef __COMPEL_BREAKPOINTS_H__
#define __COMPEL_BREAKPOINTS_H__
#define ARCH_SI_TRAP SI_KERNEL
extern int ptrace_set_breakpoint(pid_t pid, void *addr);
extern int ptrace_flush_breakpoints(pid_t pid);
#endif

1
compel/arch/x86/src/lib/include/uapi/asm/infect-types.h
View file

@ -8,6 +8,7 @@
#define SIGMAX 64
#define SIGMAX_OLD 31
#define ARCH_SI_TRAP SI_KERNEL
#define ARCH_HAS_PTRACE_GET_THREAD_AREA

64
compel/arch/x86/src/lib/infect.c
View file

@ -633,70 +633,6 @@ int arch_fetch_sas(struct parasite_ctl *ctl, struct rt_sigframe *s)
return err ? err : ret;
}
/* Copied from the gdb header gdb/nat/x86-dregs.h */
/* Debug registers' indices. */
#define DR_FIRSTADDR 0
#define DR_LASTADDR 3
#define DR_NADDR 4 /* The number of debug address registers. */
#define DR_STATUS 6 /* Index of debug status register (DR6). */
#define DR_CONTROL 7 /* Index of debug control register (DR7). */
#define DR_LOCAL_ENABLE_SHIFT 0 /* Extra shift to the local enable bit. */
#define DR_GLOBAL_ENABLE_SHIFT 1 /* Extra shift to the global enable bit. */
#define DR_ENABLE_SIZE 2 /* Two enable bits per debug register. */
/* Locally enable the break/watchpoint in the I'th debug register. */
#define X86_DR_LOCAL_ENABLE(i) (1 << (DR_LOCAL_ENABLE_SHIFT + DR_ENABLE_SIZE * (i)))
int ptrace_set_breakpoint(pid_t pid, void *addr)
{
k_rtsigset_t block;
int ret;
/* Set a breakpoint */
if (ptrace(PTRACE_POKEUSER, pid, offsetof(struct user, u_debugreg[DR_FIRSTADDR]), addr)) {
pr_perror("Unable to setup a breakpoint into %d", pid);
return -1;
}
/* Enable the breakpoint */
if (ptrace(PTRACE_POKEUSER, pid, offsetof(struct user, u_debugreg[DR_CONTROL]),
X86_DR_LOCAL_ENABLE(DR_FIRSTADDR))) {
pr_perror("Unable to enable the breakpoint for %d", pid);
return -1;
}
/*
* FIXME(issues/1429): SIGTRAP can't be blocked, otherwise its handler
* will be reset to the default one.
*/
ksigfillset(&block);
ksigdelset(&block, SIGTRAP);
if (ptrace(PTRACE_SETSIGMASK, pid, sizeof(k_rtsigset_t), &block)) {
pr_perror("Can't block signals for %d", pid);
return -1;
}
ret = ptrace(PTRACE_CONT, pid, NULL, NULL);
if (ret) {
pr_perror("Unable to restart the stopped tracee process %d", pid);
return -1;
}
return 1;
}
int ptrace_flush_breakpoints(pid_t pid)
{
/* Disable the breakpoint */
if (ptrace(PTRACE_POKEUSER, pid, offsetof(struct user, u_debugreg[DR_CONTROL]), 0)) {
pr_perror("Unable to disable the breakpoint for %d", pid);
return -1;
}
return 0;
}
int ptrace_get_regs(pid_t pid, user_regs_struct_t *regs)
{
struct iovec iov;

1
compel/include/infect-priv.h
View file

@ -19,7 +19,6 @@ struct parasite_ctl {
int rpid; /* Real pid of the victim */
void *remote_map;
void *local_map;
void *sigreturn_addr; /* A place for the breakpoint */
unsigned long map_length;
struct infect_ctx ictx;

1
compel/include/rpc-pie-priv.h
View file

@ -41,7 +41,6 @@ struct parasite_init_args {
int32_t h_addr_len;
struct sockaddr_un h_addr;
int32_t log_level;
uint64_t sigreturn_addr;
uint64_t sigframe; /* pointer to sigframe */
futex_t daemon_connected;
#ifdef ARCH_HAS_LONG_PAGES

4
compel/include/uapi/infect.h
View file

@ -95,8 +95,6 @@ enum trace_flags {
extern int __must_check compel_stop_on_syscall(int tasks, int sys_nr, int sys_nr_compat);
extern int compel_stop_tasks_on_syscall(int tasks, pid_t *pids, const int sys_nr, const int sys_nr_compat);
extern int __must_check compel_stop_pie(pid_t pid, void *addr, bool no_bp);
extern int __must_check compel_unmap(struct parasite_ctl *ctl, unsigned long addr);
extern int compel_mode_native(struct parasite_ctl *ctl);
@ -139,8 +137,6 @@ extern struct infect_ctx *compel_infect_ctx(struct parasite_ctl *);
#define INFECT_NO_MEMFD (1UL << 0)
/* Make parasite connect() fail */
#define INFECT_FAIL_CONNECT (1UL << 1)
/* No breakpoints in pie tracking */
#define INFECT_NO_BREAKPOINTS (1UL << 2)
/* Can run parasite inside compat tasks */
#define INFECT_COMPATIBLE (1UL << 3)
/* Workaround for ptrace bug on Skylake CPUs with kernels older than v4.14 */

2
compel/include/uapi/ptrace.h
View file

@ -11,8 +11,6 @@
#include <sys/ptrace.h>
#include <stdint.h>
#include <compel/asm/breakpoints.h>
/*
* Some constants for ptrace that might be missing from the
* standard library includes due to being (relatively) new.

1
compel/plugins/std/infect.c
View file

@ -153,7 +153,6 @@ static noinline __used unsigned long parasite_init_daemon(void *data)
struct parasite_init_args *args = data;
int ret;
args->sigreturn_addr = (uint64_t)(uintptr_t)fini_sigreturn;
sigframe = (void *)(uintptr_t)args->sigframe;
#ifdef ARCH_HAS_LONG_PAGES
__page_size = args->page_size;

85
compel/src/lib/infect.c
View file

@ -725,7 +725,6 @@ static int parasite_init_daemon(struct parasite_ctl *ctl)
goto err;
}
ctl->sigreturn_addr = (void *)(uintptr_t)args->sigreturn_addr;
ctl->daemonized = true;
pr_info("Parasite %d has been switched to daemon mode\n", pid);
return 0;
@ -1435,9 +1434,10 @@ static int parasite_fini_seized(struct parasite_ctl *ctl)
return -1;
/* Go to sigreturn as closer as we can */
ret = compel_stop_pie(pid, ctl->sigreturn_addr, ctl->ictx.flags & INFECT_NO_BREAKPOINTS);
if (ret < 0)
return ret;
if (ptrace(PTRACE_SYSCALL, pid, NULL, NULL)) {
pr_perror("Unable to restart the %d process", pid);
return -1;
}
if (compel_stop_on_syscall(1, __NR(rt_sigreturn, 0), __NR(rt_sigreturn, 1)))
return -1;
@ -1588,38 +1588,6 @@ err:
return ret;
}
int compel_stop_pie(pid_t pid, void *addr, bool no_bp)
{
int ret;
if (no_bp) {
pr_debug("Force no-breakpoints restore of %d\n", pid);
ret = 0;
} else
ret = ptrace_set_breakpoint(pid, addr);
if (ret < 0)
return ret;
if (ret > 0) {
/*
* PIE will stop on a breakpoint, next
* stop after that will be syscall enter.
*/
return 0;
}
/*
* No breakpoints available -- start tracing it
* in a per-syscall manner.
*/
ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
if (ret) {
pr_perror("Unable to restart the %d process", pid);
return -1;
}
return 0;
}
static bool task_is_trapped(int status, pid_t pid)
{
if (WIFSTOPPED(status) && (WSTOPSIG(status) & ~PTRACE_SYSCALL_TRAP) == SIGTRAP)
@ -1675,22 +1643,19 @@ int compel_stop_on_syscall(int tasks, const int sys_nr, const int sys_nr_compat)
pr_debug("%d was trapped\n", pid);
if ((WSTOPSIG(status) & PTRACE_SYSCALL_TRAP) == 0) {
/*
* On some platforms such as ARM64, it is impossible to
* pass through a breakpoint, so let's clear it right
* after it has been triggered.
*/
if (ptrace_flush_breakpoints(pid)) {
pr_err("Unable to clear breakpoints\n");
return -1;
}
goto goon;
if (!(WSTOPSIG(status) & PTRACE_SYSCALL_TRAP)) {
pr_err("Task %d is in unexpected state: %x\n", pid, status);
return -1;
}
if (trace == TRACE_EXIT) {
trace = TRACE_ENTER;
pr_debug("`- Expecting exit\n");
goto goon;
ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
if (ret) {
pr_perror("ptrace");
return -1;
}
continue;
}
if (trace == TRACE_ENTER)
trace = TRACE_EXIT;
@ -1721,12 +1686,17 @@ int compel_stop_on_syscall(int tasks, const int sys_nr, const int sys_nr_compat)
if (!task_is_trapped(status, pid))
return -1;
if (!(WSTOPSIG(status) & PTRACE_SYSCALL_TRAP)) {
pr_err("Task %d is in unexpected state: %x\n", pid, status);
return -1;
}
pr_debug("%d was stopped\n", pid);
tasks--;
continue;
}
goon:
ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
if (ret) {
pr_perror("ptrace");
return -1;
@ -1785,18 +1755,11 @@ int compel_stop_tasks_on_syscall(int nr_tasks, pid_t *pids, const int sys_nr, co
pr_debug("%d was trapped\n", pid);
if ((WSTOPSIG(status) & PTRACE_SYSCALL_TRAP) == 0) {
/*
* On some platforms such as ARM64, it is impossible to
* pass through a breakpoint, so let's clear it right
* after it has been triggered.
*/
if (ptrace_flush_breakpoints(pid)) {
pr_err("Unable to clear breakpoints\n");
goto err;
}
goto goon;
if (!(WSTOPSIG(status) & PTRACE_SYSCALL_TRAP)) {
pr_err("Task %d is in unexpected state: %x\n", pid, status);
goto err;
}
if (done[i] == SYS_TRAP_ENTER) {
done[i] = SYS_TRAP_EXIT;
continue;
@ -1810,7 +1773,7 @@ int compel_stop_tasks_on_syscall(int nr_tasks, pid_t *pids, const int sys_nr, co
if (is_required_syscall(&regs, pid, sys_nr, sys_nr_compat))
done[i] = SYS_TRAP_ENTER;
goon:
/* Let this task run while updating the others. */
ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
if (ret) {