Mirrors/criu
1
0
Fork 0
mirror of https://github.com/checkpoint-restore/criu.git synced 2026-07-18 17:14:28 +00:00
Code Issues Projects Releases Wiki Activity

Merge c5150042fd into b47c692bb3

Browse source
This commit is contained in:
tursulin 2026-07-14 17:37:23 +00:00 • committed by GitHub
commit a2b43eeb5f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 635 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

62
plugins/amdgpu/amdgpu_drm.h
View file

@ -58,6 +58,8 @@ extern "C" {
#define DRM_AMDGPU_USERQ_SIGNAL 0x17
#define DRM_AMDGPU_USERQ_WAIT 0x18
#define DRM_AMDGPU_GEM_LIST_HANDLES 0x19
#define DRM_AMDGPU_GEM_LIST_CONTEXTS 0x20
#define DRM_AMDGPU_GEM_COPY_BUFFER 0x21
/* not upstream */
#define DRM_AMDGPU_GEM_DGMA 0x5c
@ -84,6 +86,8 @@ extern "C" {
#define DRM_IOCTL_AMDGPU_USERQ_SIGNAL DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_SIGNAL, struct drm_amdgpu_userq_signal)
#define DRM_IOCTL_AMDGPU_USERQ_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait)
#define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles)
#define DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_CONTEXTS, struct drm_amdgpu_gem_list_contexts)
#define DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_COPY_BUFFER, union drm_amdgpu_gem_copy_buffer)
#define DRM_IOCTL_AMDGPU_GEM_DGMA DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_DGMA, struct drm_amdgpu_gem_dgma)
@ -231,6 +235,20 @@ union drm_amdgpu_gem_create {
struct drm_amdgpu_gem_create_out out;
};
struct drm_amdgpu_gem_copy_buffer_in {
__u32 src_handle;
__u32 dst_handle;
};
struct drm_amdgpu_gem_copy_buffer_out {
__u32 fence_fd;
};
union drm_amdgpu_gem_copy_buffer {
struct drm_amdgpu_gem_copy_buffer_in in;
struct drm_amdgpu_gem_copy_buffer_out out;
};
/** Opcode to create new residency list. */
#define AMDGPU_BO_LIST_OP_CREATE 0
/** Opcode to destroy previously created residency list */
@ -276,6 +294,7 @@ union drm_amdgpu_bo_list {
#define AMDGPU_CTX_OP_QUERY_STATE2 4
#define AMDGPU_CTX_OP_GET_STABLE_PSTATE 5
#define AMDGPU_CTX_OP_SET_STABLE_PSTATE 6
#define AMDGPU_CTX_OP_CHANGE_HANDLE 7
/* GPU reset status */
#define AMDGPU_CTX_NO_RESET 0
@ -898,6 +917,7 @@ struct drm_amdgpu_gem_op {
};
#define AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT (1 << 0)
#define AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR (1 << 1)
struct drm_amdgpu_gem_list_handles {
/* User pointer to array of drm_amdgpu_gem_bo_info_entry */
@ -913,7 +933,7 @@ struct drm_amdgpu_gem_list_handles_entry {
/* gem handle of buffer object */
__u32 gem_handle;
/* Currently just one flag: IS_IMPORT */
/* AMDGPU_GEM_LIST_HANDLES_FLAG_* */
__u32 flags;
/* Size of bo */
@ -922,11 +942,45 @@ struct drm_amdgpu_gem_list_handles_entry {
/* Preferred domains for GEM_CREATE */
__u64 preferred_domains;
/* GEM_CREATE flags for re-creation of buffer */
/* GEM_CREATE flags for re-creation of buffer or drm_amdgpu_gem_userptr.flags */
__u64 alloc_flags;
/* physical start_addr alignment in bytes for some HW requirements */
__u64 alignment;
union {
/* physical start_addr alignment in bytes for some HW requirements */
__u64 alignment;
/* drm_amdgpu_gem_userptr.addr for userptr objects */
__u64 userptr;
};
};
struct drm_amdgpu_gem_list_contexts {
/* User pointer to array of drm_amdgpu_gem_list_contexts_entry */
__u64 contexts;
/* Size of the contexts buffer / Number of contexts in the client (if larger than size of buffer, must retry) */
__u32 num_contexts;
__u32 padding;
};
struct drm_amdgpu_gem_list_contexts_entry {
/* gem context handle */
__u32 handle;
/* AMDGPU_GEM_LIST_CONTEXTS_FLAG_* */
__u32 flags;
/* context initial priority */
__s32 init_priority;
/* context override priority */
__s32 override_priority;
/* pstate flags */
__u32 pstate_flags;
__u32 padding;
};
#define AMDGPU_VA_OP_MAP 1

113
plugins/amdgpu/amdgpu_plugin.c
View file

@ -62,6 +62,7 @@ struct vma_metadata {
static LIST_HEAD(update_vma_info_list);
static size_t kfd_max_buffer_size;
static bool amdgpu_ignore_single_gpuid;
static bool plugin_added_to_inventory = false;
@ -404,6 +405,8 @@ int amdgpu_plugin_init(int stage)
kfd_vram_size_check = getenv_bool("KFD_VRAM_SIZE_CHECK", true);
kfd_numa_check = getenv_bool("KFD_NUMA_CHECK", true);
kfd_capability_check = getenv_bool("KFD_CAPABILITY_CHECK", true);
amdgpu_ignore_single_gpuid = getenv_bool("AMDGPU_IGNORE_SINGLE_GPUID", false);
}
kfd_max_buffer_size = getenv_size_t("KFD_MAX_BUFFER_SIZE", 0);
@ -1873,11 +1876,11 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed)
size_t img_size;
int fd, ret;
/* This is restorer plugin for renderD nodes. Criu doesn't guarantee that they will
* be called before the plugin is called for kfd file descriptor.
* TODO: Currently, this code will only work if this function is called for /dev/kfd
* first as we assume restore_maps is already filled. Need to fix this later.
/* This is restorer plugin for renderD nodes. Criu doesn't guarantee
* that they will be called before the plugin is called for kfd file
* descriptor.
*/
snprintf(img_path, sizeof(img_path), IMG_DRM_FILE, id);
ret = load_img(img_path, &buf, &img_size);
if (ret < 0) {
@ -1906,14 +1909,71 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed)
pr_info("render node gpu_id = 0x%04x\n", rd->gpu_id);
target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id);
if (!target_gpu_id) {
fd = -ENODEV;
goto fail;
if (fd_next == -1) {
ret = find_unused_fd_pid(getpid());
if (ret < 0) {
pr_err("Failed to find unused fd (fd:%d)\n", ret);
fd = ret;
goto fail;
}
fd_next = ret;
}
if (!dest_topology.parsed) {
pr_info("Parsing local topology for render node restore\n");
ret = topology_parse(&dest_topology, "Local");
if (ret) {
pr_err("Failed to parse local system topology %d\n",
ret);
fd = ret;
goto fail;
}
}
if (restore_maps.mapped_cnt) {
target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id);
if (!target_gpu_id) {
pr_err("Unable to map gpu_id 0x%04x!\n", rd->gpu_id);
fd = -ENODEV;
goto fail;
}
} else {
unsigned int num_gpus = 0;
pr_info("Assuming same system with a single gpu_id 0x%04x\n",
rd->gpu_id);
list_for_each_entry(tp_node, &dest_topology.nodes,
listm_system) {
if (NODE_IS_GPU(tp_node)) {
num_gpus++;
target_gpu_id = tp_node->gpu_id;
}
}
if (num_gpus != 1) {
pr_err("Unexpectedly found %u GPUs!\n", num_gpus);
fd = -EINVAL;
goto fail;
} else if (target_gpu_id != rd->gpu_id &&
!amdgpu_ignore_single_gpuid) {
pr_err("Unexpectedly found gpu_id 0x%04x (expected 0x%04x)!\n",
target_gpu_id, rd->gpu_id);
fd = -EINVAL;
goto fail;
}
}
tp_node = sys_get_node_by_gpu_id(&dest_topology, target_gpu_id);
if (!tp_node) {
if (!tp_node && amdgpu_ignore_single_gpuid) {
tp_node = sys_get_node_by_index(&dest_topology, 0);
if (!NODE_IS_GPU(tp_node)) {
pr_err("Cannot find the GPU node!\n");
fd = -ENODEV;
goto fail;
}
target_gpu_id = tp_node->gpu_id;
pr_warn("Forcing restore on gpu_id=0x%04x\n", target_gpu_id);
} else if (!tp_node) {
pr_err("Unable to find target gpu_id=0x%04x!\n", target_gpu_id);
fd = -ENODEV;
goto fail;
}
@ -2151,25 +2211,28 @@ int amdgpu_plugin_update_vmamap(const char *in_path, const uint64_t addr, const
}
list_for_each_entry(vma_md, &update_vma_info_list, list) {
if (addr == vma_md->vma_entry && old_offset == vma_md->old_pgoff) {
*new_offset = vma_md->new_pgoff;
if (old_offset != vma_md->old_pgoff)
continue;
if (is_kfd && addr != vma_md->vma_entry)
continue;
*updated_fd = -1;
if (is_renderD) {
int fd = dup(vma_md->fd);
if (fd == -1) {
pr_perror("unable to duplicate the render fd");
return -1;
}
*updated_fd = fd;
*new_offset = vma_md->new_pgoff;
if (is_renderD) {
int fd = dup(vma_md->fd);
if (fd == -1) {
pr_perror("unable to duplicate the render fd");
return -1;
}
pr_debug("old_pgoff=0x%lx new_pgoff=0x%lx fd=%d\n",
vma_md->old_pgoff, vma_md->new_pgoff,
*updated_fd);
return 1;
*updated_fd = fd;
} else {
*updated_fd = -1;
}
pr_debug("old_pgoff=0x%lx new_pgoff=0x%lx fd=%d\n",
vma_md->old_pgoff, vma_md->new_pgoff, *updated_fd);
return 1;
}
pr_info("No match for addr:0x%lx offset:%lx\n", addr, old_offset);
return 0;

572
plugins/amdgpu/amdgpu_plugin_drm.c
View file

@ -95,6 +95,30 @@ static int allocate_bo_entries(CriuRenderNode *e, int num_bos)
return 0;
}
static int allocate_context_entries(CriuRenderNode *e, unsigned long num_contexts)
{
e->contexts = xmalloc(sizeof(DrmContextEntry *) * num_contexts);
if (!e->contexts) {
pr_err("Failed to allocate context list\n");
return -ENOMEM;
}
for (int i = 0; i < num_contexts; i++) {
DrmContextEntry *context = xzalloc(sizeof(*context));
if (!context) {
pr_err("Failed to allocate context info\n");
return -ENOMEM;
}
drm_context_entry__init(context);
e->contexts[i] = context;
e->n_contexts++;
}
return 0;
}
static int allocate_vm_entries(DrmBoEntry *e, int num_vms)
{
e->vm_entries = xmalloc(sizeof(DrmVmEntry *) * num_vms);
@ -125,6 +149,13 @@ static void free_e(CriuRenderNode *e)
if (e->bo_entries[i])
xfree(e->bo_entries[i]);
}
xfree(e->bo_entries);
for (int i = 0; i < e->n_contexts; i++) {
if (e->contexts[i])
xfree(e->contexts[i]);
}
xfree(e->contexts);
xfree(e);
}
@ -156,11 +187,62 @@ int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *st)
return 0;
}
static int sync_fence_wait(int fd, int timeout)
{
struct pollfd fds = { fd, POLLIN };
int ret;
while (true) {
ret = poll(&fds, 1, timeout);
if (ret > 0) {
if (fds.revents & (POLLERR | POLLNVAL))
return -EINVAL;
return 0;
} else if (ret == 0) {
return -ETIME;
} else {
ret = -errno;
if (ret == -EINTR || ret == -EAGAIN)
continue;
return ret;
}
}
}
static int
amdgpu_copy_buffer(int drm_fd, uint32_t src_handle, uint32_t dst_handle)
{
union drm_amdgpu_gem_copy_buffer copy_args = {
.in.src_handle = src_handle,
.in.dst_handle = dst_handle,
};
int ret;
ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER, &copy_args);
if (ret) {
ret = -errno;
pr_perror("Failed to copy buffer object");
return ret;
}
ret = sync_fence_wait(copy_args.out.fence_fd, 10000);
close(copy_args.out.fence_fd);
if (ret) {
pr_err("Failed to wait for buffer object copy! (%s)\n",
strerror(-ret));
}
return ret;
}
static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int drm_fd, int *dmabufs)
{
size_t image_size = 0, max_bo_size = 0, buffer_size;
struct amdgpu_gpu_info gpu_info = { 0 };
size_t image_size = 0, buffer_size = 0;
struct amdgpu_gpu_info gpu_info = {};
amdgpu_device_handle h_dev;
int has_copy_buffer = -1;
uint64_t max_copy_size;
uint32_t major, minor;
void *buffer = NULL;
@ -185,13 +267,18 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int
SDMA_LINEAR_COPY_MAX_SIZE - 1;
for (i = 0; i < rd->num_of_bos; i++) {
if (rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
if (rd->bo_entries[i]->size > max_bo_size)
max_bo_size = rd->bo_entries[i]->size;
DrmBoEntry *entry = rd->bo_entries[i];
if ((entry->preferred_domains &
(AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
!entry->is_userptr) {
if (entry->size > buffer_size)
buffer_size = entry->size;
}
}
buffer_size = max_bo_size;
if (!buffer_size)
goto exit;
ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), buffer_size);
if (ret) {
@ -202,27 +289,85 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int
}
for (i = 0; i < rd->num_of_bos; i++) {
if (!(rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)))
DrmBoEntry *entry = rd->bo_entries[i];
if (entry->is_userptr ||
!(entry->preferred_domains &
(AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) ||
entry->num_of_vms == 0)
continue;
if (rd->bo_entries[i]->num_of_vms == 0)
continue;
snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->id,
drm_render_minor, i);
snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->id, drm_render_minor, i);
/* Try AMDGPU_GEM_COPY_BUFFER first */
if ((has_copy_buffer < 0 || has_copy_buffer == 1)) {
struct drm_amdgpu_gem_userptr userptr_args = {
.addr = (uintptr_t)buffer,
.size = entry->size,
};
bo_contents_fd = open_img_file(img_path, false, &image_size, true);
if (bo_contents_fd < 0) {
ret = bo_contents_fd;
break;
bo_contents_fd = open_img_file(img_path, false,
&image_size, true);
if (bo_contents_fd < 0) {
ret = bo_contents_fd;
break;
}
if (image_size != entry->size) {
pr_err("Image size mismatch!\n");
ret = -ENXIO;
break;
}
ret = img_read(bo_contents_fd, buffer, image_size);
close(bo_contents_fd);
if (ret)
break;
ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_GEM_USERPTR,
&userptr_args);
if (ret) {
ret = -errno;
pr_perror("Failed to create userptr object");
break;
} else {
struct drm_gem_close close_args = {
.handle = userptr_args.handle,
};
ret = amdgpu_copy_buffer(drm_fd,
userptr_args.handle,
entry->handle);
has_copy_buffer = ret == 0;
ret = drmIoctl(drm_fd, DRM_IOCTL_GEM_CLOSE,
&close_args);
if (ret) {
ret = -errno;
pr_perror("Failed to close userptr object");
}
}
}
ret = sdma_copy_bo(dmabufs[i], rd->bo_entries[i]->size,
bo_contents_fd, buffer, buffer_size, h_dev,
max_copy_size, SDMA_OP_VRAM_WRITE, true);
close(bo_contents_fd);
if (ret) {
pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n", i);
break;
if (!has_copy_buffer) {
bo_contents_fd = open_img_file(img_path, false,
&image_size, true);
if (bo_contents_fd < 0) {
ret = bo_contents_fd;
break;
}
ret = sdma_copy_bo(dmabufs[i], entry->size,
bo_contents_fd, buffer, buffer_size,
h_dev, max_copy_size,
SDMA_OP_VRAM_WRITE, true);
close(bo_contents_fd);
if (ret) {
pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n",
i);
break;
}
}
}
@ -238,15 +383,44 @@ exit:
return ret;
}
static int
prepare_buffer(unsigned long size, void **buf, unsigned long *bufsize)
{
int ret;
if (*bufsize >= size)
return 0;
if (*bufsize) {
free(*buf);
*buf = NULL;
}
ret = posix_memalign(buf, sysconf(_SC_PAGE_SIZE), size);
if (ret) {
errno = ret;
pr_perror("Failed to allocate userptr buffer");
} else {
*bufsize = size;
}
return -ret;
}
int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
{
struct drm_amdgpu_gem_list_handles_entry *entries = NULL;
struct drm_amdgpu_gem_list_handles handles = {
.num_entries = 8,
};
struct drm_amdgpu_gem_list_contexts_entry *contexts = NULL;
struct drm_amdgpu_gem_list_contexts contexts_query = {
.num_contexts = 8,
};
bool libdrm_initialized = false;
unsigned long buffer_size = 0;
amdgpu_device_handle h_dev;
int has_copy_buffer = -1;
char path[PATH_MAX];
CriuRenderNode *rd;
unsigned char *buf;
@ -294,37 +468,98 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
break;
} while (true);
do {
unsigned int num_contexts = contexts_query.num_contexts;
if (contexts) {
xfree(contexts);
contexts = NULL;
}
contexts = xzalloc(sizeof(*contexts) * contexts_query.num_contexts);
if (!contexts) {
ret = -ENOMEM;
goto exit;
}
contexts_query.contexts = (uintptr_t)contexts;
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS, &contexts_query);
if (ret) {
ret = -errno;
if (errno == EINVAL) {
pr_info("This kernel appears not to have AMDGPU_GEM_LIST_CONTEXTS ioctl. Consider updating your kernel.\n");
contexts_query.num_contexts = 0;
break;
} else {
pr_perror("Failed to call bo info ioctl");
goto exit;
}
}
if (contexts_query.num_contexts <= num_contexts)
break;
} while (true);
rd->num_of_bos = handles.num_entries;
ret = allocate_bo_entries(rd, handles.num_entries);
if (ret)
goto exit;
if (contexts_query.num_contexts) {
rd->num_of_contexts = contexts_query.num_contexts;
ret = allocate_context_entries(rd, contexts_query.num_contexts);
if (ret)
goto exit;
rd->has_num_of_contexts = true;
}
for (unsigned int i = 0; i < contexts_query.num_contexts; i++) {
struct drm_amdgpu_gem_list_contexts_entry *entry = &contexts[i];
DrmContextEntry *context = rd->contexts[i];
context->handle = entry->handle;
context->flags = entry->flags;
context->init_priority = entry->init_priority;
context->override_priority = entry->override_priority;
context->pstate_flags = entry->pstate_flags;
pr_info("Saving context %u/%u: %u %x %d/%d %x\n",
i, contexts_query.num_contexts,
context->handle, context->flags, context->init_priority,
context->override_priority, context->pstate_flags);
}
for (int i = 0; i < handles.num_entries; i++) {
int num_vm_entries = 8;
struct drm_amdgpu_gem_vm_entry *vm_info_entries = NULL;
DrmBoEntry *boinfo = rd->bo_entries[i];
struct drm_amdgpu_gem_list_handles_entry *entry = &entries[i];
union drm_amdgpu_gem_mmap mmap_args = { 0 };
int dmabuf_fd = KFD_INVALID_FD;
int bo_contents_fd;
int dmabuf_fd;
boinfo->size = entry->size;
boinfo->alloc_flags = entry->alloc_flags;
boinfo->preferred_domains = entry->preferred_domains;
boinfo->alignment = entry->alignment;
boinfo->alignment = entry->alignment; /* Also sets userptr address. */
boinfo->handle = entry->gem_handle;
boinfo->is_import = (entry->flags & AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT) || shared_bo_has_exporter(boinfo->handle);
if (entry->flags & AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR)
boinfo->is_userptr = boinfo->has_is_userptr = true;
mmap_args.in.handle = boinfo->handle;
if (!boinfo->is_userptr) {
union drm_amdgpu_gem_mmap mmap_args = { 0 };
if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
pr_perror("Error Failed to call mmap ioctl");
ret = -1;
goto exit;
mmap_args.in.handle = boinfo->handle;
if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
pr_perror("Error Failed to call mmap ioctl");
ret = -1;
goto exit;
}
boinfo->offset = mmap_args.out.addr_ptr;
}
boinfo->offset = mmap_args.out.addr_ptr;
while (1) {
struct drm_amdgpu_gem_op vm_info_args = {
.handle = entry->gem_handle,
@ -376,11 +611,72 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
}
xfree(vm_info_entries);
if (!libdrm_initialized) {
uint32_t major, minor;
/* Try AMDGPU_GEM_COPY_BUFFER first */
if ((has_copy_buffer < 0 || has_copy_buffer == 1) &&
!boinfo->is_userptr) {
struct drm_amdgpu_gem_userptr userptr_args = {
.size = entry->size,
};
ret = amdgpu_device_initialize(fd, &major, &minor,
ret = prepare_buffer(entry->size, &buffer,
&buffer_size);
if (ret)
goto exit;
userptr_args.addr = (uintptr_t)buffer;
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR,
&userptr_args);
if (ret) {
ret = -errno;
pr_perror("Failed to create userptr object");
goto exit;
} else {
struct drm_gem_close close_args = {
.handle = userptr_args.handle,
};
ret = amdgpu_copy_buffer(fd,
entry->gem_handle,
userptr_args.handle);
has_copy_buffer = ret == 0;
ret = drmIoctl(fd, DRM_IOCTL_GEM_CLOSE,
&close_args);
if (ret) {
ret = -errno;
pr_perror("Failed to close userptr object");
}
if (has_copy_buffer == 1) {
snprintf(path, sizeof(path),
IMG_DRM_PAGES_FILE, rd->id,
rd->drm_render_minor, i);
ret = write_img_file(path, buffer,
entry->size);
if (ret)
goto exit;
}
}
}
if (!has_copy_buffer && !boinfo->is_userptr &&
!libdrm_initialized) {
uint32_t major, minor;
char *drm_path;
int drm_fd;
/* Re-open device to avoid VA conflicts in sdma_copy_bo. */
drm_path = drmGetRenderDeviceNameFromFd(fd);
drm_fd = open(drm_path, O_RDWR | O_CLOEXEC);
if (drm_fd < 0) {
ret = -errno;
pr_perror("Failed to re-open %s", drm_path);
goto exit;
}
ret = amdgpu_device_initialize(drm_fd, &major, &minor,
&h_dev);
close(drm_fd);
if (ret) {
pr_err("Failed to initialize amdgpu device - %s\n",
strerror(-ret));
@ -390,48 +686,38 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
libdrm_initialized = true;
}
ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd);
if (ret) {
pr_perror("Failed to get dmabuf fd from handle");
goto exit;
}
snprintf(path, sizeof(path), IMG_DRM_PAGES_FILE, rd->id, rd->drm_render_minor, i);
image_size = entry->size;
bo_contents_fd = open_img_file(path, true, &image_size, true);
if (bo_contents_fd < 0) {
ret = bo_contents_fd;
close(dmabuf_fd);
goto exit;
}
if (buffer_size < entry->size) {
if (buffer_size) {
free(buffer);
buffer = NULL;
if (!has_copy_buffer && !boinfo->is_userptr) {
ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd);
if (ret) {
pr_perror("Failed to get dmabuf fd from handle");
goto exit;
}
ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE),
entry->size);
snprintf(path, sizeof(path), IMG_DRM_PAGES_FILE, rd->id, rd->drm_render_minor, i);
image_size = entry->size;
bo_contents_fd = open_img_file(path, true, &image_size, true);
if (bo_contents_fd < 0) {
ret = bo_contents_fd;
close(dmabuf_fd);
goto exit;
}
ret = prepare_buffer(entry->size, &buffer, &buffer_size);
if (ret) {
errno = ret;
pr_perror("Failed to allocate userptr buffer");
ret = -ret;
close(bo_contents_fd);
close(dmabuf_fd);
goto exit;
}
buffer_size = entry->size;
ret = sdma_copy_bo(dmabuf_fd, entry->size,
bo_contents_fd, buffer, entry->size,
h_dev, 0x1000, SDMA_OP_VRAM_READ,
false);
close(bo_contents_fd);
if (ret)
goto exit;
}
ret = sdma_copy_bo(dmabuf_fd, entry->size, bo_contents_fd,
buffer, entry->size, h_dev, 0x1000,
SDMA_OP_VRAM_READ, false);
close(bo_contents_fd);
if (ret)
goto exit;
if (dmabuf_fd != KFD_INVALID_FD)
close(dmabuf_fd);
}
@ -452,7 +738,10 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
}
/* Get the GPU_ID of the DRM device */
rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id);
if (checkpoint_maps.mapped_cnt)
rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id);
else
rd->gpu_id = tp_node->gpu_id; /* Render node only */
if (!rd->gpu_id) {
pr_err("Failed to find valid gpu_id for the device = %d\n", tp_node->gpu_id);
return -ENODEV;
@ -472,6 +761,7 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
exit:
free(buffer);
xfree(entries);
xfree(contexts);
free_e(rd);
if (libdrm_initialized)
amdgpu_device_deinitialize(h_dev);
@ -488,13 +778,72 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
return -ENOMEM;
memset(dmabufs, 0xff, sizeof(int) * rd->num_of_bos);
for (unsigned int i = 0; i < rd->num_of_contexts; i++) {
DrmContextEntry *context = rd->contexts[i];
union drm_amdgpu_ctx args = {};
args.in.op = AMDGPU_CTX_OP_ALLOC_CTX;
args.in.priority = context->init_priority;
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &args);
if (ret < 0) {
ret = -errno;
pr_perror("Failed to create context");
goto exit;
}
if (args.out.alloc.ctx_id != context->handle) {
uint32_t id = args.out.alloc.ctx_id;
args.in.op = AMDGPU_CTX_OP_CHANGE_HANDLE;
args.in.ctx_id = id;
args.in.flags = context->handle;
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &args);
if (ret < 0) {
ret = -errno;
pr_perror("Failed to rename context");
goto exit;
}
}
if (context->override_priority != AMDGPU_CTX_PRIORITY_UNSET) {
union drm_amdgpu_sched sched = {};
sched.in.op = AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE;
sched.in.ctx_id = context->handle;
sched.in.priority = context->override_priority;
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_SCHED, &sched);
if (ret < 0) {
ret = -errno;
pr_perror("Failed to override context priority");
goto exit;
}
}
if (context->pstate_flags != AMDGPU_CTX_STABLE_PSTATE_NONE) {
union drm_amdgpu_ctx pstate = {};
pstate.in.op = AMDGPU_CTX_OP_SET_STABLE_PSTATE;
pstate.in.ctx_id = context->handle;
pstate.in.flags = context->pstate_flags;
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &pstate);
if (ret < 0) {
ret = -errno;
pr_perror("Failed to set context pstate");
goto exit;
}
}
pr_info("Restored context %u/%" PRIu64 ": %u %x %d/%d %x\n",
i, rd->num_of_contexts,
context->handle, context->flags, context->init_priority,
context->override_priority, context->pstate_flags);
}
for (int i = 0; i < rd->num_of_bos; i++) {
DrmBoEntry *boinfo = rd->bo_entries[i];
int dmabuf_fd = -1;
uint32_t handle;
struct drm_gem_change_handle change_args = { 0 };
union drm_amdgpu_gem_mmap mmap_args = { 0 };
struct drm_amdgpu_gem_va va_args = { 0 };
int fd_id;
if (work_already_completed(boinfo->handle, rd->drm_render_minor)) {
@ -521,8 +870,23 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
close(dmabuf_fd);
goto exit;
}
} else if (boinfo->is_userptr) {
struct drm_amdgpu_gem_userptr userptr_args = {
.size = boinfo->size,
.addr = boinfo->alignment,
.flags = boinfo->alloc_flags & ~AMDGPU_GEM_USERPTR_VALIDATE,
};
ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR,
&userptr_args);
if (ret < 0) {
ret = -errno;
pr_perror("Failed to create userptr object");
goto exit;
}
handle = userptr_args.handle;
} else {
union drm_amdgpu_gem_create create_args = { 0 };
union drm_amdgpu_gem_create create_args = {};
create_args.in.bo_size = boinfo->size;
create_args.in.alignment = boinfo->alignment;
@ -536,10 +900,13 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
}
handle = create_args.out.handle;
ret = drmPrimeHandleToFD(fd, handle, 0, &dmabuf_fd);
if (ret) {
pr_perror("Failed to get dmabuf fd from handle");
goto exit;
if (!(boinfo->alloc_flags &
AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)) {
ret = drmPrimeHandleToFD(fd, handle, 0, &dmabuf_fd);
if (ret) {
pr_perror("Failed to get dmabuf fd from handle");
goto exit;
}
}
}
@ -552,7 +919,7 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
goto exit;
}
if (!boinfo->is_import)
if (!boinfo->is_import && dmabuf_fd != -1)
store_dmabuf_fd(boinfo->handle, dmabuf_fd);
dmabufs[i] = dmabuf_fd;
@ -561,35 +928,22 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
if (ret)
goto exit;
mmap_args.in.handle = boinfo->handle;
if (!boinfo->is_userptr) {
union drm_amdgpu_gem_mmap mmap_args = {};
if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
pr_perror("Error Failed to call mmap ioctl");
ret = -1;
goto exit;
}
mmap_args.in.handle = boinfo->handle;
for (int j = 0; j < boinfo->num_of_vms; j++) {
DrmVmEntry *vminfo = boinfo->vm_entries[j];
va_args.handle = boinfo->handle;
va_args.operation = AMDGPU_VA_OP_MAP;
va_args.flags = vminfo->flags;
va_args.va_address = vminfo->addr;
va_args.offset_in_bo = vminfo->offset;
va_args.map_size = vminfo->size;
if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_VA, &va_args) == -1) {
pr_perror("Error Failed to call gem va ioctl");
if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
pr_perror("Error Failed to call mmap ioctl");
ret = -1;
goto exit;
}
}
ret = save_vma_updates(boinfo->offset, boinfo->addr,
mmap_args.out.addr_ptr, fd);
if (ret < 0)
goto exit;
ret = save_vma_updates(boinfo->offset, boinfo->addr,
mmap_args.out.addr_ptr, fd);
if (ret < 0)
goto exit;
}
}
if (ret) {
@ -611,6 +965,28 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
goto exit;
}
for (int i = 0; i < rd->num_of_bos; i++) {
DrmBoEntry *boinfo = rd->bo_entries[i];
for (int j = 0; j < boinfo->num_of_vms; j++) {
DrmVmEntry *vminfo = boinfo->vm_entries[j];
struct drm_amdgpu_gem_va va_args = { };
va_args.handle = boinfo->handle;
va_args.operation = AMDGPU_VA_OP_MAP;
va_args.flags = vminfo->flags;
va_args.va_address = vminfo->addr;
va_args.offset_in_bo = vminfo->offset;
va_args.map_size = vminfo->size;
if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_VA, &va_args) == -1) {
ret = -errno;
pr_perror("Failed to map bo %u/%u!", i, j);
goto exit;
}
}
}
exit:
if (ret < 0 && !bo_restore_called) {
for (int i = 0; i < rd->num_of_bos; i++) {

2
plugins/amdgpu/amdgpu_plugin_topology.c
View file

@ -306,6 +306,7 @@ void maps_init(struct device_maps *maps)
INIT_LIST_HEAD(&maps->gpu_maps);
maps->tail_cpu = 0;
maps->tail_gpu = 0;
maps->mapped_cnt = 0;
}
void maps_free(struct device_maps *maps)
@ -1219,6 +1220,7 @@ static bool map_devices(struct tp_system *src_sys, struct tp_system *dest_sys, s
if (map_devices(src_sys, dest_sys, src_nodes, dest_nodes, maps)) {
pr_debug("Matched nodes 0x%04X and after\n", dest_node->gpu_id);
maps->mapped_cnt++;
return true;
} else {
/* We could not map remaining nodes in the list. Add dest node back

2
plugins/amdgpu/amdgpu_plugin_topology.h
View file

@ -92,6 +92,8 @@ struct id_map {
};
struct device_maps {
unsigned int mapped_cnt;
struct list_head cpu_maps; /* CPUs are mapped using node_id */
struct list_head gpu_maps;

11
plugins/amdgpu/criu-amdgpu.proto
View file

@ -73,6 +73,15 @@ message drm_bo_entry {
required uint32 is_import = 8;
required uint32 num_of_vms = 9;
repeated drm_vm_entry vm_entries = 10;
optional bool is_userptr = 11 [default = false];
}
message drm_context_entry {
required uint32 handle = 1;
required uint32 flags = 2;
required int32 init_priority = 3;
required int32 override_priority = 4;
required uint32 pstate_flags = 5;
}
message drm_vm_entry {
@ -88,6 +97,8 @@ message criu_render_node {
required uint32 drm_render_minor = 3;
required uint64 num_of_bos = 4;
repeated drm_bo_entry bo_entries = 5;
optional uint64 num_of_contexts = 6 [default = 0];
repeated drm_context_entry contexts = 7;
}
message criu_dmabuf_node {