diff --git a/plugins/amdgpu/amdgpu_drm.h b/plugins/amdgpu/amdgpu_drm.h index 69227a12b..a77096831 100644 --- a/plugins/amdgpu/amdgpu_drm.h +++ b/plugins/amdgpu/amdgpu_drm.h @@ -58,6 +58,8 @@ extern "C" { #define DRM_AMDGPU_USERQ_SIGNAL 0x17 #define DRM_AMDGPU_USERQ_WAIT 0x18 #define DRM_AMDGPU_GEM_LIST_HANDLES 0x19 +#define DRM_AMDGPU_GEM_LIST_CONTEXTS 0x20 +#define DRM_AMDGPU_GEM_COPY_BUFFER 0x21 /* not upstream */ #define DRM_AMDGPU_GEM_DGMA 0x5c @@ -84,6 +86,8 @@ extern "C" { #define DRM_IOCTL_AMDGPU_USERQ_SIGNAL DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_SIGNAL, struct drm_amdgpu_userq_signal) #define DRM_IOCTL_AMDGPU_USERQ_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait) #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles) +#define DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_CONTEXTS, struct drm_amdgpu_gem_list_contexts) +#define DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_COPY_BUFFER, union drm_amdgpu_gem_copy_buffer) #define DRM_IOCTL_AMDGPU_GEM_DGMA DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_DGMA, struct drm_amdgpu_gem_dgma) @@ -231,6 +235,20 @@ union drm_amdgpu_gem_create { struct drm_amdgpu_gem_create_out out; }; +struct drm_amdgpu_gem_copy_buffer_in { + __u32 src_handle; + __u32 dst_handle; +}; + +struct drm_amdgpu_gem_copy_buffer_out { + __u32 fence_fd; +}; + +union drm_amdgpu_gem_copy_buffer { + struct drm_amdgpu_gem_copy_buffer_in in; + struct drm_amdgpu_gem_copy_buffer_out out; +}; + /** Opcode to create new residency list. */ #define AMDGPU_BO_LIST_OP_CREATE 0 /** Opcode to destroy previously created residency list */ @@ -276,6 +294,7 @@ union drm_amdgpu_bo_list { #define AMDGPU_CTX_OP_QUERY_STATE2 4 #define AMDGPU_CTX_OP_GET_STABLE_PSTATE 5 #define AMDGPU_CTX_OP_SET_STABLE_PSTATE 6 +#define AMDGPU_CTX_OP_CHANGE_HANDLE 7 /* GPU reset status */ #define AMDGPU_CTX_NO_RESET 0 @@ -898,6 +917,7 @@ struct drm_amdgpu_gem_op { }; #define AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT (1 << 0) +#define AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR (1 << 1) struct drm_amdgpu_gem_list_handles { /* User pointer to array of drm_amdgpu_gem_bo_info_entry */ @@ -913,7 +933,7 @@ struct drm_amdgpu_gem_list_handles_entry { /* gem handle of buffer object */ __u32 gem_handle; - /* Currently just one flag: IS_IMPORT */ + /* AMDGPU_GEM_LIST_HANDLES_FLAG_* */ __u32 flags; /* Size of bo */ @@ -922,11 +942,45 @@ struct drm_amdgpu_gem_list_handles_entry { /* Preferred domains for GEM_CREATE */ __u64 preferred_domains; - /* GEM_CREATE flags for re-creation of buffer */ + /* GEM_CREATE flags for re-creation of buffer or drm_amdgpu_gem_userptr.flags */ __u64 alloc_flags; - /* physical start_addr alignment in bytes for some HW requirements */ - __u64 alignment; + union { + /* physical start_addr alignment in bytes for some HW requirements */ + __u64 alignment; + + /* drm_amdgpu_gem_userptr.addr for userptr objects */ + __u64 userptr; + }; +}; + +struct drm_amdgpu_gem_list_contexts { + /* User pointer to array of drm_amdgpu_gem_list_contexts_entry */ + __u64 contexts; + + /* Size of the contexts buffer / Number of contexts in the client (if larger than size of buffer, must retry) */ + __u32 num_contexts; + + __u32 padding; +}; + +struct drm_amdgpu_gem_list_contexts_entry { + /* gem context handle */ + __u32 handle; + + /* AMDGPU_GEM_LIST_CONTEXTS_FLAG_* */ + __u32 flags; + + /* context initial priority */ + __s32 init_priority; + + /* context override priority */ + __s32 override_priority; + + /* pstate flags */ + __u32 pstate_flags; + + __u32 padding; }; #define AMDGPU_VA_OP_MAP 1 diff --git a/plugins/amdgpu/amdgpu_plugin.c b/plugins/amdgpu/amdgpu_plugin.c index 5054e1fe5..68798a112 100644 --- a/plugins/amdgpu/amdgpu_plugin.c +++ b/plugins/amdgpu/amdgpu_plugin.c @@ -62,6 +62,7 @@ struct vma_metadata { static LIST_HEAD(update_vma_info_list); static size_t kfd_max_buffer_size; +static bool amdgpu_ignore_single_gpuid; static bool plugin_added_to_inventory = false; @@ -404,6 +405,8 @@ int amdgpu_plugin_init(int stage) kfd_vram_size_check = getenv_bool("KFD_VRAM_SIZE_CHECK", true); kfd_numa_check = getenv_bool("KFD_NUMA_CHECK", true); kfd_capability_check = getenv_bool("KFD_CAPABILITY_CHECK", true); + + amdgpu_ignore_single_gpuid = getenv_bool("AMDGPU_IGNORE_SINGLE_GPUID", false); } kfd_max_buffer_size = getenv_size_t("KFD_MAX_BUFFER_SIZE", 0); @@ -1873,11 +1876,11 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed) size_t img_size; int fd, ret; - /* This is restorer plugin for renderD nodes. Criu doesn't guarantee that they will - * be called before the plugin is called for kfd file descriptor. - * TODO: Currently, this code will only work if this function is called for /dev/kfd - * first as we assume restore_maps is already filled. Need to fix this later. + /* This is restorer plugin for renderD nodes. Criu doesn't guarantee + * that they will be called before the plugin is called for kfd file + * descriptor. */ + snprintf(img_path, sizeof(img_path), IMG_DRM_FILE, id); ret = load_img(img_path, &buf, &img_size); if (ret < 0) { @@ -1906,14 +1909,71 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed) pr_info("render node gpu_id = 0x%04x\n", rd->gpu_id); - target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id); - if (!target_gpu_id) { - fd = -ENODEV; - goto fail; + if (fd_next == -1) { + ret = find_unused_fd_pid(getpid()); + if (ret < 0) { + pr_err("Failed to find unused fd (fd:%d)\n", ret); + fd = ret; + goto fail; + } + fd_next = ret; + } + + if (!dest_topology.parsed) { + pr_info("Parsing local topology for render node restore\n"); + ret = topology_parse(&dest_topology, "Local"); + if (ret) { + pr_err("Failed to parse local system topology %d\n", + ret); + fd = ret; + goto fail; + } + } + + if (restore_maps.mapped_cnt) { + target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id); + if (!target_gpu_id) { + pr_err("Unable to map gpu_id 0x%04x!\n", rd->gpu_id); + fd = -ENODEV; + goto fail; + } + } else { + unsigned int num_gpus = 0; + + pr_info("Assuming same system with a single gpu_id 0x%04x\n", + rd->gpu_id); + list_for_each_entry(tp_node, &dest_topology.nodes, + listm_system) { + if (NODE_IS_GPU(tp_node)) { + num_gpus++; + target_gpu_id = tp_node->gpu_id; + } + } + if (num_gpus != 1) { + pr_err("Unexpectedly found %u GPUs!\n", num_gpus); + fd = -EINVAL; + goto fail; + } else if (target_gpu_id != rd->gpu_id && + !amdgpu_ignore_single_gpuid) { + pr_err("Unexpectedly found gpu_id 0x%04x (expected 0x%04x)!\n", + target_gpu_id, rd->gpu_id); + fd = -EINVAL; + goto fail; + } } tp_node = sys_get_node_by_gpu_id(&dest_topology, target_gpu_id); - if (!tp_node) { + if (!tp_node && amdgpu_ignore_single_gpuid) { + tp_node = sys_get_node_by_index(&dest_topology, 0); + if (!NODE_IS_GPU(tp_node)) { + pr_err("Cannot find the GPU node!\n"); + fd = -ENODEV; + goto fail; + } + target_gpu_id = tp_node->gpu_id; + pr_warn("Forcing restore on gpu_id=0x%04x\n", target_gpu_id); + } else if (!tp_node) { + pr_err("Unable to find target gpu_id=0x%04x!\n", target_gpu_id); fd = -ENODEV; goto fail; } @@ -2151,25 +2211,28 @@ int amdgpu_plugin_update_vmamap(const char *in_path, const uint64_t addr, const } list_for_each_entry(vma_md, &update_vma_info_list, list) { - if (addr == vma_md->vma_entry && old_offset == vma_md->old_pgoff) { - *new_offset = vma_md->new_pgoff; + if (old_offset != vma_md->old_pgoff) + continue; + if (is_kfd && addr != vma_md->vma_entry) + continue; - *updated_fd = -1; - if (is_renderD) { - int fd = dup(vma_md->fd); - if (fd == -1) { - pr_perror("unable to duplicate the render fd"); - return -1; - } - *updated_fd = fd; + *new_offset = vma_md->new_pgoff; + if (is_renderD) { + int fd = dup(vma_md->fd); + + if (fd == -1) { + pr_perror("unable to duplicate the render fd"); + return -1; } - - pr_debug("old_pgoff=0x%lx new_pgoff=0x%lx fd=%d\n", - vma_md->old_pgoff, vma_md->new_pgoff, - *updated_fd); - - return 1; + *updated_fd = fd; + } else { + *updated_fd = -1; } + + pr_debug("old_pgoff=0x%lx new_pgoff=0x%lx fd=%d\n", + vma_md->old_pgoff, vma_md->new_pgoff, *updated_fd); + + return 1; } pr_info("No match for addr:0x%lx offset:%lx\n", addr, old_offset); return 0; diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c index dee0a811a..3e88a6d00 100644 --- a/plugins/amdgpu/amdgpu_plugin_drm.c +++ b/plugins/amdgpu/amdgpu_plugin_drm.c @@ -95,6 +95,30 @@ static int allocate_bo_entries(CriuRenderNode *e, int num_bos) return 0; } +static int allocate_context_entries(CriuRenderNode *e, unsigned long num_contexts) +{ + e->contexts = xmalloc(sizeof(DrmContextEntry *) * num_contexts); + if (!e->contexts) { + pr_err("Failed to allocate context list\n"); + return -ENOMEM; + } + + for (int i = 0; i < num_contexts; i++) { + DrmContextEntry *context = xzalloc(sizeof(*context)); + + if (!context) { + pr_err("Failed to allocate context info\n"); + return -ENOMEM; + } + + drm_context_entry__init(context); + + e->contexts[i] = context; + e->n_contexts++; + } + return 0; +} + static int allocate_vm_entries(DrmBoEntry *e, int num_vms) { e->vm_entries = xmalloc(sizeof(DrmVmEntry *) * num_vms); @@ -125,6 +149,13 @@ static void free_e(CriuRenderNode *e) if (e->bo_entries[i]) xfree(e->bo_entries[i]); } + xfree(e->bo_entries); + + for (int i = 0; i < e->n_contexts; i++) { + if (e->contexts[i]) + xfree(e->contexts[i]); + } + xfree(e->contexts); xfree(e); } @@ -156,11 +187,62 @@ int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *st) return 0; } +static int sync_fence_wait(int fd, int timeout) +{ + struct pollfd fds = { fd, POLLIN }; + int ret; + + while (true) { + ret = poll(&fds, 1, timeout); + if (ret > 0) { + if (fds.revents & (POLLERR | POLLNVAL)) + return -EINVAL; + + return 0; + } else if (ret == 0) { + return -ETIME; + } else { + ret = -errno; + if (ret == -EINTR || ret == -EAGAIN) + continue; + return ret; + } + } +} + +static int +amdgpu_copy_buffer(int drm_fd, uint32_t src_handle, uint32_t dst_handle) +{ + union drm_amdgpu_gem_copy_buffer copy_args = { + .in.src_handle = src_handle, + .in.dst_handle = dst_handle, + }; + int ret; + + ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER, ©_args); + if (ret) { + ret = -errno; + pr_perror("Failed to copy buffer object"); + + return ret; + } + + ret = sync_fence_wait(copy_args.out.fence_fd, 10000); + close(copy_args.out.fence_fd); + if (ret) { + pr_err("Failed to wait for buffer object copy! (%s)\n", + strerror(-ret)); + } + + return ret; +} + static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int drm_fd, int *dmabufs) { - size_t image_size = 0, max_bo_size = 0, buffer_size; - struct amdgpu_gpu_info gpu_info = { 0 }; + size_t image_size = 0, buffer_size = 0; + struct amdgpu_gpu_info gpu_info = {}; amdgpu_device_handle h_dev; + int has_copy_buffer = -1; uint64_t max_copy_size; uint32_t major, minor; void *buffer = NULL; @@ -185,13 +267,18 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int SDMA_LINEAR_COPY_MAX_SIZE - 1; for (i = 0; i < rd->num_of_bos; i++) { - if (rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) { - if (rd->bo_entries[i]->size > max_bo_size) - max_bo_size = rd->bo_entries[i]->size; + DrmBoEntry *entry = rd->bo_entries[i]; + + if ((entry->preferred_domains & + (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && + !entry->is_userptr) { + if (entry->size > buffer_size) + buffer_size = entry->size; } } - buffer_size = max_bo_size; + if (!buffer_size) + goto exit; ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), buffer_size); if (ret) { @@ -202,27 +289,85 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int } for (i = 0; i < rd->num_of_bos; i++) { - if (!(rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT))) + DrmBoEntry *entry = rd->bo_entries[i]; + + if (entry->is_userptr || + !(entry->preferred_domains & + (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) || + entry->num_of_vms == 0) continue; - if (rd->bo_entries[i]->num_of_vms == 0) - continue; + snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->id, + drm_render_minor, i); - snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->id, drm_render_minor, i); + /* Try AMDGPU_GEM_COPY_BUFFER first */ + if ((has_copy_buffer < 0 || has_copy_buffer == 1)) { + struct drm_amdgpu_gem_userptr userptr_args = { + .addr = (uintptr_t)buffer, + .size = entry->size, + }; - bo_contents_fd = open_img_file(img_path, false, &image_size, true); - if (bo_contents_fd < 0) { - ret = bo_contents_fd; - break; + bo_contents_fd = open_img_file(img_path, false, + &image_size, true); + if (bo_contents_fd < 0) { + ret = bo_contents_fd; + break; + } + + if (image_size != entry->size) { + pr_err("Image size mismatch!\n"); + ret = -ENXIO; + break; + } + + ret = img_read(bo_contents_fd, buffer, image_size); + close(bo_contents_fd); + if (ret) + break; + + ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, + &userptr_args); + if (ret) { + ret = -errno; + pr_perror("Failed to create userptr object"); + break; + } else { + struct drm_gem_close close_args = { + .handle = userptr_args.handle, + }; + + ret = amdgpu_copy_buffer(drm_fd, + userptr_args.handle, + entry->handle); + has_copy_buffer = ret == 0; + + ret = drmIoctl(drm_fd, DRM_IOCTL_GEM_CLOSE, + &close_args); + if (ret) { + ret = -errno; + pr_perror("Failed to close userptr object"); + } + } } - ret = sdma_copy_bo(dmabufs[i], rd->bo_entries[i]->size, - bo_contents_fd, buffer, buffer_size, h_dev, - max_copy_size, SDMA_OP_VRAM_WRITE, true); - close(bo_contents_fd); - if (ret) { - pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n", i); - break; + if (!has_copy_buffer) { + bo_contents_fd = open_img_file(img_path, false, + &image_size, true); + if (bo_contents_fd < 0) { + ret = bo_contents_fd; + break; + } + + ret = sdma_copy_bo(dmabufs[i], entry->size, + bo_contents_fd, buffer, buffer_size, + h_dev, max_copy_size, + SDMA_OP_VRAM_WRITE, true); + close(bo_contents_fd); + if (ret) { + pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n", + i); + break; + } } } @@ -238,15 +383,44 @@ exit: return ret; } +static int +prepare_buffer(unsigned long size, void **buf, unsigned long *bufsize) +{ + int ret; + + if (*bufsize >= size) + return 0; + + if (*bufsize) { + free(*buf); + *buf = NULL; + } + + ret = posix_memalign(buf, sysconf(_SC_PAGE_SIZE), size); + if (ret) { + errno = ret; + pr_perror("Failed to allocate userptr buffer"); + } else { + *bufsize = size; + } + + return -ret; +} + int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm) { struct drm_amdgpu_gem_list_handles_entry *entries = NULL; struct drm_amdgpu_gem_list_handles handles = { .num_entries = 8, }; + struct drm_amdgpu_gem_list_contexts_entry *contexts = NULL; + struct drm_amdgpu_gem_list_contexts contexts_query = { + .num_contexts = 8, + }; bool libdrm_initialized = false; unsigned long buffer_size = 0; amdgpu_device_handle h_dev; + int has_copy_buffer = -1; char path[PATH_MAX]; CriuRenderNode *rd; unsigned char *buf; @@ -294,37 +468,98 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm) break; } while (true); + do { + unsigned int num_contexts = contexts_query.num_contexts; + + if (contexts) { + xfree(contexts); + contexts = NULL; + } + + contexts = xzalloc(sizeof(*contexts) * contexts_query.num_contexts); + if (!contexts) { + ret = -ENOMEM; + goto exit; + } + + contexts_query.contexts = (uintptr_t)contexts; + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS, &contexts_query); + if (ret) { + ret = -errno; + if (errno == EINVAL) { + pr_info("This kernel appears not to have AMDGPU_GEM_LIST_CONTEXTS ioctl. Consider updating your kernel.\n"); + contexts_query.num_contexts = 0; + break; + } else { + pr_perror("Failed to call bo info ioctl"); + goto exit; + } + } + + if (contexts_query.num_contexts <= num_contexts) + break; + } while (true); + rd->num_of_bos = handles.num_entries; ret = allocate_bo_entries(rd, handles.num_entries); if (ret) goto exit; + if (contexts_query.num_contexts) { + rd->num_of_contexts = contexts_query.num_contexts; + ret = allocate_context_entries(rd, contexts_query.num_contexts); + if (ret) + goto exit; + rd->has_num_of_contexts = true; + } + + for (unsigned int i = 0; i < contexts_query.num_contexts; i++) { + struct drm_amdgpu_gem_list_contexts_entry *entry = &contexts[i]; + DrmContextEntry *context = rd->contexts[i]; + + context->handle = entry->handle; + context->flags = entry->flags; + context->init_priority = entry->init_priority; + context->override_priority = entry->override_priority; + context->pstate_flags = entry->pstate_flags; + + pr_info("Saving context %u/%u: %u %x %d/%d %x\n", + i, contexts_query.num_contexts, + context->handle, context->flags, context->init_priority, + context->override_priority, context->pstate_flags); + } + for (int i = 0; i < handles.num_entries; i++) { int num_vm_entries = 8; struct drm_amdgpu_gem_vm_entry *vm_info_entries = NULL; DrmBoEntry *boinfo = rd->bo_entries[i]; struct drm_amdgpu_gem_list_handles_entry *entry = &entries[i]; - union drm_amdgpu_gem_mmap mmap_args = { 0 }; + int dmabuf_fd = KFD_INVALID_FD; int bo_contents_fd; - int dmabuf_fd; boinfo->size = entry->size; boinfo->alloc_flags = entry->alloc_flags; boinfo->preferred_domains = entry->preferred_domains; - boinfo->alignment = entry->alignment; + boinfo->alignment = entry->alignment; /* Also sets userptr address. */ boinfo->handle = entry->gem_handle; boinfo->is_import = (entry->flags & AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT) || shared_bo_has_exporter(boinfo->handle); + if (entry->flags & AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR) + boinfo->is_userptr = boinfo->has_is_userptr = true; - mmap_args.in.handle = boinfo->handle; + if (!boinfo->is_userptr) { + union drm_amdgpu_gem_mmap mmap_args = { 0 }; - if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) { - pr_perror("Error Failed to call mmap ioctl"); - ret = -1; - goto exit; + mmap_args.in.handle = boinfo->handle; + + if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) { + pr_perror("Error Failed to call mmap ioctl"); + ret = -1; + goto exit; + } + + boinfo->offset = mmap_args.out.addr_ptr; } - boinfo->offset = mmap_args.out.addr_ptr; - while (1) { struct drm_amdgpu_gem_op vm_info_args = { .handle = entry->gem_handle, @@ -376,11 +611,72 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm) } xfree(vm_info_entries); - if (!libdrm_initialized) { - uint32_t major, minor; + /* Try AMDGPU_GEM_COPY_BUFFER first */ + if ((has_copy_buffer < 0 || has_copy_buffer == 1) && + !boinfo->is_userptr) { + struct drm_amdgpu_gem_userptr userptr_args = { + .size = entry->size, + }; - ret = amdgpu_device_initialize(fd, &major, &minor, + ret = prepare_buffer(entry->size, &buffer, + &buffer_size); + if (ret) + goto exit; + + userptr_args.addr = (uintptr_t)buffer; + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, + &userptr_args); + if (ret) { + ret = -errno; + pr_perror("Failed to create userptr object"); + goto exit; + } else { + struct drm_gem_close close_args = { + .handle = userptr_args.handle, + }; + + ret = amdgpu_copy_buffer(fd, + entry->gem_handle, + userptr_args.handle); + has_copy_buffer = ret == 0; + + ret = drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, + &close_args); + if (ret) { + ret = -errno; + pr_perror("Failed to close userptr object"); + } + + if (has_copy_buffer == 1) { + snprintf(path, sizeof(path), + IMG_DRM_PAGES_FILE, rd->id, + rd->drm_render_minor, i); + ret = write_img_file(path, buffer, + entry->size); + if (ret) + goto exit; + } + } + } + + if (!has_copy_buffer && !boinfo->is_userptr && + !libdrm_initialized) { + uint32_t major, minor; + char *drm_path; + int drm_fd; + + /* Re-open device to avoid VA conflicts in sdma_copy_bo. */ + drm_path = drmGetRenderDeviceNameFromFd(fd); + drm_fd = open(drm_path, O_RDWR | O_CLOEXEC); + if (drm_fd < 0) { + ret = -errno; + pr_perror("Failed to re-open %s", drm_path); + goto exit; + } + + ret = amdgpu_device_initialize(drm_fd, &major, &minor, &h_dev); + close(drm_fd); if (ret) { pr_err("Failed to initialize amdgpu device - %s\n", strerror(-ret)); @@ -390,48 +686,38 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm) libdrm_initialized = true; } - ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd); - if (ret) { - pr_perror("Failed to get dmabuf fd from handle"); - goto exit; - } - - snprintf(path, sizeof(path), IMG_DRM_PAGES_FILE, rd->id, rd->drm_render_minor, i); - image_size = entry->size; - bo_contents_fd = open_img_file(path, true, &image_size, true); - if (bo_contents_fd < 0) { - ret = bo_contents_fd; - close(dmabuf_fd); - goto exit; - } - - if (buffer_size < entry->size) { - if (buffer_size) { - free(buffer); - buffer = NULL; + if (!has_copy_buffer && !boinfo->is_userptr) { + ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd); + if (ret) { + pr_perror("Failed to get dmabuf fd from handle"); + goto exit; } - ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), - entry->size); + snprintf(path, sizeof(path), IMG_DRM_PAGES_FILE, rd->id, rd->drm_render_minor, i); + image_size = entry->size; + bo_contents_fd = open_img_file(path, true, &image_size, true); + if (bo_contents_fd < 0) { + ret = bo_contents_fd; + close(dmabuf_fd); + goto exit; + } + + ret = prepare_buffer(entry->size, &buffer, &buffer_size); if (ret) { - errno = ret; - pr_perror("Failed to allocate userptr buffer"); - ret = -ret; close(bo_contents_fd); close(dmabuf_fd); goto exit; } - buffer_size = entry->size; + ret = sdma_copy_bo(dmabuf_fd, entry->size, + bo_contents_fd, buffer, entry->size, + h_dev, 0x1000, SDMA_OP_VRAM_READ, + false); + close(bo_contents_fd); + if (ret) + goto exit; } - ret = sdma_copy_bo(dmabuf_fd, entry->size, bo_contents_fd, - buffer, entry->size, h_dev, 0x1000, - SDMA_OP_VRAM_READ, false); - close(bo_contents_fd); - if (ret) - goto exit; - if (dmabuf_fd != KFD_INVALID_FD) close(dmabuf_fd); } @@ -452,7 +738,10 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm) } /* Get the GPU_ID of the DRM device */ - rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id); + if (checkpoint_maps.mapped_cnt) + rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id); + else + rd->gpu_id = tp_node->gpu_id; /* Render node only */ if (!rd->gpu_id) { pr_err("Failed to find valid gpu_id for the device = %d\n", tp_node->gpu_id); return -ENODEV; @@ -472,6 +761,7 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm) exit: free(buffer); xfree(entries); + xfree(contexts); free_e(rd); if (libdrm_initialized) amdgpu_device_deinitialize(h_dev); @@ -488,13 +778,72 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd) return -ENOMEM; memset(dmabufs, 0xff, sizeof(int) * rd->num_of_bos); + for (unsigned int i = 0; i < rd->num_of_contexts; i++) { + DrmContextEntry *context = rd->contexts[i]; + union drm_amdgpu_ctx args = {}; + + args.in.op = AMDGPU_CTX_OP_ALLOC_CTX; + args.in.priority = context->init_priority; + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &args); + if (ret < 0) { + ret = -errno; + pr_perror("Failed to create context"); + goto exit; + } + + if (args.out.alloc.ctx_id != context->handle) { + uint32_t id = args.out.alloc.ctx_id; + + args.in.op = AMDGPU_CTX_OP_CHANGE_HANDLE; + args.in.ctx_id = id; + args.in.flags = context->handle; + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &args); + if (ret < 0) { + ret = -errno; + pr_perror("Failed to rename context"); + goto exit; + } + } + + if (context->override_priority != AMDGPU_CTX_PRIORITY_UNSET) { + union drm_amdgpu_sched sched = {}; + + sched.in.op = AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE; + sched.in.ctx_id = context->handle; + sched.in.priority = context->override_priority; + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_SCHED, &sched); + if (ret < 0) { + ret = -errno; + pr_perror("Failed to override context priority"); + goto exit; + } + } + + if (context->pstate_flags != AMDGPU_CTX_STABLE_PSTATE_NONE) { + union drm_amdgpu_ctx pstate = {}; + + pstate.in.op = AMDGPU_CTX_OP_SET_STABLE_PSTATE; + pstate.in.ctx_id = context->handle; + pstate.in.flags = context->pstate_flags; + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &pstate); + if (ret < 0) { + ret = -errno; + pr_perror("Failed to set context pstate"); + goto exit; + } + } + + pr_info("Restored context %u/%" PRIu64 ": %u %x %d/%d %x\n", + i, rd->num_of_contexts, + context->handle, context->flags, context->init_priority, + context->override_priority, context->pstate_flags); + } + for (int i = 0; i < rd->num_of_bos; i++) { DrmBoEntry *boinfo = rd->bo_entries[i]; int dmabuf_fd = -1; uint32_t handle; struct drm_gem_change_handle change_args = { 0 }; - union drm_amdgpu_gem_mmap mmap_args = { 0 }; - struct drm_amdgpu_gem_va va_args = { 0 }; int fd_id; if (work_already_completed(boinfo->handle, rd->drm_render_minor)) { @@ -521,8 +870,23 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd) close(dmabuf_fd); goto exit; } + } else if (boinfo->is_userptr) { + struct drm_amdgpu_gem_userptr userptr_args = { + .size = boinfo->size, + .addr = boinfo->alignment, + .flags = boinfo->alloc_flags & ~AMDGPU_GEM_USERPTR_VALIDATE, + }; + + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, + &userptr_args); + if (ret < 0) { + ret = -errno; + pr_perror("Failed to create userptr object"); + goto exit; + } + handle = userptr_args.handle; } else { - union drm_amdgpu_gem_create create_args = { 0 }; + union drm_amdgpu_gem_create create_args = {}; create_args.in.bo_size = boinfo->size; create_args.in.alignment = boinfo->alignment; @@ -536,10 +900,13 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd) } handle = create_args.out.handle; - ret = drmPrimeHandleToFD(fd, handle, 0, &dmabuf_fd); - if (ret) { - pr_perror("Failed to get dmabuf fd from handle"); - goto exit; + if (!(boinfo->alloc_flags & + AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)) { + ret = drmPrimeHandleToFD(fd, handle, 0, &dmabuf_fd); + if (ret) { + pr_perror("Failed to get dmabuf fd from handle"); + goto exit; + } } } @@ -552,7 +919,7 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd) goto exit; } - if (!boinfo->is_import) + if (!boinfo->is_import && dmabuf_fd != -1) store_dmabuf_fd(boinfo->handle, dmabuf_fd); dmabufs[i] = dmabuf_fd; @@ -561,35 +928,22 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd) if (ret) goto exit; - mmap_args.in.handle = boinfo->handle; + if (!boinfo->is_userptr) { + union drm_amdgpu_gem_mmap mmap_args = {}; - if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) { - pr_perror("Error Failed to call mmap ioctl"); - ret = -1; - goto exit; - } + mmap_args.in.handle = boinfo->handle; - for (int j = 0; j < boinfo->num_of_vms; j++) { - DrmVmEntry *vminfo = boinfo->vm_entries[j]; - - va_args.handle = boinfo->handle; - va_args.operation = AMDGPU_VA_OP_MAP; - va_args.flags = vminfo->flags; - va_args.va_address = vminfo->addr; - va_args.offset_in_bo = vminfo->offset; - va_args.map_size = vminfo->size; - - if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_VA, &va_args) == -1) { - pr_perror("Error Failed to call gem va ioctl"); + if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) { + pr_perror("Error Failed to call mmap ioctl"); ret = -1; goto exit; } - } - ret = save_vma_updates(boinfo->offset, boinfo->addr, - mmap_args.out.addr_ptr, fd); - if (ret < 0) - goto exit; + ret = save_vma_updates(boinfo->offset, boinfo->addr, + mmap_args.out.addr_ptr, fd); + if (ret < 0) + goto exit; + } } if (ret) { @@ -611,6 +965,28 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd) goto exit; } + for (int i = 0; i < rd->num_of_bos; i++) { + DrmBoEntry *boinfo = rd->bo_entries[i]; + + for (int j = 0; j < boinfo->num_of_vms; j++) { + DrmVmEntry *vminfo = boinfo->vm_entries[j]; + struct drm_amdgpu_gem_va va_args = { }; + + va_args.handle = boinfo->handle; + va_args.operation = AMDGPU_VA_OP_MAP; + va_args.flags = vminfo->flags; + va_args.va_address = vminfo->addr; + va_args.offset_in_bo = vminfo->offset; + va_args.map_size = vminfo->size; + + if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_VA, &va_args) == -1) { + ret = -errno; + pr_perror("Failed to map bo %u/%u!", i, j); + goto exit; + } + } + } + exit: if (ret < 0 && !bo_restore_called) { for (int i = 0; i < rd->num_of_bos; i++) { diff --git a/plugins/amdgpu/amdgpu_plugin_topology.c b/plugins/amdgpu/amdgpu_plugin_topology.c index cb861d50d..555e04345 100644 --- a/plugins/amdgpu/amdgpu_plugin_topology.c +++ b/plugins/amdgpu/amdgpu_plugin_topology.c @@ -306,6 +306,7 @@ void maps_init(struct device_maps *maps) INIT_LIST_HEAD(&maps->gpu_maps); maps->tail_cpu = 0; maps->tail_gpu = 0; + maps->mapped_cnt = 0; } void maps_free(struct device_maps *maps) @@ -1219,6 +1220,7 @@ static bool map_devices(struct tp_system *src_sys, struct tp_system *dest_sys, s if (map_devices(src_sys, dest_sys, src_nodes, dest_nodes, maps)) { pr_debug("Matched nodes 0x%04X and after\n", dest_node->gpu_id); + maps->mapped_cnt++; return true; } else { /* We could not map remaining nodes in the list. Add dest node back diff --git a/plugins/amdgpu/amdgpu_plugin_topology.h b/plugins/amdgpu/amdgpu_plugin_topology.h index e19f8e7ce..ab850d6bd 100644 --- a/plugins/amdgpu/amdgpu_plugin_topology.h +++ b/plugins/amdgpu/amdgpu_plugin_topology.h @@ -92,6 +92,8 @@ struct id_map { }; struct device_maps { + unsigned int mapped_cnt; + struct list_head cpu_maps; /* CPUs are mapped using node_id */ struct list_head gpu_maps; diff --git a/plugins/amdgpu/criu-amdgpu.proto b/plugins/amdgpu/criu-amdgpu.proto index 7682a8f21..f322a07e3 100644 --- a/plugins/amdgpu/criu-amdgpu.proto +++ b/plugins/amdgpu/criu-amdgpu.proto @@ -73,6 +73,15 @@ message drm_bo_entry { required uint32 is_import = 8; required uint32 num_of_vms = 9; repeated drm_vm_entry vm_entries = 10; + optional bool is_userptr = 11 [default = false]; +} + +message drm_context_entry { + required uint32 handle = 1; + required uint32 flags = 2; + required int32 init_priority = 3; + required int32 override_priority = 4; + required uint32 pstate_flags = 5; } message drm_vm_entry { @@ -88,6 +97,8 @@ message criu_render_node { required uint32 drm_render_minor = 3; required uint64 num_of_bos = 4; repeated drm_bo_entry bo_entries = 5; + optional uint64 num_of_contexts = 6 [default = 0]; + repeated drm_context_entry contexts = 7; } message criu_dmabuf_node {