From 44739b7b1c8c8a6167cfc182af62977622f3fc71 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Thu, 12 Feb 2026 15:42:26 +0000
Subject: [PATCH 01/10] plugins/amdgpu: Enable restore with only the drm node
 open

It is currently not possible to restore a process which did not have a KFD
device open.

We can work around that by assuming the restore is being done on the same
system with the same GPU id. This can fail over reboots when the GPU id
changes, but that also could be worked around with perhaps something like
a "--ignore-gpu-id" command line switch.

In the long term we probably need to think of adding the relevant GPU
discovery APIs to the amdgpu kernel driver, which would enable this to
work more smoothly.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_plugin.c          | 63 +++++++++++++++++++++----
 plugins/amdgpu/amdgpu_plugin_drm.c      |  5 +-
 plugins/amdgpu/amdgpu_plugin_topology.c |  2 +
 plugins/amdgpu/amdgpu_plugin_topology.h |  2 +
 4 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_plugin.c b/plugins/amdgpu/amdgpu_plugin.c
index 5054e1fe5..e108b4b77 100644
--- a/plugins/amdgpu/amdgpu_plugin.c
+++ b/plugins/amdgpu/amdgpu_plugin.c
@@ -1873,11 +1873,11 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed)
 	size_t img_size;
 	int fd, ret;
 
-	/* This is restorer plugin for renderD nodes. Criu doesn't guarantee that they will
-	 * be called before the plugin is called for kfd file descriptor.
-	 * TODO: Currently, this code will only work if this function is called for /dev/kfd
-	 * first as we assume restore_maps is already filled. Need to fix this later.
+	/* This is restorer plugin for renderD nodes. Criu doesn't guarantee
+	 * that they will be called before the plugin is called for kfd file
+	 * descriptor.
 	 */
+
 	snprintf(img_path, sizeof(img_path), IMG_DRM_FILE, id);
 	ret = load_img(img_path, &buf, &img_size);
 	if (ret < 0) {
@@ -1906,14 +1906,61 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed)
 
 	pr_info("render node gpu_id = 0x%04x\n", rd->gpu_id);
 
-	target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id);
-	if (!target_gpu_id) {
-		fd = -ENODEV;
-		goto fail;
+	if (fd_next == -1) {
+		ret = find_unused_fd_pid(getpid());
+		if (ret < 0) {
+			pr_err("Failed to find unused fd (fd:%d)\n", ret);
+			fd = ret;
+			goto fail;
+		}
+		fd_next = ret;
+	}
+
+	if (!dest_topology.parsed) {
+		pr_info("Parsing local topology for render node restore\n");
+		ret = topology_parse(&dest_topology, "Local");
+		if (ret) {
+			pr_err("Failed to parse local system topology %d\n",
+			       ret);
+			fd = ret;
+			goto fail;
+		}
+	}
+
+	if (restore_maps.mapped_cnt) {
+		target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id);
+		if (!target_gpu_id) {
+			pr_err("Unable to map gpu_id 0x%04x!\n", rd->gpu_id);
+			fd = -ENODEV;
+			goto fail;
+		}
+	} else {
+		unsigned int num_gpus = 0;
+
+		pr_info("Assuming same system with a single gpu_id 0x%04x\n",
+			rd->gpu_id);
+		list_for_each_entry(tp_node, &dest_topology.nodes,
+				    listm_system) {
+			if (NODE_IS_GPU(tp_node)) {
+				num_gpus++;
+				target_gpu_id = tp_node->gpu_id;
+			}
+		}
+		if (num_gpus != 1) {
+			pr_err("Unexpectedly found %u GPUs!\n", num_gpus);
+			fd = -EINVAL;
+			goto fail;
+		} else if (target_gpu_id != rd->gpu_id) {
+			pr_err("Unexpectedly found gpu_id 0x%04x (expected 0x%04x)!\n",
+			       target_gpu_id, rd->gpu_id);
+			fd = -EINVAL;
+			goto fail;
+		}
 	}
 
 	tp_node = sys_get_node_by_gpu_id(&dest_topology, target_gpu_id);
 	if (!tp_node) {
+		pr_err("Unable to find target gpu_id=0x%04x!\n", target_gpu_id);
 		fd = -ENODEV;
 		goto fail;
 	}
diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index dee0a811a..85a0ce5f0 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -452,7 +452,10 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 	}
 
 	/* Get the GPU_ID of the DRM device */
-	rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id);
+	if (checkpoint_maps.mapped_cnt)
+		rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id);
+	else
+		rd->gpu_id = tp_node->gpu_id; /* Render node only */
 	if (!rd->gpu_id) {
 		pr_err("Failed to find valid gpu_id for the device = %d\n", tp_node->gpu_id);
 		return -ENODEV;
diff --git a/plugins/amdgpu/amdgpu_plugin_topology.c b/plugins/amdgpu/amdgpu_plugin_topology.c
index cb861d50d..555e04345 100644
--- a/plugins/amdgpu/amdgpu_plugin_topology.c
+++ b/plugins/amdgpu/amdgpu_plugin_topology.c
@@ -306,6 +306,7 @@ void maps_init(struct device_maps *maps)
 	INIT_LIST_HEAD(&maps->gpu_maps);
 	maps->tail_cpu = 0;
 	maps->tail_gpu = 0;
+	maps->mapped_cnt = 0;
 }
 
 void maps_free(struct device_maps *maps)
@@ -1219,6 +1220,7 @@ static bool map_devices(struct tp_system *src_sys, struct tp_system *dest_sys, s
 
 			if (map_devices(src_sys, dest_sys, src_nodes, dest_nodes, maps)) {
 				pr_debug("Matched nodes 0x%04X and after\n", dest_node->gpu_id);
+				maps->mapped_cnt++;
 				return true;
 			} else {
 				/* We could not map remaining nodes in the list. Add dest node back
diff --git a/plugins/amdgpu/amdgpu_plugin_topology.h b/plugins/amdgpu/amdgpu_plugin_topology.h
index e19f8e7ce..ab850d6bd 100644
--- a/plugins/amdgpu/amdgpu_plugin_topology.h
+++ b/plugins/amdgpu/amdgpu_plugin_topology.h
@@ -92,6 +92,8 @@ struct id_map {
 };
 
 struct device_maps {
+	unsigned int mapped_cnt;
+
 	struct list_head cpu_maps; /* CPUs are mapped using node_id */
 	struct list_head gpu_maps;
 

From 792e68ed40bd6c475780aa5a1f43ea858b7d0c28 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Fri, 1 May 2026 09:36:13 +0100
Subject: [PATCH 02/10] plugins/amdgpu: Allow forcing restore on a single GPU
 system

Add a new boolean environment variable AMDGPU_IGNORE_SINGLE_GPUID which
can be used to force restoring of a single GPU image on a single GPU
machine. This is useful for images which had no kfd open and if the gpu id
had changed for example after re-booting to a different kernel version.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_plugin.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_plugin.c b/plugins/amdgpu/amdgpu_plugin.c
index e108b4b77..56e02d510 100644
--- a/plugins/amdgpu/amdgpu_plugin.c
+++ b/plugins/amdgpu/amdgpu_plugin.c
@@ -62,6 +62,7 @@ struct vma_metadata {
 static LIST_HEAD(update_vma_info_list);
 
 static size_t kfd_max_buffer_size;
+static bool amdgpu_ignore_single_gpuid;
 
 static bool plugin_added_to_inventory = false;
 
@@ -404,6 +405,8 @@ int amdgpu_plugin_init(int stage)
 		kfd_vram_size_check = getenv_bool("KFD_VRAM_SIZE_CHECK", true);
 		kfd_numa_check = getenv_bool("KFD_NUMA_CHECK", true);
 		kfd_capability_check = getenv_bool("KFD_CAPABILITY_CHECK", true);
+
+		amdgpu_ignore_single_gpuid = getenv_bool("AMDGPU_IGNORE_SINGLE_GPUID", false);
 	}
 
 	kfd_max_buffer_size = getenv_size_t("KFD_MAX_BUFFER_SIZE", 0);
@@ -1950,7 +1953,8 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed)
 			pr_err("Unexpectedly found %u GPUs!\n", num_gpus);
 			fd = -EINVAL;
 			goto fail;
-		} else if (target_gpu_id != rd->gpu_id) {
+		} else if (target_gpu_id != rd->gpu_id &&
+			   !amdgpu_ignore_single_gpuid) {
 			pr_err("Unexpectedly found gpu_id 0x%04x (expected 0x%04x)!\n",
 			       target_gpu_id, rd->gpu_id);
 			fd = -EINVAL;
@@ -1959,7 +1963,16 @@ static int amdgpu_plugin_restore_drm_file(int id, bool *retry_needed)
 	}
 
 	tp_node = sys_get_node_by_gpu_id(&dest_topology, target_gpu_id);
-	if (!tp_node) {
+	if (!tp_node && amdgpu_ignore_single_gpuid) {
+		tp_node = sys_get_node_by_index(&dest_topology, 0);
+		if (!NODE_IS_GPU(tp_node)) {
+			pr_err("Cannot find the GPU node!\n");
+			fd = -ENODEV;
+			goto fail;
+		}
+		target_gpu_id = tp_node->gpu_id;
+		pr_warn("Forcing restore on gpu_id=0x%04x\n", target_gpu_id);
+	} else if (!tp_node) {
 		pr_err("Unable to find target gpu_id=0x%04x!\n", target_gpu_id);
 		fd = -ENODEV;
 		goto fail;

From e1fba1f7a00212b2e27447d544464344cea833e8 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Tue, 17 Feb 2026 12:16:43 +0000
Subject: [PATCH 03/10] plugins/amdgpu: Handle DRM render node mmaps

Outside the KFD world the CPU virtual memory address of a mmapp-ed buffer
object is not guaranteed to be same as the GPU virtual address. Therefore
the address check in VMA restore will prevent any restore of a render node
process.

For now we relax the criteria in case of render node to match only on the
offset. In the future this will need to be made more robust so strange
partial mmaps can be handled as well.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_plugin.c | 35 ++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_plugin.c b/plugins/amdgpu/amdgpu_plugin.c
index 56e02d510..68798a112 100644
--- a/plugins/amdgpu/amdgpu_plugin.c
+++ b/plugins/amdgpu/amdgpu_plugin.c
@@ -2211,25 +2211,28 @@ int amdgpu_plugin_update_vmamap(const char *in_path, const uint64_t addr, const
 	}
 
 	list_for_each_entry(vma_md, &update_vma_info_list, list) {
-		if (addr == vma_md->vma_entry && old_offset == vma_md->old_pgoff) {
-			*new_offset = vma_md->new_pgoff;
+		if (old_offset != vma_md->old_pgoff)
+			continue;
+		if (is_kfd && addr != vma_md->vma_entry)
+			continue;
 
-			*updated_fd = -1;
-			if (is_renderD) {
-				int fd = dup(vma_md->fd);
-				if (fd == -1) {
-					pr_perror("unable to duplicate the render fd");
-					return -1;
-				}
-				*updated_fd = fd;
+		*new_offset = vma_md->new_pgoff;
+		if (is_renderD) {
+			int fd = dup(vma_md->fd);
+
+			if (fd == -1) {
+				pr_perror("unable to duplicate the render fd");
+				return -1;
 			}
-
-			pr_debug("old_pgoff=0x%lx new_pgoff=0x%lx fd=%d\n",
-				 vma_md->old_pgoff, vma_md->new_pgoff,
-				 *updated_fd);
-
-			return 1;
+			*updated_fd = fd;
+		} else {
+			*updated_fd = -1;
 		}
+
+		pr_debug("old_pgoff=0x%lx new_pgoff=0x%lx fd=%d\n",
+			 vma_md->old_pgoff, vma_md->new_pgoff, *updated_fd);
+
+		return 1;
 	}
 	pr_info("No match for addr:0x%lx offset:%lx\n", addr, old_offset);
 	return 0;

From cb14adf67c3231a9641e4363d397d99f446b5e87 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Fri, 1 May 2026 12:36:08 +0100
Subject: [PATCH 04/10] plugins/amdgpu: Move GPU VA map to after content
 restore

Move GPU VA mapping operation to after the buffer object content is
restored in order to avoid libdrm VA range manager conflicting with the
already mapped buffers and so avoid failing the restore process.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_plugin_drm.c | 40 ++++++++++++++++--------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index 85a0ce5f0..caaed10e0 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -497,7 +497,6 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 		uint32_t handle;
 		struct drm_gem_change_handle change_args = { 0 };
 		union drm_amdgpu_gem_mmap mmap_args = { 0 };
-		struct drm_amdgpu_gem_va va_args = { 0 };
 		int fd_id;
 
 		if (work_already_completed(boinfo->handle, rd->drm_render_minor)) {
@@ -572,23 +571,6 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 			goto exit;
 		}
 
-		for (int j = 0; j < boinfo->num_of_vms; j++) {
-			DrmVmEntry *vminfo = boinfo->vm_entries[j];
-
-			va_args.handle = boinfo->handle;
-			va_args.operation = AMDGPU_VA_OP_MAP;
-			va_args.flags = vminfo->flags;
-			va_args.va_address = vminfo->addr;
-			va_args.offset_in_bo = vminfo->offset;
-			va_args.map_size = vminfo->size;
-
-			if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_VA, &va_args) == -1) {
-				pr_perror("Error Failed to call gem va ioctl");
-				ret = -1;
-				goto exit;
-			}
-		}
-
 		ret = save_vma_updates(boinfo->offset, boinfo->addr,
 				       mmap_args.out.addr_ptr, fd);
 		if (ret < 0)
@@ -614,6 +596,28 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 			goto exit;
 	}
 
+	for (int i = 0; i < rd->num_of_bos; i++) {
+		DrmBoEntry *boinfo = rd->bo_entries[i];
+
+		for (int j = 0; j < boinfo->num_of_vms; j++) {
+			DrmVmEntry *vminfo = boinfo->vm_entries[j];
+			struct drm_amdgpu_gem_va va_args = { };
+
+			va_args.handle = boinfo->handle;
+			va_args.operation = AMDGPU_VA_OP_MAP;
+			va_args.flags = vminfo->flags;
+			va_args.va_address = vminfo->addr;
+			va_args.offset_in_bo = vminfo->offset;
+			va_args.map_size = vminfo->size;
+
+			if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_VA, &va_args) == -1) {
+				ret = -errno;
+				pr_perror("Failed to map bo %u/%u!", i, j);
+				goto exit;
+			}
+		}
+	}
+
 exit:
 	if (ret < 0 && !bo_restore_called) {
 		for (int i = 0; i < rd->num_of_bos; i++) {

From ac3d6d23b577e2f9149b73f5505db5ce002b6187 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Mon, 4 May 2026 16:26:34 +0100
Subject: [PATCH 05/10] plugins/amdgpu: Re-open the DRM device when saving
 buffer object content

Re-open the DRM device when dumping buffer object content in order to
avoid the libdrm VA range manager conflicting with the client being
check-pointed. Otherwise buffer content cannot be saved and the process
aborts with a failure.

The current approach of initializing libdrm of the current fd is
insufficient because libdrm, despite libdrm "opening" a different fd it
does that by dup() which does not create a new DRM client, so no new
GPU VA address space gets created.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_plugin_drm.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index caaed10e0..b8bdd9abc 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -378,9 +378,21 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 
 		if (!libdrm_initialized) {
 			uint32_t major, minor;
+			char *drm_path;
+			int drm_fd;
 
-			ret = amdgpu_device_initialize(fd, &major, &minor,
+			/* Re-open device to avoid VA conflicts in sdma_copy_bo. */
+			drm_path = drmGetRenderDeviceNameFromFd(fd);
+			drm_fd = open(drm_path, O_RDWR | O_CLOEXEC);
+			if (drm_fd < 0) {
+				ret = -errno;
+				pr_perror("Failed to re-open %s", drm_path);
+				goto exit;
+			}
+
+			ret = amdgpu_device_initialize(drm_fd, &major, &minor,
 						       &h_dev);
+			close(drm_fd);
 			if (ret) {
 				pr_err("Failed to initialize amdgpu device - %s\n",
 				       strerror(-ret));

From 3c07837bada20c26ef83e560f8233a838edff837 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Wed, 13 May 2026 11:49:26 +0100
Subject: [PATCH 06/10] plugins/amdgpu: Add userptr support

Use the AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR flag returned from the
DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES ioctl to detect userptr objects.

For userptr mmap is not supported so it needs to be skipped, likewise the
dumping and restoring of buffer object content since that lives in the
normal process memory space.

The only trick is objects needs to be re-created without the
AMDGPU_GEM_USERPTR_VALIDATE flag, given that the restore stage runs before
the process address space has been restored. But that is fine, given that
if those object exists and have had this flag set, that means they have
past the validation on creation already.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_drm.h        |  14 ++-
 plugins/amdgpu/amdgpu_plugin_drm.c | 156 ++++++++++++++++++-----------
 plugins/amdgpu/criu-amdgpu.proto   |   1 +
 3 files changed, 106 insertions(+), 65 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_drm.h b/plugins/amdgpu/amdgpu_drm.h
index 69227a12b..09c31008c 100644
--- a/plugins/amdgpu/amdgpu_drm.h
+++ b/plugins/amdgpu/amdgpu_drm.h
@@ -898,6 +898,7 @@ struct drm_amdgpu_gem_op {
 };
 
 #define AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT	(1 << 0)
+#define AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR	(1 << 1)
 
 struct drm_amdgpu_gem_list_handles {
 	/* User pointer to array of drm_amdgpu_gem_bo_info_entry */
@@ -913,7 +914,7 @@ struct drm_amdgpu_gem_list_handles_entry {
 	/* gem handle of buffer object */
 	__u32 gem_handle;
 
-	/* Currently just one flag: IS_IMPORT */
+	/* AMDGPU_GEM_LIST_HANDLES_FLAG_* */
 	__u32 flags;
 
 	/* Size of bo */
@@ -922,11 +923,16 @@ struct drm_amdgpu_gem_list_handles_entry {
 	/* Preferred domains for GEM_CREATE */
 	__u64 preferred_domains;
 
-	/* GEM_CREATE flags for re-creation of buffer */
+	/* GEM_CREATE flags for re-creation of buffer or drm_amdgpu_gem_userptr.flags */
 	__u64 alloc_flags;
 
-	/* physical start_addr alignment in bytes for some HW requirements */
-	__u64 alignment;
+	union {
+		/* physical start_addr alignment in bytes for some HW requirements */
+		__u64 alignment;
+
+		/* drm_amdgpu_gem_userptr.addr for userptr objects */
+		__u64 userptr;
+	};
 };
 
 #define AMDGPU_VA_OP_MAP			1
diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index b8bdd9abc..32ed6691c 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -158,8 +158,8 @@ int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *st)
 
 static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int drm_fd, int *dmabufs)
 {
-	size_t image_size = 0, max_bo_size = 0, buffer_size;
-	struct amdgpu_gpu_info gpu_info = { 0 };
+	size_t image_size = 0, buffer_size = 0;
+	struct amdgpu_gpu_info gpu_info = {};
 	amdgpu_device_handle h_dev;
 	uint64_t max_copy_size;
 	uint32_t major, minor;
@@ -185,13 +185,18 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int
 								   SDMA_LINEAR_COPY_MAX_SIZE - 1;
 
 	for (i = 0; i < rd->num_of_bos; i++) {
-		if (rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
-			if (rd->bo_entries[i]->size > max_bo_size)
-				max_bo_size = rd->bo_entries[i]->size;
+		DrmBoEntry *entry = rd->bo_entries[i];
+
+		if ((entry->preferred_domains &
+		     (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
+		    !entry->is_userptr) {
+			if (entry->size > buffer_size)
+				buffer_size = entry->size;
 		}
 	}
 
-	buffer_size = max_bo_size;
+	if (!buffer_size)
+		goto exit;
 
 	ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), buffer_size);
 	if (ret) {
@@ -202,6 +207,9 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int
 	}
 
 	for (i = 0; i < rd->num_of_bos; i++) {
+		if (rd->bo_entries[i]->is_userptr)
+			continue;
+
 		if (!(rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)))
 			continue;
 
@@ -304,27 +312,32 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 		struct drm_amdgpu_gem_vm_entry *vm_info_entries = NULL;
 		DrmBoEntry *boinfo = rd->bo_entries[i];
 		struct drm_amdgpu_gem_list_handles_entry *entry = &entries[i];
-		union drm_amdgpu_gem_mmap mmap_args = { 0 };
 		int bo_contents_fd;
 		int dmabuf_fd;
 
 		boinfo->size = entry->size;
 		boinfo->alloc_flags = entry->alloc_flags;
 		boinfo->preferred_domains = entry->preferred_domains;
-		boinfo->alignment = entry->alignment;
+		boinfo->alignment = entry->alignment; /* Also sets userptr address. */
 		boinfo->handle = entry->gem_handle;
 		boinfo->is_import = (entry->flags & AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT) || shared_bo_has_exporter(boinfo->handle);
+		if (entry->flags & AMDGPU_GEM_LIST_HANDLES_FLAG_IS_USERPTR)
+			boinfo->is_userptr = boinfo->has_is_userptr = true;
 
-		mmap_args.in.handle = boinfo->handle;
+		if (!boinfo->is_userptr) {
+			union drm_amdgpu_gem_mmap mmap_args = { 0 };
 
-		if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
-			pr_perror("Error Failed to call mmap ioctl");
-			ret = -1;
-			goto exit;
+			mmap_args.in.handle = boinfo->handle;
+
+			if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
+				pr_perror("Error Failed to call mmap ioctl");
+				ret = -1;
+				goto exit;
+			}
+
+			boinfo->offset = mmap_args.out.addr_ptr;
 		}
 
-		boinfo->offset = mmap_args.out.addr_ptr;
-
 		while (1) {
 			struct drm_amdgpu_gem_op vm_info_args = {
 				.handle = entry->gem_handle,
@@ -402,47 +415,50 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 			libdrm_initialized = true;
 		}
 
-		ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd);
-		if (ret) {
-			pr_perror("Failed to get dmabuf fd from handle");
-			goto exit;
-		}
-
-		snprintf(path, sizeof(path), IMG_DRM_PAGES_FILE, rd->id, rd->drm_render_minor, i);
-		image_size = entry->size;
-		bo_contents_fd = open_img_file(path, true, &image_size, true);
-		if (bo_contents_fd < 0) {
-			ret = bo_contents_fd;
-			close(dmabuf_fd);
-			goto exit;
-		}
-
-		if (buffer_size < entry->size) {
-			if (buffer_size) {
-				free(buffer);
-				buffer = NULL;
+		if (!boinfo->is_userptr) {
+			ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd);
+			if (ret) {
+				pr_perror("Failed to get dmabuf fd from handle");
+				goto exit;
 			}
 
-			ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE),
-					     entry->size);
-			if (ret) {
-				errno = ret;
-				pr_perror("Failed to allocate userptr buffer");
-				ret = -ret;
-				close(bo_contents_fd);
+			snprintf(path, sizeof(path), IMG_DRM_PAGES_FILE, rd->id, rd->drm_render_minor, i);
+			image_size = entry->size;
+			bo_contents_fd = open_img_file(path, true, &image_size, true);
+			if (bo_contents_fd < 0) {
+				ret = bo_contents_fd;
 				close(dmabuf_fd);
 				goto exit;
 			}
 
-			buffer_size = entry->size;
-		}
+			if (buffer_size < entry->size) {
+				if (buffer_size) {
+					free(buffer);
+					buffer = NULL;
+				}
 
-		ret = sdma_copy_bo(dmabuf_fd, entry->size, bo_contents_fd,
-				   buffer, entry->size, h_dev, 0x1000,
-				   SDMA_OP_VRAM_READ, false);
-		close(bo_contents_fd);
-		if (ret)
-			goto exit;
+				ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE),
+						     entry->size);
+				if (ret) {
+					errno = ret;
+					pr_perror("Failed to allocate userptr buffer");
+					ret = -ret;
+					close(bo_contents_fd);
+					close(dmabuf_fd);
+					goto exit;
+				}
+
+				buffer_size = entry->size;
+			}
+
+			ret = sdma_copy_bo(dmabuf_fd, entry->size,
+					   bo_contents_fd, buffer, entry->size,
+					   h_dev, 0x1000, SDMA_OP_VRAM_READ,
+					   false);
+			close(bo_contents_fd);
+			if (ret)
+				goto exit;
+		}
 
 		if (dmabuf_fd != KFD_INVALID_FD)
 			close(dmabuf_fd);
@@ -508,7 +524,6 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 		int dmabuf_fd = -1;
 		uint32_t handle;
 		struct drm_gem_change_handle change_args = { 0 };
-		union drm_amdgpu_gem_mmap mmap_args = { 0 };
 		int fd_id;
 
 		if (work_already_completed(boinfo->handle, rd->drm_render_minor)) {
@@ -535,8 +550,23 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 				close(dmabuf_fd);
 				goto exit;
 			}
+		} else if (boinfo->is_userptr) {
+			struct drm_amdgpu_gem_userptr userptr_args = {
+				.size = boinfo->size,
+				.addr = boinfo->alignment,
+				.flags = boinfo->alloc_flags & ~AMDGPU_GEM_USERPTR_VALIDATE,
+			};
+
+			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR,
+				       &userptr_args);
+			if (ret < 0) {
+				ret = -errno;
+				pr_perror("Failed to create userptr object");
+				goto exit;
+			}
+			handle = userptr_args.handle;
 		} else {
-			union drm_amdgpu_gem_create create_args = { 0 };
+			union drm_amdgpu_gem_create create_args = {};
 
 			create_args.in.bo_size = boinfo->size;
 			create_args.in.alignment = boinfo->alignment;
@@ -575,18 +605,22 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 		if (ret)
 			goto exit;
 
-		mmap_args.in.handle = boinfo->handle;
+		if (!boinfo->is_userptr) {
+			union drm_amdgpu_gem_mmap mmap_args = {};
 
-		if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
-			pr_perror("Error Failed to call mmap ioctl");
-			ret = -1;
-			goto exit;
+			mmap_args.in.handle = boinfo->handle;
+
+			if (drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_args) == -1) {
+				pr_perror("Error Failed to call mmap ioctl");
+				ret = -1;
+				goto exit;
+			}
+
+			ret = save_vma_updates(boinfo->offset, boinfo->addr,
+					       mmap_args.out.addr_ptr, fd);
+			if (ret < 0)
+				goto exit;
 		}
-
-		ret = save_vma_updates(boinfo->offset, boinfo->addr,
-				       mmap_args.out.addr_ptr, fd);
-		if (ret < 0)
-			goto exit;
 	}
 
 	if (ret) {
diff --git a/plugins/amdgpu/criu-amdgpu.proto b/plugins/amdgpu/criu-amdgpu.proto
index 7682a8f21..89e59e2c6 100644
--- a/plugins/amdgpu/criu-amdgpu.proto
+++ b/plugins/amdgpu/criu-amdgpu.proto
@@ -73,6 +73,7 @@ message drm_bo_entry {
 	required uint32 is_import = 8;
 	required uint32 num_of_vms = 9;
 	repeated drm_vm_entry vm_entries = 10;
+	optional bool is_userptr = 11 [default = false];
 }
 
 message drm_vm_entry {

From 3797ff9d396c7cf352a54c0927da73ecddb87678 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Wed, 27 May 2026 11:10:59 +0100
Subject: [PATCH 07/10] plugins/amdgpu: Free buffer object container

Free the buffer object container upon successful dump.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_plugin_drm.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index 32ed6691c..37c858d26 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -125,6 +125,7 @@ static void free_e(CriuRenderNode *e)
 		if (e->bo_entries[i])
 			xfree(e->bo_entries[i]);
 	}
+	xfree(e->bo_entries);
 
 	xfree(e);
 }

From 9d11548e0a7700a9e12b4299acb6e21c62698517 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Thu, 14 May 2026 14:24:17 +0100
Subject: [PATCH 08/10] plugins/amdgpu: Add contexts support

Use the DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS ioctl to enumerate all
rendering contexts userspace had created, save them in a new array within
the existing file image, and restore them including the correct priorities
and pstate configuration.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_drm.h        |  31 +++++++
 plugins/amdgpu/amdgpu_plugin_drm.c | 144 +++++++++++++++++++++++++++++
 plugins/amdgpu/criu-amdgpu.proto   |  10 ++
 3 files changed, 185 insertions(+)

diff --git a/plugins/amdgpu/amdgpu_drm.h b/plugins/amdgpu/amdgpu_drm.h
index 09c31008c..39034fff7 100644
--- a/plugins/amdgpu/amdgpu_drm.h
+++ b/plugins/amdgpu/amdgpu_drm.h
@@ -58,6 +58,7 @@ extern "C" {
 #define DRM_AMDGPU_USERQ_SIGNAL		0x17
 #define DRM_AMDGPU_USERQ_WAIT		0x18
 #define DRM_AMDGPU_GEM_LIST_HANDLES	0x19
+#define DRM_AMDGPU_GEM_LIST_CONTEXTS	0x20
 /* not upstream */
 #define DRM_AMDGPU_GEM_DGMA		0x5c
 
@@ -84,6 +85,7 @@ extern "C" {
 #define DRM_IOCTL_AMDGPU_USERQ_SIGNAL	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_SIGNAL, struct drm_amdgpu_userq_signal)
 #define DRM_IOCTL_AMDGPU_USERQ_WAIT	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait)
 #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles)
+#define DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_CONTEXTS, struct drm_amdgpu_gem_list_contexts)
 
 #define DRM_IOCTL_AMDGPU_GEM_DGMA	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_DGMA, struct drm_amdgpu_gem_dgma)
 
@@ -935,6 +937,35 @@ struct drm_amdgpu_gem_list_handles_entry {
 	};
 };
 
+struct drm_amdgpu_gem_list_contexts {
+	/* User pointer to array of drm_amdgpu_gem_list_contexts_entry */
+	__u64 contexts;
+
+	/* Size of the contexts buffer / Number of contexts in the client (if larger than size of buffer, must retry) */
+	__u32 num_contexts;
+
+	__u32 padding;
+};
+
+struct drm_amdgpu_gem_list_contexts_entry {
+	/* gem context handle */
+	__u32 handle;
+
+	/* AMDGPU_GEM_LIST_CONTEXTS_FLAG_* */
+	__u32 flags;
+
+	/* context initial priority */
+	__s32 init_priority;
+
+	/* context override priority */
+	__s32 override_priority;
+
+	/* pstate flags */
+	__u32 pstate_flags;
+
+	__u32 padding;
+};
+
 #define AMDGPU_VA_OP_MAP			1
 #define AMDGPU_VA_OP_UNMAP			2
 #define AMDGPU_VA_OP_CLEAR			3
diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index 37c858d26..5d5157d67 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -95,6 +95,30 @@ static int allocate_bo_entries(CriuRenderNode *e, int num_bos)
 	return 0;
 }
 
+static int allocate_context_entries(CriuRenderNode *e, unsigned long num_contexts)
+{
+	e->contexts = xmalloc(sizeof(DrmContextEntry *) * num_contexts);
+	if (!e->contexts) {
+		pr_err("Failed to allocate context list\n");
+		return -ENOMEM;
+	}
+
+	for (int i = 0; i < num_contexts; i++) {
+		DrmContextEntry *context = xzalloc(sizeof(*context));
+
+		if (!context) {
+			pr_err("Failed to allocate context info\n");
+			return -ENOMEM;
+		}
+
+		drm_context_entry__init(context);
+
+		e->contexts[i] = context;
+		e->n_contexts++;
+	}
+	return 0;
+}
+
 static int allocate_vm_entries(DrmBoEntry *e, int num_vms)
 {
 	e->vm_entries = xmalloc(sizeof(DrmVmEntry *) * num_vms);
@@ -127,6 +151,12 @@ static void free_e(CriuRenderNode *e)
 	}
 	xfree(e->bo_entries);
 
+	for (int i = 0; i < e->n_contexts; i++) {
+		if (e->contexts[i])
+			xfree(e->contexts[i]);
+	}
+	xfree(e->contexts);
+
 	xfree(e);
 }
 
@@ -253,6 +283,10 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 	struct drm_amdgpu_gem_list_handles handles = {
 		.num_entries = 8,
 	};
+	struct drm_amdgpu_gem_list_contexts_entry *contexts = NULL;
+	struct drm_amdgpu_gem_list_contexts contexts_query = {
+		.num_contexts = 8,
+	};
 	bool libdrm_initialized = false;
 	unsigned long buffer_size = 0;
 	amdgpu_device_handle h_dev;
@@ -303,11 +337,67 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 			break;
 	} while (true);
 
+	do {
+		unsigned int num_contexts = contexts_query.num_contexts;
+
+		if (contexts) {
+			xfree(contexts);
+			contexts = NULL;
+		}
+
+		contexts = xzalloc(sizeof(*contexts) * contexts_query.num_contexts);
+		if (!contexts) {
+			ret = -ENOMEM;
+			goto exit;
+		}
+
+		contexts_query.contexts = (uintptr_t)contexts;
+		ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS, &contexts_query);
+		if (ret) {
+			ret = -errno;
+			if (errno == EINVAL) {
+				pr_info("This kernel appears not to have AMDGPU_GEM_LIST_CONTEXTS ioctl. Consider updating your kernel.\n");
+				contexts_query.num_contexts = 0;
+				break;
+			} else {
+				pr_perror("Failed to call bo info ioctl");
+				goto exit;
+			}
+		}
+
+		if (contexts_query.num_contexts <= num_contexts)
+			break;
+	} while (true);
+
 	rd->num_of_bos = handles.num_entries;
 	ret = allocate_bo_entries(rd, handles.num_entries);
 	if (ret)
 		goto exit;
 
+	if (contexts_query.num_contexts) {
+		rd->num_of_contexts = contexts_query.num_contexts;
+		ret = allocate_context_entries(rd, contexts_query.num_contexts);
+		if (ret)
+			goto exit;
+		rd->has_num_of_contexts = true;
+	}
+
+	for (unsigned int i = 0; i < contexts_query.num_contexts; i++) {
+		struct drm_amdgpu_gem_list_contexts_entry *entry = &contexts[i];
+		DrmContextEntry *context = rd->contexts[i];
+
+		context->handle = entry->handle;
+		context->flags = entry->flags;
+		context->init_priority = entry->init_priority;
+		context->override_priority = entry->override_priority;
+		context->pstate_flags = entry->pstate_flags;
+
+		pr_info("Saving context %u/%u: %u %x %d/%d %x\n",
+			i, contexts_query.num_contexts,
+			context->handle, context->flags, context->init_priority,
+			context->override_priority, context->pstate_flags);
+	}
+
 	for (int i = 0; i < handles.num_entries; i++) {
 		int num_vm_entries = 8;
 		struct drm_amdgpu_gem_vm_entry *vm_info_entries = NULL;
@@ -504,6 +594,7 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 exit:
 	free(buffer);
 	xfree(entries);
+	xfree(contexts);
 	free_e(rd);
 	if (libdrm_initialized)
 		amdgpu_device_deinitialize(h_dev);
@@ -520,6 +611,59 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 		return -ENOMEM;
 	memset(dmabufs, 0xff, sizeof(int) * rd->num_of_bos);
 
+	for (unsigned int i = 0; i < rd->num_of_contexts; i++) {
+		DrmContextEntry *context = rd->contexts[i];
+		union drm_amdgpu_ctx args = {};
+
+		args.in.op = AMDGPU_CTX_OP_ALLOC_CTX;
+		args.in.priority = context->init_priority;
+		ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &args);
+		if (ret < 0) {
+			ret = -errno;
+			pr_perror("Failed to create context");
+			goto exit;
+		}
+
+		if (args.out.alloc.ctx_id != context->handle) {
+			// FIXME
+			pr_err("Handle mismatch!\n");
+			goto exit;
+		}
+
+		if (context->override_priority != AMDGPU_CTX_PRIORITY_UNSET) {
+			union drm_amdgpu_sched sched = {};
+
+			sched.in.op = AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE;
+			sched.in.ctx_id = args.out.alloc.ctx_id;
+			sched.in.priority = context->override_priority;
+			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_SCHED, &sched);
+			if (ret < 0) {
+				ret = -errno;
+				pr_perror("Failed to override context priority");
+				goto exit;
+			}
+		}
+
+		if (context->pstate_flags != AMDGPU_CTX_STABLE_PSTATE_NONE) {
+			union drm_amdgpu_ctx pstate = {};
+
+			pstate.in.op = AMDGPU_CTX_OP_SET_STABLE_PSTATE;
+			pstate.in.ctx_id = args.out.alloc.ctx_id;
+			pstate.in.flags = context->pstate_flags;
+			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &pstate);
+			if (ret < 0) {
+				ret = -errno;
+				pr_perror("Failed to set context pstate");
+				goto exit;
+			}
+		}
+
+		pr_info("Restored context %u/%" PRIu64 ": %u %x %d/%d %x\n",
+			i, rd->num_of_contexts,
+			context->handle, context->flags, context->init_priority,
+			context->override_priority, context->pstate_flags);
+	}
+
 	for (int i = 0; i < rd->num_of_bos; i++) {
 		DrmBoEntry *boinfo = rd->bo_entries[i];
 		int dmabuf_fd = -1;
diff --git a/plugins/amdgpu/criu-amdgpu.proto b/plugins/amdgpu/criu-amdgpu.proto
index 89e59e2c6..f322a07e3 100644
--- a/plugins/amdgpu/criu-amdgpu.proto
+++ b/plugins/amdgpu/criu-amdgpu.proto
@@ -76,6 +76,14 @@ message drm_bo_entry {
 	optional bool is_userptr = 11 [default = false];
 }
 
+message drm_context_entry {
+	required uint32 handle = 1;
+	required uint32 flags = 2;
+	required  int32 init_priority = 3;
+	required  int32 override_priority = 4;
+	required uint32 pstate_flags = 5;
+}
+
 message drm_vm_entry {
 	required uint64 addr = 1;
 	required uint64 size = 2;
@@ -89,6 +97,8 @@ message criu_render_node {
 	required uint32 drm_render_minor = 3;
 	required uint64 num_of_bos = 4;
 	repeated drm_bo_entry bo_entries = 5;
+	optional uint64 num_of_contexts = 6 [default = 0];
+	repeated drm_context_entry contexts = 7;
 }
 
 message criu_dmabuf_node {

From 204cd33d1771a41fb03be2d101a1d393c05a3e77 Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Fri, 15 May 2026 09:46:06 +0100
Subject: [PATCH 09/10] plugins/amdgpu: Add support for renaming context
 handles

Following a similar approach as with buffer object handles, and for the
very same reason of not being able to request a specific handle when
creating contexts, use the newly added AMDGPU_CTX_OP_CHANGE_HANDLE
operation supported by the DRM_IOCTL_AMDGPU_CTX ioctl so we can re-create
the context handles as they were in the original process.

Before this change we could only restore a trivial userspace which only
created and never closed any contexts, but now we can handle "sparse"
namespaces as well.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_drm.h        |  1 +
 plugins/amdgpu/amdgpu_plugin_drm.c | 18 +++++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_drm.h b/plugins/amdgpu/amdgpu_drm.h
index 39034fff7..0e66707c1 100644
--- a/plugins/amdgpu/amdgpu_drm.h
+++ b/plugins/amdgpu/amdgpu_drm.h
@@ -278,6 +278,7 @@ union drm_amdgpu_bo_list {
 #define AMDGPU_CTX_OP_QUERY_STATE2	4
 #define AMDGPU_CTX_OP_GET_STABLE_PSTATE	5
 #define AMDGPU_CTX_OP_SET_STABLE_PSTATE	6
+#define AMDGPU_CTX_OP_CHANGE_HANDLE	7
 
 /* GPU reset status */
 #define AMDGPU_CTX_NO_RESET		0
diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index 5d5157d67..c5021bf12 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -625,16 +625,24 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 		}
 
 		if (args.out.alloc.ctx_id != context->handle) {
-			// FIXME
-			pr_err("Handle mismatch!\n");
-			goto exit;
+			uint32_t id = args.out.alloc.ctx_id;
+
+			args.in.op = AMDGPU_CTX_OP_CHANGE_HANDLE;
+			args.in.ctx_id = id;
+			args.in.flags = context->handle;
+			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &args);
+			if (ret < 0) {
+				ret = -errno;
+				pr_perror("Failed to rename context");
+				goto exit;
+			}
 		}
 
 		if (context->override_priority != AMDGPU_CTX_PRIORITY_UNSET) {
 			union drm_amdgpu_sched sched = {};
 
 			sched.in.op = AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE;
-			sched.in.ctx_id = args.out.alloc.ctx_id;
+			sched.in.ctx_id = context->handle;
 			sched.in.priority = context->override_priority;
 			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_SCHED, &sched);
 			if (ret < 0) {
@@ -648,7 +656,7 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 			union drm_amdgpu_ctx pstate = {};
 
 			pstate.in.op = AMDGPU_CTX_OP_SET_STABLE_PSTATE;
-			pstate.in.ctx_id = args.out.alloc.ctx_id;
+			pstate.in.ctx_id = context->handle;
 			pstate.in.flags = context->pstate_flags;
 			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CTX, &pstate);
 			if (ret < 0) {

From c5150042fde06c5152bf670d7402262115f0923f Mon Sep 17 00:00:00 2001
From: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Date: Tue, 19 May 2026 11:55:10 +0100
Subject: [PATCH 10/10] plugins/amdgpu: Add support for
 AMDGPU_GEM_CREATE_VM_ALWAYS_VALID objects

Always valid buffer objects are special in that they cannot be exported
via dmabuf and as such the current code paths for dumping and restoring
their content cannot work.

To support them we make use of the new DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER
ioctl, which allows access to the kernel side content copy code.

This is beneficial not just for handling VM always valid objects, but
also removes issues around GPU virtual address space conflicts, where
neither the dump or restore stages do not need to know, or attempt to work
work around not being able to know, the current VM virtual address range
allocations.

Therefore we implement this as the preferred method of copying buffer
object content and only fall-back on the existing code paths if the
current kernel does not support it.

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
---
 plugins/amdgpu/amdgpu_drm.h        |  16 ++
 plugins/amdgpu/amdgpu_plugin_drm.c | 256 ++++++++++++++++++++++++-----
 2 files changed, 229 insertions(+), 43 deletions(-)

diff --git a/plugins/amdgpu/amdgpu_drm.h b/plugins/amdgpu/amdgpu_drm.h
index 0e66707c1..a77096831 100644
--- a/plugins/amdgpu/amdgpu_drm.h
+++ b/plugins/amdgpu/amdgpu_drm.h
@@ -59,6 +59,7 @@ extern "C" {
 #define DRM_AMDGPU_USERQ_WAIT		0x18
 #define DRM_AMDGPU_GEM_LIST_HANDLES	0x19
 #define DRM_AMDGPU_GEM_LIST_CONTEXTS	0x20
+#define DRM_AMDGPU_GEM_COPY_BUFFER	0x21
 /* not upstream */
 #define DRM_AMDGPU_GEM_DGMA		0x5c
 
@@ -86,6 +87,7 @@ extern "C" {
 #define DRM_IOCTL_AMDGPU_USERQ_WAIT	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait)
 #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles)
 #define DRM_IOCTL_AMDGPU_GEM_LIST_CONTEXTS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_CONTEXTS, struct drm_amdgpu_gem_list_contexts)
+#define DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_COPY_BUFFER, union drm_amdgpu_gem_copy_buffer)
 
 #define DRM_IOCTL_AMDGPU_GEM_DGMA	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_DGMA, struct drm_amdgpu_gem_dgma)
 
@@ -233,6 +235,20 @@ union drm_amdgpu_gem_create {
 	struct drm_amdgpu_gem_create_out	out;
 };
 
+struct drm_amdgpu_gem_copy_buffer_in {
+	__u32	src_handle;
+	__u32	dst_handle;
+};
+
+struct drm_amdgpu_gem_copy_buffer_out {
+	__u32	fence_fd;
+};
+
+union drm_amdgpu_gem_copy_buffer {
+	struct drm_amdgpu_gem_copy_buffer_in	in;
+	struct drm_amdgpu_gem_copy_buffer_out	out;
+};
+
 /** Opcode to create new residency list.  */
 #define AMDGPU_BO_LIST_OP_CREATE	0
 /** Opcode to destroy previously created residency list */
diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
index c5021bf12..3e88a6d00 100644
--- a/plugins/amdgpu/amdgpu_plugin_drm.c
+++ b/plugins/amdgpu/amdgpu_plugin_drm.c
@@ -187,11 +187,62 @@ int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *st)
 	return 0;
 }
 
+static int sync_fence_wait(int fd, int timeout)
+{
+	struct pollfd fds = { fd, POLLIN };
+	int ret;
+
+	while (true) {
+		ret = poll(&fds, 1, timeout);
+		if (ret > 0) {
+			if (fds.revents & (POLLERR | POLLNVAL))
+				return -EINVAL;
+
+			return 0;
+		} else if (ret == 0) {
+			return -ETIME;
+		} else {
+			ret = -errno;
+			if (ret == -EINTR || ret == -EAGAIN)
+				continue;
+			return ret;
+		}
+	}
+}
+
+static int
+amdgpu_copy_buffer(int drm_fd, uint32_t src_handle, uint32_t dst_handle)
+{
+	union drm_amdgpu_gem_copy_buffer copy_args = {
+		.in.src_handle = src_handle,
+		.in.dst_handle = dst_handle,
+	};
+	int ret;
+
+	ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_GEM_COPY_BUFFER, &copy_args);
+	if (ret) {
+		ret = -errno;
+		pr_perror("Failed to copy buffer object");
+
+		return ret;
+	}
+
+	ret = sync_fence_wait(copy_args.out.fence_fd, 10000);
+	close(copy_args.out.fence_fd);
+	if (ret) {
+		pr_err("Failed to wait for buffer object copy! (%s)\n",
+		       strerror(-ret));
+	}
+
+	return ret;
+}
+
 static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int drm_fd, int *dmabufs)
 {
 	size_t image_size = 0, buffer_size = 0;
 	struct amdgpu_gpu_info gpu_info = {};
 	amdgpu_device_handle h_dev;
+	int has_copy_buffer = -1;
 	uint64_t max_copy_size;
 	uint32_t major, minor;
 	void *buffer = NULL;
@@ -238,30 +289,85 @@ static int restore_bo_contents_drm(int drm_render_minor, CriuRenderNode *rd, int
 	}
 
 	for (i = 0; i < rd->num_of_bos; i++) {
-		if (rd->bo_entries[i]->is_userptr)
+		DrmBoEntry *entry = rd->bo_entries[i];
+
+		if (entry->is_userptr ||
+		    !(entry->preferred_domains &
+		      (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) ||
+		    entry->num_of_vms == 0)
 			continue;
 
-		if (!(rd->bo_entries[i]->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)))
-			continue;
+		snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->id,
+			 drm_render_minor, i);
 
-		if (rd->bo_entries[i]->num_of_vms == 0)
-			continue;
+		/* Try AMDGPU_GEM_COPY_BUFFER first */
+		if ((has_copy_buffer < 0 || has_copy_buffer == 1)) {
+			struct drm_amdgpu_gem_userptr userptr_args = {
+				.addr = (uintptr_t)buffer,
+				.size = entry->size,
+			};
 
-		snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->id, drm_render_minor, i);
+			bo_contents_fd = open_img_file(img_path, false,
+						       &image_size, true);
+			if (bo_contents_fd < 0) {
+				ret = bo_contents_fd;
+				break;
+			}
 
-		bo_contents_fd = open_img_file(img_path, false, &image_size, true);
-		if (bo_contents_fd < 0) {
-			ret = bo_contents_fd;
-			break;
+			if (image_size != entry->size) {
+				pr_err("Image size mismatch!\n");
+				ret = -ENXIO;
+				break;
+			}
+
+			ret = img_read(bo_contents_fd, buffer, image_size);
+			close(bo_contents_fd);
+			if (ret)
+				break;
+
+			ret = drmIoctl(drm_fd, DRM_IOCTL_AMDGPU_GEM_USERPTR,
+				       &userptr_args);
+			if (ret) {
+				ret = -errno;
+				pr_perror("Failed to create userptr object");
+				break;
+			} else {
+				struct drm_gem_close close_args = {
+					.handle = userptr_args.handle,
+				};
+
+				ret = amdgpu_copy_buffer(drm_fd,
+							 userptr_args.handle,
+							 entry->handle);
+				has_copy_buffer = ret == 0;
+
+				ret = drmIoctl(drm_fd, DRM_IOCTL_GEM_CLOSE,
+					       &close_args);
+				if (ret) {
+					ret = -errno;
+					pr_perror("Failed to close userptr object");
+				}
+			}
 		}
 
-		ret = sdma_copy_bo(dmabufs[i], rd->bo_entries[i]->size,
-				   bo_contents_fd, buffer, buffer_size, h_dev,
-				   max_copy_size, SDMA_OP_VRAM_WRITE, true);
-		close(bo_contents_fd);
-		if (ret) {
-			pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n", i);
-			break;
+		if (!has_copy_buffer) {
+			bo_contents_fd = open_img_file(img_path, false,
+						       &image_size, true);
+			if (bo_contents_fd < 0) {
+				ret = bo_contents_fd;
+				break;
+			}
+
+			ret = sdma_copy_bo(dmabufs[i], entry->size,
+					   bo_contents_fd, buffer, buffer_size,
+					   h_dev, max_copy_size,
+					   SDMA_OP_VRAM_WRITE, true);
+			close(bo_contents_fd);
+			if (ret) {
+				pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n",
+				       i);
+				break;
+			}
 		}
 	}
 
@@ -277,6 +383,30 @@ exit:
 	return ret;
 }
 
+static int
+prepare_buffer(unsigned long size, void **buf, unsigned long *bufsize)
+{
+	int ret;
+
+	if (*bufsize >= size)
+		return 0;
+
+	if (*bufsize) {
+		free(*buf);
+		*buf = NULL;
+	}
+
+	ret = posix_memalign(buf, sysconf(_SC_PAGE_SIZE), size);
+	if (ret) {
+		errno = ret;
+		pr_perror("Failed to allocate userptr buffer");
+	} else {
+		*bufsize = size;
+	}
+
+	return -ret;
+}
+
 int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 {
 	struct drm_amdgpu_gem_list_handles_entry *entries = NULL;
@@ -290,6 +420,7 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 	bool libdrm_initialized = false;
 	unsigned long buffer_size = 0;
 	amdgpu_device_handle h_dev;
+	int has_copy_buffer = -1;
 	char path[PATH_MAX];
 	CriuRenderNode *rd;
 	unsigned char *buf;
@@ -403,8 +534,8 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 		struct drm_amdgpu_gem_vm_entry *vm_info_entries = NULL;
 		DrmBoEntry *boinfo = rd->bo_entries[i];
 		struct drm_amdgpu_gem_list_handles_entry *entry = &entries[i];
+		int dmabuf_fd = KFD_INVALID_FD;
 		int bo_contents_fd;
-		int dmabuf_fd;
 
 		boinfo->size = entry->size;
 		boinfo->alloc_flags = entry->alloc_flags;
@@ -480,7 +611,56 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 		}
 		xfree(vm_info_entries);
 
-		if (!libdrm_initialized) {
+		/* Try AMDGPU_GEM_COPY_BUFFER first */
+		if ((has_copy_buffer < 0 || has_copy_buffer == 1) &&
+		    !boinfo->is_userptr) {
+			struct drm_amdgpu_gem_userptr userptr_args = {
+				.size = entry->size,
+			};
+
+			ret = prepare_buffer(entry->size, &buffer,
+					     &buffer_size);
+			if (ret)
+				goto exit;
+
+			userptr_args.addr = (uintptr_t)buffer;
+			ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR,
+				       &userptr_args);
+			if (ret) {
+				ret = -errno;
+				pr_perror("Failed to create userptr object");
+				goto exit;
+			} else {
+				struct drm_gem_close close_args = {
+					.handle = userptr_args.handle,
+				};
+
+				ret = amdgpu_copy_buffer(fd,
+							 entry->gem_handle,
+							 userptr_args.handle);
+				has_copy_buffer = ret == 0;
+
+				ret = drmIoctl(fd, DRM_IOCTL_GEM_CLOSE,
+					       &close_args);
+				if (ret) {
+					ret = -errno;
+					pr_perror("Failed to close userptr object");
+				}
+
+				if (has_copy_buffer == 1) {
+					snprintf(path, sizeof(path),
+						 IMG_DRM_PAGES_FILE, rd->id,
+						 rd->drm_render_minor, i);
+					ret = write_img_file(path, buffer,
+							     entry->size);
+					if (ret)
+						goto exit;
+				}
+			}
+		}
+
+		if (!has_copy_buffer && !boinfo->is_userptr &&
+		    !libdrm_initialized) {
 			uint32_t major, minor;
 			char *drm_path;
 			int drm_fd;
@@ -506,7 +686,7 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 			libdrm_initialized = true;
 		}
 
-		if (!boinfo->is_userptr) {
+		if (!has_copy_buffer && !boinfo->is_userptr) {
 			ret = drmPrimeHandleToFD(fd, boinfo->handle, 0, &dmabuf_fd);
 			if (ret) {
 				pr_perror("Failed to get dmabuf fd from handle");
@@ -522,24 +702,11 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
 				goto exit;
 			}
 
-			if (buffer_size < entry->size) {
-				if (buffer_size) {
-					free(buffer);
-					buffer = NULL;
-				}
-
-				ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE),
-						     entry->size);
-				if (ret) {
-					errno = ret;
-					pr_perror("Failed to allocate userptr buffer");
-					ret = -ret;
-					close(bo_contents_fd);
-					close(dmabuf_fd);
-					goto exit;
-				}
-
-				buffer_size = entry->size;
+			ret = prepare_buffer(entry->size, &buffer, &buffer_size);
+			if (ret) {
+				close(bo_contents_fd);
+				close(dmabuf_fd);
+				goto exit;
 			}
 
 			ret = sdma_copy_bo(dmabuf_fd, entry->size,
@@ -733,10 +900,13 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 			}
 			handle = create_args.out.handle;
 
-			ret = drmPrimeHandleToFD(fd, handle, 0, &dmabuf_fd);
-			if (ret) {
-				pr_perror("Failed to get dmabuf fd from handle");
-				goto exit;
+			if (!(boinfo->alloc_flags &
+			      AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)) {
+				ret = drmPrimeHandleToFD(fd, handle, 0, &dmabuf_fd);
+				if (ret) {
+					pr_perror("Failed to get dmabuf fd from handle");
+					goto exit;
+				}
 			}
 		}
 
@@ -749,7 +919,7 @@ int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
 			goto exit;
 		}
 
-		if (!boinfo->is_import)
+		if (!boinfo->is_import && dmabuf_fd != -1)
 			store_dmabuf_fd(boinfo->handle, dmabuf_fd);
 
 		dmabufs[i] = dmabuf_fd;