memfd: add file support

See "man memfd_create" for more information of what memfd is.

This adds support for memfd open files, that are not not memory mapped.

* We add a new kind of file: MEMFD.
* We add two image types MEMFD_FILE, and MEMFD_INODE.
  MEMFD_FILE contains usual file information (e.g., position).
  MEMFD_INODE contains the memfd name, and a shmid identifier
  referring to the content.
* We reuse the shmem facilities for dumping memfd content as it
  would be easier to support incremental checkpoints in the future.

Signed-off-by: Nicolas Viennot <Nicolas.Viennot@twosigma.com>

Makefile.config

@@ -64,7 +64,7 @@ export DEFINES += $(FEATURE_DEFINES)
 export CFLAGS += $(FEATURE_DEFINES)
 
 FEATURES_LIST	:= TCP_REPAIR STRLCPY STRLCAT PTRACE_PEEKSIGINFO \
-	SETPROCTITLE_INIT MEMFD TCP_REPAIR_WINDOW FSCONFIG
+	SETPROCTITLE_INIT MEMFD TCP_REPAIR_WINDOW FSCONFIG MEMFD_CREATE
 
 # $1 - config name
 define gen-feature-test

criu/Makefile.crtools

@@ -37,6 +37,7 @@ obj-y			+= libnetlink.o
 obj-y			+= log.o
 obj-y			+= lsm.o
 obj-y			+= mem.o
+obj-y			+= memfd.o
 obj-y			+= mount.o
 obj-y			+= filesystems.o
 obj-y			+= namespaces.o

criu/cr-restore.c

@@ -76,6 +76,7 @@
 #include "sigframe.h"
 #include "fdstore.h"
 #include "string.h"
+#include "memfd.h"
 
 #include "parasite-syscall.h"
 #include "files-reg.h"
@@ -289,6 +290,7 @@ static struct collect_image_info *cinfos_files[] = {
 	&fanotify_cinfo,
 	&fanotify_mark_cinfo,
 	&ext_file_cinfo,
+	&memfd_cinfo,
 };
 
 /* These images are required to restore namespaces */

criu/files.c

@@ -34,6 +34,7 @@
 #include "sk-packet.h"
 #include "mount.h"
 #include "signalfd.h"
+#include "memfd.h"
 #include "namespaces.h"
 #include "tun.h"
 #include "timerfd.h"
@@ -546,13 +547,17 @@ static int dump_one_file(struct pid *pid, int fd, int lfd, struct fd_opts *opts,
 			return -1;
 
 		p.link = &link;
-		if (link.name[1] == '/')
-			return do_dump_gen_file(&p, lfd, &regfile_dump_ops, e);
 
-		if (check_ns_proc(&link))
-			return do_dump_gen_file(&p, lfd, &nsfile_dump_ops, e);
+		if (is_memfd(p.stat.st_dev, &link.name[1]))
+			ops = &memfd_dump_ops;
+		else if (link.name[1] == '/')
+			ops = &regfile_dump_ops;
+		else if (check_ns_proc(&link))
+			ops = &nsfile_dump_ops;
+		else
+			return dump_unsupp_fd(&p, lfd, "reg", link.name + 1, e);
 
-		return dump_unsupp_fd(&p, lfd, "reg", link.name + 1, e);
+		return do_dump_gen_file(&p, lfd, ops, e);
 	}
 
 	if (S_ISFIFO(p.stat.st_mode)) {
@@ -1721,6 +1726,9 @@ static int collect_one_file(void *o, ProtobufCMessage *base, struct cr_img *i)
 	case FD_TYPES__TTY:
 		ret = collect_one_file_entry(fe, fe->tty->id, &fe->tty->base, &tty_cinfo);
 		break;
+	case FD_TYPES__MEMFD:
+		ret = collect_one_file_entry(fe, fe->memfd->id, &fe->memfd->base, &memfd_cinfo);
+		break;
 	}
 
 	return ret;

criu/image-desc.c

@@ -66,6 +66,7 @@ struct cr_fd_desc_tmpl imgset_template[CR_FD_MAX] = {
 	FD_ENTRY(FS,		"fs-%u"),
 	FD_ENTRY(REMAP_FPATH,	"remap-fpath"),
 	FD_ENTRY_F(GHOST_FILE,	"ghost-file-%x", O_NOBUF),
+	FD_ENTRY_F(MEMFD_INODE,	"memfd-%u", O_NOBUF),
 	FD_ENTRY(TCP_STREAM,	"tcp-stream-%x"),
 	FD_ENTRY(MNTS,		"mountpoints-%u"),
 	FD_ENTRY(NETDEV,	"netdev-%u"),

criu/include/image-desc.h

@@ -106,6 +106,8 @@ enum {
 	CR_FD_FIFO,
 	CR_FD_PIPES,
 	CR_FD_TTY_FILES,
+	CR_FD_MEMFD_FILE,
+	CR_FD_MEMFD_INODE,
 
 	CR_FD_AUTOFS,
 

criu/include/magic.h

@@ -94,6 +94,7 @@
 #define BINFMT_MISC_MAGIC	0x67343323 /* Apatity */
 #define AUTOFS_MAGIC		0x49353943 /* Sochi */
 #define FILES_MAGIC		0x56303138 /* Toropets */
+#define MEMFD_INODE_MAGIC	0x48453499 /* Dnipro */
 
 #define IFADDR_MAGIC		RAW_IMAGE_MAGIC
 #define ROUTE_MAGIC		RAW_IMAGE_MAGIC

criu/include/memfd.h

@@ -0,0 +1,24 @@
+#ifndef __CR_MEMFD_H__
+#define __CR_MEMFD_H__
+
+#include <sys/stat.h>
+#include "int.h"
+#include "common/config.h"
+
+extern int is_memfd(dev_t dev, const char *path);
+extern const struct fdtype_ops memfd_dump_ops;
+
+extern struct collect_image_info memfd_cinfo;
+
+#ifdef CONFIG_HAS_MEMFD_CREATE
+# include <sys/mman.h>
+#else
+# include <sys/syscall.h>
+# include <linux/memfd.h>
+static inline int memfd_create(const char *name, unsigned int flags)
+{
+	return syscall(SYS_memfd_create, name, flags);
+}
+#endif /* CONFIG_HAS_MEMFD_CREATE */
+
+#endif /* __CR_MEMFD_H__ */

criu/include/protobuf-desc.h

@@ -61,6 +61,8 @@ enum {
 	PB_AUTOFS,
 	PB_GHOST_CHUNK,
 	PB_FILE,
+	PB_MEMFD_FILE,
+	PB_MEMFD_INODE,		/* 60 */
 
 	/* PB_AUTOGEN_STOP */
 

criu/include/shmem.h

@@ -13,8 +13,11 @@ extern int collect_sysv_shmem(unsigned long shmid, unsigned long size);
 extern int cr_dump_shmem(void);
 extern int add_shmem_area(pid_t pid, VmaEntry *vma, u64 *map);
 extern int fixup_sysv_shmems(void);
+extern int dump_one_memfd_shmem(int fd, unsigned long shmid, unsigned long size);
 extern int dump_one_sysv_shmem(void *addr, unsigned long size, unsigned long shmid);
 extern int restore_sysv_shmem_content(void *addr, unsigned long size, unsigned long shmid);
+extern int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size);
+
 
 #define SYSV_SHMEM_SKIP_FD	(0x7fffffff)
 

criu/kerndat.c

@@ -42,6 +42,7 @@
 #include "vdso.h"
 #include "kcmp.h"
 #include "sched.h"
+#include "memfd.h"
 
 struct kerndat_s kdat = {
 };
@@ -409,7 +410,7 @@ static bool kerndat_has_memfd_create(void)
 {
 	int ret;
 
-	ret = syscall(SYS_memfd_create, NULL, 0);
+	ret = memfd_create(NULL, 0);
 
 	if (ret == -1 && errno == ENOSYS)
 		kdat.has_memfd = false;

criu/memfd.c

@@ -0,0 +1,350 @@
+#include <unistd.h>
+
+#include "common/compiler.h"
+#include "common/lock.h"
+#include "memfd.h"
+#include "fdinfo.h"
+#include "imgset.h"
+#include "image.h"
+#include "util.h"
+#include "log.h"
+#include "files.h"
+#include "fs-magic.h"
+#include "kerndat.h"
+#include "files-reg.h"
+#include "rst-malloc.h"
+#include "fdstore.h"
+#include "file-ids.h"
+#include "namespaces.h"
+#include "shmem.h"
+
+#include "protobuf.h"
+#include "images/memfd.pb-c.h"
+
+#define MEMFD_PREFIX "/memfd:"
+#define MEMFD_PREFIX_LEN (sizeof(MEMFD_PREFIX)-1)
+
+struct memfd_inode {
+	struct list_head	list;
+	u32			id;
+	union {
+		/* Only for dump */
+		struct {
+			u32	dev;
+			u32	ino;
+		};
+		/* Only for restore */
+		struct {
+			mutex_t	lock;
+			int	fdstore_id;
+		};
+	};
+};
+
+static LIST_HEAD(memfd_inodes);
+
+/*
+ * Dump only
+ */
+
+static u32 memfd_inode_ids = 1;
+
+int is_memfd(dev_t dev, const char *path)
+{
+	/*
+	 * TODO When MAP_HUGETLB is used, the file device is not shmem_dev,
+	 * Note that other parts of CRIU have similar issues, see
+	 * is_anon_shmem_map().
+	 */
+	return dev == kdat.shmem_dev &&
+		!strncmp(path, MEMFD_PREFIX, MEMFD_PREFIX_LEN);
+}
+
+static int dump_memfd_inode(int fd, struct memfd_inode *inode,
+			    const char *name, const struct stat *st)
+{
+	int ret = -1;
+	struct cr_img *img = NULL;
+	MemfdInodeEntry mie = MEMFD_INODE_ENTRY__INIT;
+	u32 shmid;
+
+	 /*
+	  * shmids are chosen as the inode number of the corresponding mmaped
+	  * file. See handle_vma() in proc_parse.c.
+	  * It works for memfd too, because we share the same device as the
+	  * shmem device.
+	  */
+	shmid = inode->ino;
+
+	pr_info("Dumping memfd:%s contents (id %#x, shmid: %#x, size: %"PRIu64")\n",
+		name, inode->id, shmid, st->st_size);
+
+	if (dump_one_memfd_shmem(fd, shmid, st->st_size) < 0)
+		goto out;
+
+	img = open_image(CR_FD_MEMFD_INODE, O_DUMP, inode->id);
+	if (!img)
+		goto out;
+
+	mie.uid = userns_uid(st->st_uid);
+	mie.gid = userns_gid(st->st_gid);
+	mie.name = (char *)name;
+	mie.size = st->st_size;
+	mie.shmid = shmid;
+
+	if (pb_write_one(img, &mie, PB_MEMFD_INODE))
+		goto out;
+
+	ret = 0;
+
+out:
+	if (img)
+		close_image(img);
+	return ret;
+}
+
+static struct memfd_inode *dump_unique_memfd_inode(int lfd, const char *name, const struct stat *st)
+{
+	struct memfd_inode *inode;
+
+	list_for_each_entry(inode, &memfd_inodes, list)
+		if ((inode->dev == st->st_dev) && (inode->ino == st->st_ino))
+			return inode;
+
+	inode = xmalloc(sizeof(*inode));
+	if (inode == NULL)
+		return NULL;
+
+	inode->dev = st->st_dev;
+	inode->ino = st->st_ino;
+	inode->id = memfd_inode_ids++;
+
+	if (dump_memfd_inode(lfd, inode, name, st)) {
+		xfree(inode);
+		return NULL;
+	}
+
+	list_add_tail(&inode->list, &memfd_inodes);
+
+	return inode;
+}
+
+static int dump_one_memfd(int lfd, u32 id, const struct fd_parms *p)
+{
+	MemfdFileEntry mfe = MEMFD_FILE_ENTRY__INIT;
+	FileEntry fe = FILE_ENTRY__INIT;
+	struct memfd_inode *inode;
+	struct fd_link _link, *link;
+	const char *name;
+
+	if (!p->link) {
+		if (fill_fdlink(lfd, p, &_link))
+			return -1;
+		link = &_link;
+	} else
+		link = p->link;
+
+	strip_deleted(link);
+	name = &link->name[1+MEMFD_PREFIX_LEN];
+
+	inode = dump_unique_memfd_inode(lfd, name, &p->stat);
+	if (!inode)
+		return -1;
+
+	mfe.id = id;
+	mfe.flags = p->flags;
+	mfe.pos = p->pos;
+	mfe.fown = (FownEntry *)&p->fown;
+	mfe.inode_id = inode->id;
+
+	fe.type = FD_TYPES__MEMFD;
+	fe.id = mfe.id;
+	fe.memfd = &mfe;
+
+	return pb_write_one(img_from_set(glob_imgset, CR_FD_FILES), &fe, PB_FILE);
+}
+
+const struct fdtype_ops memfd_dump_ops = {
+	.type		= FD_TYPES__MEMFD,
+	.dump		= dump_one_memfd,
+};
+
+
+/*
+ * Restore only
+ */
+
+struct memfd_info {
+	MemfdFileEntry		*mfe;
+	struct file_desc	d;
+	struct memfd_inode	*inode;
+};
+
+static struct memfd_inode *memfd_alloc_inode(int id)
+{
+	struct memfd_inode *inode;
+
+	list_for_each_entry(inode, &memfd_inodes, list)
+		if (inode->id == id)
+			return inode;
+
+	inode = shmalloc(sizeof(*inode));
+	if (!inode)
+		return NULL;
+
+	inode->id = id;
+	mutex_init(&inode->lock);
+	inode->fdstore_id = -1;
+
+	list_add_tail(&inode->list, &memfd_inodes);
+	return inode;
+}
+
+extern int restore_memfd_shm(int fd, u64 id, u64 size);
+static int memfd_open_inode_nocache(struct memfd_inode *inode)
+{
+	MemfdInodeEntry *mie = NULL;
+	struct cr_img *img = NULL;
+	int fd = -1;
+	int ret = -1;
+	int flags;
+
+	img = open_image(CR_FD_MEMFD_INODE, O_RSTR, inode->id);
+	if (!img)
+		goto out;
+
+	if (pb_read_one(img, &mie, PB_MEMFD_INODE) < 0)
+		goto out;
+
+	fd = memfd_create(mie->name, 0);
+	if (fd < 0) {
+		pr_perror("Can't create memfd:%s", mie->name);
+		goto out;
+	}
+
+	if (restore_memfd_shmem_content(fd, mie->shmid, mie->size))
+		goto out;
+
+	if (fchown(fd, mie->uid, mie->gid)) {
+		pr_perror("Can't change uid %d gid %d of memfd:%s",
+			  (int)mie->uid, (int)mie->gid, mie->name);
+		goto out;
+	}
+
+	inode->fdstore_id = fdstore_add(fd);
+	if (inode->fdstore_id < 0)
+		goto out;
+
+	ret = fd;
+	fd = -1;
+
+out:
+	if (fd != -1)
+		close(fd);
+	if (img)
+		close_image(img);
+	if (mie)
+		memfd_inode_entry__free_unpacked(mie, NULL);
+	return ret;
+}
+
+static int memfd_open_inode(struct memfd_inode *inode)
+{
+	int fd;
+
+	if (inode->fdstore_id != -1)
+		return fdstore_get(inode->fdstore_id);
+
+	mutex_lock(&inode->lock);
+	if (inode->fdstore_id != -1)
+		fd = fdstore_get(inode->fdstore_id);
+	else
+		fd = memfd_open_inode_nocache(inode);
+	mutex_unlock(&inode->lock);
+
+	return fd;
+}
+
+static int memfd_open(struct file_desc *d, u32 *fdflags)
+{
+	char lpath[PSFDS];
+	struct memfd_info *mfi;
+	MemfdFileEntry *mfe;
+	int fd, _fd;
+	u32 flags;
+
+	mfi = container_of(d, struct memfd_info, d);
+	mfe = mfi->mfe;
+
+	pr_info("Restoring memfd id=%d\n", mfe->id);
+
+	fd = memfd_open_inode(mfi->inode);
+	if (fd < 0)
+		goto err;
+
+	/* Reopen the fd with original permissions */
+	sprintf(lpath, "/proc/self/fd/%d", fd);
+	flags = fdflags ? *fdflags : mfe->flags;
+	/*
+	 * Ideally we should call compat version open() to not force the
+	 * O_LARGEFILE file flag with regular open(). It doesn't seem that
+	 * important though.
+	 */
+	_fd = open(lpath, flags);
+	if (_fd < 0) {
+		pr_perror("Can't reopen memfd id=%d", mfe->id);
+		goto err;
+	}
+	close(fd);
+	fd = _fd;
+
+	if (restore_fown(fd, mfe->fown) < 0)
+		goto err;
+
+	if (lseek(fd, mfe->pos, SEEK_SET) < 0) {
+		pr_perror("Can't restore file position of memfd id=%d", mfe->id);
+		goto err;
+	}
+
+	return fd;
+
+err:
+	if (fd >= 0)
+		close(fd);
+	return -1;
+}
+
+static int memfd_open_fe_fd(struct file_desc *fd, int *new_fd)
+{
+	int tmp;
+
+	tmp = memfd_open(fd, NULL);
+	if (tmp < 0)
+		return -1;
+	*new_fd = tmp;
+	return 0;
+}
+
+static struct file_desc_ops memfd_desc_ops = {
+	.type = FD_TYPES__MEMFD,
+	.open = memfd_open_fe_fd,
+};
+
+static int collect_one_memfd(void *o, ProtobufCMessage *msg, struct cr_img *i)
+{
+	struct memfd_info *info = o;
+
+	info->mfe = pb_msg(msg, MemfdFileEntry);
+	info->inode = memfd_alloc_inode(info->mfe->inode_id);
+	if (!info->inode)
+		return -1;
+
+	return file_desc_add(&info->d, info->mfe->id, &memfd_desc_ops);
+}
+
+struct collect_image_info memfd_cinfo = {
+	.fd_type = CR_FD_MEMFD_FILE,
+	.pb_type = PB_MEMFD_FILE,
+	.priv_size = sizeof(struct memfd_info),
+	.collect = collect_one_memfd,
+};

criu/shmem.c

@@ -23,6 +23,7 @@
 #include "types.h"
 #include "page.h"
 #include "util.h"
+#include "memfd.h"
 #include "protobuf.h"
 #include "images/pagemap.pb-c.h"
 
@@ -490,7 +491,7 @@ static int do_restore_shmem_content(void *addr, unsigned long size, unsigned lon
 	return ret;
 }
 
-static int restore_shmem_content(void *addr, struct shmem_info *si)
+int restore_shmem_content(void *addr, struct shmem_info *si)
 {
 	return do_restore_shmem_content(addr, si->size, si->shmid);
 }
@@ -500,6 +501,41 @@ int restore_sysv_shmem_content(void *addr, unsigned long size, unsigned long shm
 	return do_restore_shmem_content(addr, round_up(size, PAGE_SIZE), shmid);
 }
 
+int restore_memfd_shmem_content(int fd, unsigned long shmid, unsigned long size)
+{
+	void *addr = NULL;
+	int ret = 1;
+
+	if (size == 0)
+		return 0;
+
+	if (ftruncate(fd, size) < 0) {
+		pr_perror("Can't resize shmem 0x%lx size=%ld", shmid, size);
+		goto out;
+	}
+
+	addr = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
+	if (addr == MAP_FAILED) {
+		pr_perror("Can't mmap shmem 0x%lx size=%ld", shmid, size);
+		goto out;
+	}
+
+	/*
+	 * do_restore_shmem_content needs size to be page aligned.
+	 */
+	if (do_restore_shmem_content(addr, round_up(size, PAGE_SIZE), shmid) < 0) {
+		pr_err("Can't restore shmem content\n");
+		goto out;
+	}
+
+	ret = 0;
+
+out:
+	if (addr)
+		munmap(addr, size);
+	return ret;
+}
+
 static int open_shmem(int pid, struct vma_area *vma)
 {
 	VmaEntry *vi = vma->e;
@@ -532,7 +568,7 @@ static int open_shmem(int pid, struct vma_area *vma)
 
 	flags = MAP_SHARED;
 	if (kdat.has_memfd) {
-		f = syscall(SYS_memfd_create, "", 0);
+		f = memfd_create("", 0);
 		if (f < 0) {
 			pr_perror("Unable to create memfd");
 			goto err;
@@ -779,6 +815,32 @@ err:
 	return ret;
 }
 
+int dump_one_memfd_shmem(int fd, unsigned long shmid, unsigned long size)
+{
+	int ret = -1;
+	void *addr;
+	struct shmem_info si;
+
+	if (size == 0)
+		return 0;
+
+	memset(&si, 0, sizeof(si));
+	si.shmid = shmid;
+	si.size = size;
+
+	addr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (addr == MAP_FAILED) {
+		pr_perror("Can't mmap shmem 0x%lx", shmid);
+		goto err;
+	}
+
+	ret = do_dump_one_shmem(fd, addr, &si);
+
+	munmap(addr, size);
+err:
+	return ret;
+}
+
 int dump_one_sysv_shmem(void *addr, unsigned long size, unsigned long shmid)
 {
 	int fd, ret;

images/Makefile

@@ -63,6 +63,7 @@ proto-obj-y	+= sysctl.o
 proto-obj-y	+= autofs.o
 proto-obj-y	+= macvlan.o
 proto-obj-y	+= sit.o
+proto-obj-y	+= memfd.o
 
 CFLAGS		+= -iquote $(obj)/
 

images/fdinfo.proto

@@ -16,6 +16,7 @@ import "sk-unix.proto";
 import "fifo.proto";
 import "pipe.proto";
 import "tty.proto";
+import "memfd.proto";
 
 enum fd_types {
 	UND		= 0;
@@ -36,6 +37,7 @@ enum fd_types {
 	TUNF		= 15;
 	EXT		= 16;
 	TIMERFD		= 17;
+	MEMFD		= 18;
 
 	/* Any number above the real used. Not stored to image */
 	CTL_TTY		= 65534;
@@ -70,4 +72,5 @@ message file_entry {
 	optional fifo_entry		fifo	= 17;
 	optional pipe_entry		pipe	= 18;
 	optional tty_file_entry		tty	= 19;
+	optional memfd_file_entry	memfd	= 20;
 }

images/memfd.proto

@@ -0,0 +1,20 @@
+syntax = "proto2";
+
+import "opts.proto";
+import "fown.proto";
+
+message memfd_file_entry {
+	required uint32		id		= 1;
+	required uint32		flags		= 2 [(criu).flags = "rfile.flags"];
+	required uint64		pos		= 3;
+	required fown_entry	fown		= 4;
+	required uint32		inode_id	= 5;
+};
+
+message memfd_inode_entry {
+	required string		name		= 1;
+	required uint32		uid		= 2;
+	required uint32		gid		= 3;
+	required uint64		size		= 4;
+	required uint32		shmid		= 5;
+};

lib/py/images/images.py

@@ -522,6 +522,8 @@ handlers = {
     'AUTOFS': entry_handler(pb.autofs_entry),
     'FILES': entry_handler(pb.file_entry),
     'CPUINFO': entry_handler(pb.cpuinfo_entry),
+    'MEMFD_FILE': entry_handler(pb.memfd_file_entry),
+    'MEMFD_INODE': entry_handler(pb.memfd_inode_entry),
 }
 
 

scripts/feature-tests.mak

@@ -171,3 +171,14 @@ int main(int argc, char **argv)
 }
 
 endef
+
+define FEATURE_TEST_MEMFD_CREATE
+
+#include <sys/mman.h>
+#include <stddef.h>
+
+int main(void)
+{
+	return memfd_create(NULL, 0);
+}
+endef