diff --git a/cgroup.c b/cgroup.c
index 6a222a8ab..299c2f4f0 100644
--- a/cgroup.c
+++ b/cgroup.c
@@ -619,7 +619,6 @@ static int move_in_cgroup(CgSetEntry *se)
 		}
 	}
 
-	close_service_fd(CGROUP_YARD);
 	return 0;
 }
 
diff --git a/cr-restore.c b/cr-restore.c
index 56ff90a04..c82cdd807 100644
--- a/cr-restore.c
+++ b/cr-restore.c
@@ -1388,6 +1388,14 @@ static int restore_task_with_children(void *_arg)
 	if (create_children_and_session())
 		goto err;
 
+	/*
+	 * This must be done after forking to allow child
+	 * to get the cgroup fd so it can move into the
+	 * correct /tasks file if it is in a different cgroup
+	 * set than its parent
+	 */
+	close_service_fd(CGROUP_YARD);
+
 	if (restore_task_mnt_ns(current))
 		goto err;