c/r: add a new ->migrate API call

This patch adds a new ->migrate API call with three commands:

MIGRATE_DUMP: this is basically just ->checkpoint()
MIGRATE_RESTORE: this is just ->restore()
MIGRATE_PRE_DUMP: this can be used to invoke criu's pre-dump command on the
    container.

A small addition to the (pre-)dump commands is the ability to specify a
previous partial dump directory, so that one can use a pre-dump of a
container.

Finally, this new API call uses a structure to pass options so that it can
be easily extended in the future (e.g. to CRIU's --leave-frozen option in
the future, for potentially smarter failure handling on restore).

v2: remember to flip the return code for legacy ->checkpoint and ->restore
    calls

Signed-off-by: Tycho Andersen <tycho.andersen@canonical.com>
Acked-by: Serge E. Hallyn <serge.hallyn@ubuntu.com>

src/lxc/criu.c

@@ -64,12 +64,16 @@ void exec_criu(struct criu_opts *opts)
 	 * --enable-fs hugetlbfs --enable-fs tracefs
 	 * +1 for final NULL */
 
-	if (strcmp(opts->action, "dump") == 0) {
+	if (strcmp(opts->action, "dump") == 0 || strcmp(opts->action, "pre-dump") == 0) {
 		/* -t pid --freeze-cgroup /lxc/ct */
 		static_args += 4;
 
-		/* --leave-running */
+		/* --prev-images-dir <path-to-directory-A-relative-to-B> */
-		if (!opts->stop)
+		if (opts->predump_dir)
+			static_args += 2;
+
+		/* --leave-running (only for final dump) */
+		if (strcmp(opts->action, "dump") == 0 && !opts->stop)
 			static_args++;
 	} else if (strcmp(opts->action, "restore") == 0) {
 		/* --root $(lxc_mount_point) --restore-detached
@@ -133,13 +137,12 @@ void exec_criu(struct criu_opts *opts)
 	if (opts->verbose)
 		DECLARE_ARG("-vvvvvv");
 
-	if (strcmp(opts->action, "dump") == 0) {
+	if (strcmp(opts->action, "dump") == 0 || strcmp(opts->action, "pre-dump") == 0) {
 		char pid[32], *freezer_relative;
 
 		if (sprintf(pid, "%d", opts->c->init_pid(opts->c)) < 0)
 			goto err;
 
-
 		DECLARE_ARG("-t");
 		DECLARE_ARG(pid);
 
@@ -158,7 +161,13 @@ void exec_criu(struct criu_opts *opts)
 		DECLARE_ARG("--freeze-cgroup");
 		DECLARE_ARG(log);
 
-		if (!opts->stop)
+		if (opts->predump_dir) {
+			DECLARE_ARG("--prev-images-dir");
+			DECLARE_ARG(opts->predump_dir);
+		}
+
+		/* only for final dump */
+		if (strcmp(opts->action, "dump") == 0 && !opts->stop)
 			DECLARE_ARG("--leave-running");
 	} else if (strcmp(opts->action, "restore") == 0) {
 		void *m;
@@ -402,6 +411,8 @@ out_unlock:
 	return !has_error;
 }
 
+// do_restore never returns, the calling process is used as the
+// monitor process. do_restore calls exit() if it fails.
 void do_restore(struct lxc_container *c, int pipe, char *directory, bool verbose)
 {
 	pid_t pid;
@@ -560,3 +571,135 @@ out:
 
 	exit(1);
 }
+
+/* do one of either predump or a regular dump */
+static bool do_dump(struct lxc_container *c, char *mode, char *directory,
+		    bool stop, bool verbose, char *predump_dir)
+{
+	pid_t pid;
+
+	if (!criu_ok(c))
+		return false;
+
+	if (mkdir_p(directory, 0700) < 0)
+		return false;
+
+	pid = fork();
+	if (pid < 0) {
+		SYSERROR("fork failed");
+		return false;
+	}
+
+	if (pid == 0) {
+		struct criu_opts os;
+
+		os.action = mode;
+		os.directory = directory;
+		os.c = c;
+		os.stop = stop;
+		os.verbose = verbose;
+		os.predump_dir = predump_dir;
+
+		/* exec_criu() returning is an error */
+		exec_criu(&os);
+		exit(1);
+	} else {
+		int status;
+		pid_t w = waitpid(pid, &status, 0);
+		if (w == -1) {
+			SYSERROR("waitpid");
+			return false;
+		}
+
+		if (WIFEXITED(status)) {
+			if (WEXITSTATUS(status)) {
+				ERROR("dump failed with %d\n", WEXITSTATUS(status));
+				return false;
+			}
+
+			return true;
+		} else if (WIFSIGNALED(status)) {
+			ERROR("dump signaled with %d\n", WTERMSIG(status));
+			return false;
+		} else {
+			ERROR("unknown dump exit %d\n", status);
+			return false;
+		}
+	}
+}
+
+bool pre_dump(struct lxc_container *c, char *directory, bool verbose, char *predump_dir)
+{
+	return do_dump(c, "pre-dump", directory, false, verbose, predump_dir);
+}
+
+bool dump(struct lxc_container *c, char *directory, bool stop, bool verbose, char *predump_dir)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = snprintf(path, sizeof(path), "%s/inventory.img", directory);
+	if (ret < 0 || ret >= sizeof(path))
+		return false;
+
+	if (access(path, F_OK) == 0) {
+		ERROR("please use a fresh directory for the dump directory\n");
+		return false;
+	}
+
+	return do_dump(c, "dump", directory, stop, verbose, predump_dir);
+}
+
+bool restore(struct lxc_container *c, char *directory, bool verbose)
+{
+	pid_t pid;
+	int status, nread;
+	int pipefd[2];
+
+	if (!criu_ok(c))
+		return false;
+
+	if (geteuid()) {
+		ERROR("Must be root to restore\n");
+		return false;
+	}
+
+	if (pipe(pipefd)) {
+		ERROR("failed to create pipe");
+		return false;
+	}
+
+	pid = fork();
+	if (pid < 0) {
+		close(pipefd[0]);
+		close(pipefd[1]);
+		return false;
+	}
+
+	if (pid == 0) {
+		close(pipefd[0]);
+		// this never returns
+		do_restore(c, pipefd[1], directory, verbose);
+	}
+
+	close(pipefd[1]);
+
+	nread = read(pipefd[0], &status, sizeof(status));
+	close(pipefd[0]);
+	if (sizeof(status) != nread) {
+		ERROR("reading status from pipe failed");
+		goto err_wait;
+	}
+
+	// If the criu process was killed or exited nonzero, wait() for the
+	// handler, since the restore process died. Otherwise, we don't need to
+	// wait, since the child becomes the monitor process.
+	if (!WIFEXITED(status) || WEXITSTATUS(status))
+		goto err_wait;
+	return true;
+
+err_wait:
+	if (wait_for_pid(pid))
+		ERROR("restore process died");
+	return false;
+}

src/lxc/criu.h

@@ -47,6 +47,9 @@ struct criu_opts {
 	/* Enable criu verbose mode? */
 	bool verbose;
 
+	/* (pre-)dump: a directory for the previous dump's images */
+	char *predump_dir;
+
 	/* dump: stop the container or not after dumping? */
 	bool stop;
 
@@ -61,8 +64,8 @@ void exec_criu(struct criu_opts *opts);
  * dump. */
 bool criu_ok(struct lxc_container *c);
 
-// do_restore never returns, the calling process is used as the
+bool pre_dump(struct lxc_container *c, char *directory, bool verbose, char *predump_dir);
-// monitor process. do_restore calls exit() if it fails.
+bool dump(struct lxc_container *c, char *directory, bool stop, bool verbose, char *predump_dir);
-void do_restore(struct lxc_container *c, int pipe, char *directory, bool verbose);
+bool restore(struct lxc_container *c, char *directory, bool verbose);
 
 #endif

src/lxc/lxccontainer.c

@@ -4004,112 +4004,69 @@ static bool do_lxcapi_detach_interface(struct lxc_container *c, const char *ifna
 
 WRAP_API_2(bool, lxcapi_detach_interface, const char *, const char *)
 
+static int do_lxcapi_migrate(struct lxc_container *c, unsigned int cmd,
+			     struct migrate_opts *opts, unsigned int size)
+{
+	int ret;
+
+	/* If the caller has a bigger (newer) struct migrate_opts, let's make
+	 * sure that the stuff on the end is zero, i.e. that they didn't ask us
+	 * to do anything special.
+	 */
+	if (size > sizeof(*opts)) {
+		unsigned char *addr;
+		unsigned char *end;
+
+		addr = (void *)opts + sizeof(*opts);
+		end  = (void *)opts + size;
+		for (; addr < end; addr++) {
+			if (*addr) {
+				return -E2BIG;
+			}
+		}
+	}
+
+	switch (cmd) {
+	case MIGRATE_PRE_DUMP:
+		ret = !pre_dump(c, opts->directory, opts->verbose, opts->predump_dir);
+		break;
+	case MIGRATE_DUMP:
+		ret = !dump(c, opts->directory, opts->stop, opts->verbose, opts->predump_dir);
+		break;
+	case MIGRATE_RESTORE:
+		ret = !restore(c, opts->directory, opts->verbose);
+		break;
+	default:
+		ERROR("invalid migrate command %u", cmd);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+WRAP_API_3(int, lxcapi_migrate, unsigned int, struct migrate_opts *, unsigned int)
+
 static bool do_lxcapi_checkpoint(struct lxc_container *c, char *directory, bool stop, bool verbose)
 {
-	pid_t pid;
+	struct migrate_opts opts = {
-	int status;
+		.directory = directory,
-	char path[PATH_MAX];
+		.stop = stop,
+		.verbose = verbose,
+	};
 
-	if (!criu_ok(c))
+	return !do_lxcapi_migrate(c, MIGRATE_DUMP, &opts, sizeof(opts));
-		return false;
-
-	if (mkdir(directory, 0700) < 0 && errno != EEXIST)
-		return false;
-
-	status = snprintf(path, sizeof(path), "%s/inventory.img", directory);
-	if (status < 0 || status >= sizeof(path))
-		return false;
-
-	if (access(path, F_OK) == 0) {
-		ERROR("please use a fresh directory for the dump directory\n");
-		return false;
-	}
-
-	pid = fork();
-	if (pid < 0)
-		return false;
-
-	if (pid == 0) {
-		struct criu_opts os;
-
-		os.action = "dump";
-		os.directory = directory;
-		os.c = c;
-		os.stop = stop;
-		os.verbose = verbose;
-
-		/* exec_criu() returning is an error */
-		exec_criu(&os);
-		exit(1);
-	} else {
-		pid_t w = waitpid(pid, &status, 0);
-		if (w == -1) {
-			SYSERROR("waitpid");
-			return false;
-		}
-
-		if (WIFEXITED(status)) {
-			return !WEXITSTATUS(status);
-		}
-
-		return false;
-	}
 }
 
 WRAP_API_3(bool, lxcapi_checkpoint, char *, bool, bool)
 
 static bool do_lxcapi_restore(struct lxc_container *c, char *directory, bool verbose)
 {
-	pid_t pid;
+	struct migrate_opts opts = {
-	int status, nread;
+		.directory = directory,
-	int pipefd[2];
+		.verbose = verbose,
+	};
 
-	if (!criu_ok(c))
+	return !do_lxcapi_migrate(c, MIGRATE_RESTORE, &opts, sizeof(opts));
-		return false;
-
-	if (geteuid()) {
-		ERROR("Must be root to restore\n");
-		return false;
-	}
-
-	if (pipe(pipefd)) {
-		ERROR("failed to create pipe");
-		return false;
-	}
-
-	pid = fork();
-	if (pid < 0) {
-		close(pipefd[0]);
-		close(pipefd[1]);
-		return false;
-	}
-
-	if (pid == 0) {
-		close(pipefd[0]);
-		// this never returns
-		do_restore(c, pipefd[1], directory, verbose);
-	}
-
-	close(pipefd[1]);
-
-	nread = read(pipefd[0], &status, sizeof(status));
-	close(pipefd[0]);
-	if (sizeof(status) != nread) {
-		ERROR("reading status from pipe failed");
-		goto err_wait;
-	}
-
-	// If the criu process was killed or exited nonzero, wait() for the
-	// handler, since the restore process died. Otherwise, we don't need to
-	// wait, since the child becomes the monitor process.
-	if (!WIFEXITED(status) || WEXITSTATUS(status))
-		goto err_wait;
-	return true;
-
-err_wait:
-	if (wait_for_pid(pid))
-		ERROR("restore process died");
-	return false;
 }
 
 WRAP_API_2(bool, lxcapi_restore, char *, bool)
@@ -4255,6 +4212,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 	c->detach_interface = lxcapi_detach_interface;
 	c->checkpoint = lxcapi_checkpoint;
 	c->restore = lxcapi_restore;
+	c->migrate = lxcapi_migrate;
 
 	return c;
 

src/lxc/lxccontainer.h

@@ -49,6 +49,8 @@ struct lxc_snapshot;
 
 struct lxc_lock;
 
+struct migrate_opts;
+
 /*!
  * An LXC container.
  *
@@ -812,6 +814,16 @@ struct lxc_container {
 	bool (*snapshot_destroy_all)(struct lxc_container *c);
 
 	/* Post LXC-1.1 additions */
+	/*!
+	 * \brief An API call to perform various migration operations
+	 *
+	 * \param cmd One of the MIGRATE_ contstants.
+	 * \param opts A migrate_opts struct filled with relevant options.
+	 * \param size The size of the migrate_opts struct, i.e. sizeof(struct migrate_opts).
+	 *
+	 * \return \c 0 on success, nonzero on failure.
+	 */
+	int (*migrate)(struct lxc_container *c, unsigned int cmd, struct migrate_opts *opts, unsigned int size);
 };
 
 /*!
@@ -848,6 +860,27 @@ struct bdev_specs {
 	char *dir; /*!< Directory path */
 };
 
+/*!
+ * \brief Commands for the migrate API call.
+ */
+enum {
+	MIGRATE_PRE_DUMP,
+	MIGRATE_DUMP,
+	MIGRATE_RESTORE,
+};
+
+/*!
+ * \brief Options for the migrate API call.
+ */
+struct migrate_opts {
+	/* new members should be added at the end */
+	char *directory;
+	bool verbose;
+
+	bool stop; /* stop the container after dump? */
+	char *predump_dir; /* relative to directory above */
+};
+
 /*!
  * \brief Create a new container.
  *