c/r: move criu code to its own file

Trying to cage the beast that is lxccontainer.c.

Signed-off-by: Tycho Andersen <tycho.andersen@canonical.com>
Acked-by: Serge E. Hallyn <serge.hallyn@ubuntu.com>

src/lxc/Makefile.am

@@ -21,7 +21,8 @@ noinst_HEADERS = \
 	namespace.h \
 	start.h \
 	state.h \
-	utils.h
+	utils.h \
+	criu.h
 
 if IS_BIONIC
 noinst_HEADERS += \
@@ -75,6 +76,7 @@ liblxc_so_SOURCES = \
 	state.c state.h \
 	log.c log.h \
 	attach.c attach.h \
+	criu.c criu.h \
 	\
 	network.c network.h \
 	nl.c nl.h \

src/lxc/criu.c

@@ -0,0 +1,477 @@
+/*
+ * lxc: linux Container library
+ *
+ * Copyright © 2014-2015 Canonical Ltd.
+ *
+ * Authors:
+ * Tycho Andersen <tycho.andersen@canonical.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#define _GNU_SOURCE
+#include <assert.h>
+#include <linux/limits.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "config.h"
+
+#include "bdev.h"
+#include "cgroup.h"
+#include "conf.h"
+#include "criu.h"
+#include "log.h"
+#include "lxc.h"
+#include "lxclock.h"
+#include "network.h"
+#include "utils.h"
+
+lxc_log_define(lxc_criu, lxc);
+
+void exec_criu(struct criu_opts *opts)
+{
+	char **argv, log[PATH_MAX];
+	int static_args = 18, argc = 0, i, ret;
+	int netnr = 0;
+	struct lxc_list *it;
+
+	char buf[4096];
+	FILE *mnts = NULL;
+
+	/* The command line always looks like:
+	 * criu $(action) --tcp-established --file-locks --link-remap --force-irmap \
+	 * --manage-cgroups action-script foo.sh -D $(directory) \
+	 * -o $(directory)/$(action).log --ext-mount-map auto
+	 * --enable-external-sharing --enable-external-masters
+	 * +1 for final NULL */
+
+	if (strcmp(opts->action, "dump") == 0) {
+		/* -t pid */
+		static_args += 2;
+
+		/* --leave-running */
+		if (!opts->stop)
+			static_args++;
+	} else if (strcmp(opts->action, "restore") == 0) {
+		/* --root $(lxc_mount_point) --restore-detached
+		 * --restore-sibling --pidfile $foo --cgroup-root $foo */
+		static_args += 8;
+	} else {
+		return;
+	}
+
+	if (opts->verbose)
+		static_args++;
+
+	ret = snprintf(log, PATH_MAX, "%s/%s.log", opts->directory, opts->action);
+	if (ret < 0 || ret >= PATH_MAX) {
+		ERROR("logfile name too long\n");
+		return;
+	}
+
+	argv = malloc(static_args * sizeof(*argv));
+	if (!argv)
+		return;
+
+	memset(argv, 0, static_args * sizeof(*argv));
+
+#define DECLARE_ARG(arg) 					\
+	do {							\
+		if (arg == NULL) {				\
+			ERROR("Got NULL argument for criu");	\
+			goto err;				\
+		}						\
+		argv[argc++] = strdup(arg);			\
+		if (!argv[argc-1])				\
+			goto err;				\
+	} while (0)
+
+	argv[argc++] = on_path("criu", NULL);
+	if (!argv[argc-1]) {
+		ERROR("Couldn't find criu binary\n");
+		goto err;
+	}
+
+	DECLARE_ARG(opts->action);
+	DECLARE_ARG("--tcp-established");
+	DECLARE_ARG("--file-locks");
+	DECLARE_ARG("--link-remap");
+	DECLARE_ARG("--force-irmap");
+	DECLARE_ARG("--manage-cgroups");
+	DECLARE_ARG("--ext-mount-map");
+	DECLARE_ARG("auto");
+	DECLARE_ARG("--enable-external-sharing");
+	DECLARE_ARG("--enable-external-masters");
+	DECLARE_ARG("-D");
+	DECLARE_ARG(opts->directory);
+	DECLARE_ARG("-o");
+	DECLARE_ARG(log);
+
+	if (opts->verbose)
+		DECLARE_ARG("-vvvvvv");
+
+	if (strcmp(opts->action, "dump") == 0) {
+		char pid[32];
+
+		if (sprintf(pid, "%d", opts->c->init_pid(opts->c)) < 0)
+			goto err;
+
+		DECLARE_ARG("-t");
+		DECLARE_ARG(pid);
+		if (!opts->stop)
+			DECLARE_ARG("--leave-running");
+	} else if (strcmp(opts->action, "restore") == 0) {
+		void *m;
+		int additional;
+
+		DECLARE_ARG("--root");
+		DECLARE_ARG(opts->c->lxc_conf->rootfs.mount);
+		DECLARE_ARG("--restore-detached");
+		DECLARE_ARG("--restore-sibling");
+		DECLARE_ARG("--pidfile");
+		DECLARE_ARG(opts->pidfile);
+		DECLARE_ARG("--cgroup-root");
+		DECLARE_ARG(opts->cgroup_path);
+
+		additional = lxc_list_len(&opts->c->lxc_conf->network) * 2;
+
+		m = realloc(argv, (argc + additional + 1) * sizeof(*argv));	\
+		if (!m)								\
+			goto err;						\
+		argv = m;
+
+		lxc_list_for_each(it, &opts->c->lxc_conf->network) {
+			char eth[128], *veth;
+			struct lxc_netdev *n = it->elem;
+
+			if (n->name) {
+				if (strlen(n->name) >= sizeof(eth))
+					goto err;
+				strncpy(eth, n->name, sizeof(eth));
+			} else
+				sprintf(eth, "eth%d", netnr);
+
+			veth = n->priv.veth_attr.pair;
+
+			ret = snprintf(buf, sizeof(buf), "%s=%s@%s", eth, veth, n->link);
+			if (ret < 0 || ret >= sizeof(buf))
+				goto err;
+
+			DECLARE_ARG("--veth-pair");
+			DECLARE_ARG(buf);
+		}
+
+	}
+
+	argv[argc] = NULL;
+
+#undef DECLARE_ARG
+	execv(argv[0], argv);
+err:
+	if (mnts)
+		fclose(mnts);
+	for (i = 0; argv[i]; i++)
+		free(argv[i]);
+	free(argv);
+}
+
+/* Check and make sure the container has a configuration that we know CRIU can
+ * dump. */
+bool criu_ok(struct lxc_container *c)
+{
+	struct lxc_list *it;
+	bool found_deny_rule = false;
+
+	if (geteuid()) {
+		ERROR("Must be root to checkpoint\n");
+		return false;
+	}
+
+	/* We only know how to restore containers with veth networks. */
+	lxc_list_for_each(it, &c->lxc_conf->network) {
+		struct lxc_netdev *n = it->elem;
+		if (n->type != LXC_NET_VETH && n->type != LXC_NET_NONE) {
+			ERROR("Found network that is not VETH or NONE\n");
+			return false;
+		}
+	}
+
+	// These requirements come from http://criu.org/LXC
+	if (c->lxc_conf->console.path &&
+			strcmp(c->lxc_conf->console.path, "none") != 0) {
+		ERROR("lxc.console must be none\n");
+		return false;
+	}
+
+	if (c->lxc_conf->tty != 0) {
+		ERROR("lxc.tty must be 0\n");
+		return false;
+	}
+
+	lxc_list_for_each(it, &c->lxc_conf->cgroup) {
+		struct lxc_cgroup *cg = it->elem;
+		if (strcmp(cg->subsystem, "devices.deny") == 0 &&
+				strcmp(cg->value, "c 5:1 rwm") == 0) {
+
+			found_deny_rule = true;
+			break;
+		}
+	}
+
+	if (!found_deny_rule) {
+		ERROR("couldn't find devices.deny = c 5:1 rwm");
+		return false;
+	}
+
+	return true;
+}
+
+bool dump_net_info(struct lxc_container *c, char *directory)
+{
+	int netnr;
+	struct lxc_list *it;
+
+	netnr = 0;
+	lxc_list_for_each(it, &c->lxc_conf->network) {
+		char *veth = NULL, *bridge = NULL, veth_path[PATH_MAX], eth[128];
+		struct lxc_netdev *n = it->elem;
+		bool has_error = true;
+		int pret;
+
+		pret = snprintf(veth_path, PATH_MAX, "lxc.network.%d.veth.pair", netnr);
+		if (pret < 0 || pret >= PATH_MAX)
+			goto out;
+
+		veth = c->get_running_config_item(c, veth_path);
+		if (!veth) {
+			/* criu_ok() checks that all interfaces are
+			 * LXC_NET{VETH,NONE}, and VETHs should have this
+			 * config */
+			assert(n->type == LXC_NET_NONE);
+			break;
+		}
+
+		bridge = c->get_running_config_item(c, veth_path);
+		if (!bridge)
+			goto out;
+
+		pret = snprintf(veth_path, PATH_MAX, "%s/veth%d", directory, netnr);
+		if (pret < 0 || pret >= PATH_MAX || print_to_file(veth_path, veth) < 0)
+			goto out;
+
+		if (n->name) {
+			if (strlen(n->name) >= 128)
+				goto out;
+			strncpy(eth, n->name, 128);
+		} else
+			sprintf(eth, "eth%d", netnr);
+
+		has_error = false;
+out:
+		free(veth);
+		free(bridge);
+		if (has_error)
+			return false;
+	}
+
+	return true;
+}
+
+static bool restore_net_info(struct lxc_container *c)
+{
+	struct lxc_list *it;
+	bool has_error = true;
+
+	if (container_mem_lock(c))
+		return false;
+
+	lxc_list_for_each(it, &c->lxc_conf->network) {
+		struct lxc_netdev *netdev = it->elem;
+		char template[IFNAMSIZ];
+		snprintf(template, sizeof(template), "vethXXXXXX");
+
+		if (!netdev->priv.veth_attr.pair)
+			netdev->priv.veth_attr.pair = lxc_mkifname(template);
+
+		if (!netdev->priv.veth_attr.pair)
+			goto out_unlock;
+	}
+
+	has_error = false;
+
+out_unlock:
+	container_mem_unlock(c);
+	return !has_error;
+}
+
+void do_restore(struct lxc_container *c, int pipe, char *directory, bool verbose)
+{
+	pid_t pid;
+	char pidfile[L_tmpnam];
+	struct lxc_handler *handler;
+	int status;
+
+	if (!tmpnam(pidfile))
+		goto out;
+
+	handler = lxc_init(c->name, c->lxc_conf, c->config_path);
+	if (!handler)
+		goto out;
+
+	if (!cgroup_init(handler)) {
+		ERROR("failed initing cgroups");
+		goto out_fini_handler;
+	}
+
+	if (!cgroup_create(handler)) {
+		ERROR("failed creating groups");
+		goto out_fini_handler;
+	}
+
+	if (!restore_net_info(c)) {
+		ERROR("failed restoring network info");
+		goto out_fini_handler;
+	}
+
+	resolve_clone_flags(handler);
+
+	pid = fork();
+	if (pid < 0)
+		goto out_fini_handler;
+
+	if (pid == 0) {
+		struct criu_opts os;
+		struct lxc_rootfs *rootfs;
+
+		close(pipe);
+		pipe = -1;
+
+		if (unshare(CLONE_NEWNS))
+			goto out_fini_handler;
+
+		/* CRIU needs the lxc root bind mounted so that it is the root of some
+		 * mount. */
+		rootfs = &c->lxc_conf->rootfs;
+
+		if (rootfs_is_blockdev(c->lxc_conf)) {
+			if (do_rootfs_setup(c->lxc_conf, c->name, c->config_path) < 0)
+				goto out_fini_handler;
+		} else {
+			if (mkdir(rootfs->mount, 0755) < 0 && errno != EEXIST)
+				goto out_fini_handler;
+
+			if (mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL) < 0) {
+				SYSERROR("remount / to private failed");
+				goto out_fini_handler;
+			}
+
+			if (mount(rootfs->path, rootfs->mount, NULL, MS_BIND, NULL) < 0) {
+				rmdir(rootfs->mount);
+				goto out_fini_handler;
+			}
+		}
+
+		os.action = "restore";
+		os.directory = directory;
+		os.c = c;
+		os.pidfile = pidfile;
+		os.verbose = verbose;
+		os.cgroup_path = cgroup_canonical_path(handler);
+
+		/* exec_criu() returning is an error */
+		exec_criu(&os);
+		umount(rootfs->mount);
+		rmdir(rootfs->mount);
+		goto out_fini_handler;
+	} else {
+		int ret;
+		char title[2048];
+
+		pid_t w = waitpid(pid, &status, 0);
+		if (w == -1) {
+			SYSERROR("waitpid");
+			goto out_fini_handler;
+		}
+
+		ret = write(pipe, &status, sizeof(status));
+		close(pipe);
+		pipe = -1;
+
+		if (sizeof(status) != ret) {
+			SYSERROR("failed to write all of status");
+			goto out_fini_handler;
+		}
+
+		if (WIFEXITED(status)) {
+			if (WEXITSTATUS(status)) {
+				goto out_fini_handler;
+			} else {
+				int ret;
+				FILE *f = fopen(pidfile, "r");
+				if (!f) {
+					SYSERROR("couldn't read restore's init pidfile %s\n", pidfile);
+					goto out_fini_handler;
+				}
+
+				ret = fscanf(f, "%d", (int*) &handler->pid);
+				fclose(f);
+				if (ret != 1) {
+					ERROR("reading restore pid failed");
+					goto out_fini_handler;
+				}
+
+				if (lxc_set_state(c->name, handler, RUNNING))
+					goto out_fini_handler;
+			}
+		} else {
+			ERROR("CRIU was killed with signal %d\n", WTERMSIG(status));
+			goto out_fini_handler;
+		}
+
+		/*
+		 * See comment in lxcapi_start; we don't care if these
+		 * fail because it's just a beauty thing. We just
+		 * assign the return here to silence potential.
+		 */
+		ret = snprintf(title, sizeof(title), "[lxc monitor] %s %s", c->config_path, c->name);
+		ret = setproctitle(title);
+
+		ret = lxc_poll(c->name, handler);
+		if (ret)
+			lxc_abort(c->name, handler);
+		lxc_fini(c->name, handler);
+		exit(ret);
+	}
+
+out_fini_handler:
+	lxc_fini(c->name, handler);
+
+out:
+	if (pipe >= 0) {
+		status = 1;
+		if (write(pipe, &status, sizeof(status)) != sizeof(status)) {
+			SYSERROR("writing status failed");
+		}
+		close(pipe);
+	}
+
+	exit(1);
+}

src/lxc/criu.h

@@ -0,0 +1,70 @@
+/*
+ * lxc: linux Container library
+ *
+ * Copyright © 2014-2015 Canonical Ltd.
+ *
+ * Authors:
+ * Tycho Andersen <tycho.andersen@canonical.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __LXC_CRIU_H
+#define __LXC_CRIU_H
+
+#include <stdbool.h>
+
+#include <lxc/lxccontainer.h>
+
+// We require either the criu major/minor version, or the criu GITID if criu
+// was built from git.
+#define CRIU_VERSION 		"1.6"
+
+#define CRIU_GITID_VERSION	"1.5"
+#define CRIU_GITID_PATCHLEVEL	133
+
+struct criu_opts {
+	/* The type of criu invocation, one of "dump" or "restore" */
+	char *action;
+
+	/* The directory to pass to criu */
+	char *directory;
+
+	/* The container to dump */
+	struct lxc_container *c;
+
+	/* Enable criu verbose mode? */
+	bool verbose;
+
+	/* dump: stop the container or not after dumping? */
+	bool stop;
+
+	/* restore: the file to write the init process' pid into */
+	char *pidfile;
+	const char *cgroup_path;
+};
+
+void exec_criu(struct criu_opts *opts);
+
+/* Check and make sure the container has a configuration that we know CRIU can
+ * dump. */
+bool criu_ok(struct lxc_container *c);
+
+bool dump_net_info(struct lxc_container *c, char *directory);
+
+// do_restore never returns, the calling process is used as the
+// monitor process. do_restore calls exit() if it fails.
+void do_restore(struct lxc_container *c, int pipe, char *directory, bool verbose);
+
+#endif

src/lxc/lxccontainer.c

@@ -50,6 +50,7 @@
 #include "console.h"
 #include "cgroup.h"
 #include "commands.h"
+#include "criu.h"
 #include "log.h"
 #include "bdev.h"
 #include "utils.h"
@@ -3496,276 +3497,6 @@ static bool lxcapi_detach_interface(struct lxc_container *c, const char *ifname,
 	return true;
 }
 
-struct criu_opts {
-	/* The type of criu invocation, one of "dump" or "restore" */
-	char *action;
-
-	/* The directory to pass to criu */
-	char *directory;
-
-	/* The container to dump */
-	struct lxc_container *c;
-
-	/* Enable criu verbose mode? */
-	bool verbose;
-
-	/* dump: stop the container or not after dumping? */
-	bool stop;
-
-	/* restore: the file to write the init process' pid into */
-	char *pidfile;
-	const char *cgroup_path;
-};
-
-static void exec_criu(struct criu_opts *opts)
-{
-	char **argv, log[PATH_MAX];
-	int static_args = 18, argc = 0, i, ret;
-	int netnr = 0;
-	struct lxc_list *it;
-
-	char buf[4096];
-	FILE *mnts = NULL;
-
-	/* The command line always looks like:
-	 * criu $(action) --tcp-established --file-locks --link-remap --force-irmap \
-	 * --manage-cgroups action-script foo.sh -D $(directory) \
-	 * -o $(directory)/$(action).log --ext-mount-map auto
-	 * --enable-external-sharing --enable-external-masters
-	 * +1 for final NULL */
-
-	if (strcmp(opts->action, "dump") == 0) {
-		/* -t pid */
-		static_args += 2;
-
-		/* --leave-running */
-		if (!opts->stop)
-			static_args++;
-	} else if (strcmp(opts->action, "restore") == 0) {
-		/* --root $(lxc_mount_point) --restore-detached
-		 * --restore-sibling --pidfile $foo --cgroup-root $foo */
-		static_args += 8;
-	} else {
-		return;
-	}
-
-	if (opts->verbose)
-		static_args++;
-
-	ret = snprintf(log, PATH_MAX, "%s/%s.log", opts->directory, opts->action);
-	if (ret < 0 || ret >= PATH_MAX) {
-		ERROR("logfile name too long\n");
-		return;
-	}
-
-	argv = malloc(static_args * sizeof(*argv));
-	if (!argv)
-		return;
-
-	memset(argv, 0, static_args * sizeof(*argv));
-
-#define DECLARE_ARG(arg) 					\
-	do {							\
-		if (arg == NULL) {				\
-			ERROR("Got NULL argument for criu");	\
-			goto err;				\
-		}						\
-		argv[argc++] = strdup(arg);			\
-		if (!argv[argc-1])				\
-			goto err;				\
-	} while (0)
-
-	argv[argc++] = on_path("criu", NULL);
-	if (!argv[argc-1]) {
-		ERROR("Couldn't find criu binary\n");
-		goto err;
-	}
-
-	DECLARE_ARG(opts->action);
-	DECLARE_ARG("--tcp-established");
-	DECLARE_ARG("--file-locks");
-	DECLARE_ARG("--link-remap");
-	DECLARE_ARG("--force-irmap");
-	DECLARE_ARG("--manage-cgroups");
-	DECLARE_ARG("--ext-mount-map");
-	DECLARE_ARG("auto");
-	DECLARE_ARG("--enable-external-sharing");
-	DECLARE_ARG("--enable-external-masters");
-	DECLARE_ARG("-D");
-	DECLARE_ARG(opts->directory);
-	DECLARE_ARG("-o");
-	DECLARE_ARG(log);
-
-	if (opts->verbose)
-		DECLARE_ARG("-vvvvvv");
-
-	if (strcmp(opts->action, "dump") == 0) {
-		char pid[32];
-
-		if (sprintf(pid, "%d", lxcapi_init_pid(opts->c)) < 0)
-			goto err;
-
-		DECLARE_ARG("-t");
-		DECLARE_ARG(pid);
-		if (!opts->stop)
-			DECLARE_ARG("--leave-running");
-	} else if (strcmp(opts->action, "restore") == 0) {
-		void *m;
-		int additional;
-
-		DECLARE_ARG("--root");
-		DECLARE_ARG(opts->c->lxc_conf->rootfs.mount);
-		DECLARE_ARG("--restore-detached");
-		DECLARE_ARG("--restore-sibling");
-		DECLARE_ARG("--pidfile");
-		DECLARE_ARG(opts->pidfile);
-		DECLARE_ARG("--cgroup-root");
-		DECLARE_ARG(opts->cgroup_path);
-
-		additional = lxc_list_len(&opts->c->lxc_conf->network) * 2;
-
-		m = realloc(argv, (argc + additional + 1) * sizeof(*argv));	\
-		if (!m)								\
-			goto err;						\
-		argv = m;
-
-		lxc_list_for_each(it, &opts->c->lxc_conf->network) {
-			char eth[128], *veth;
-			struct lxc_netdev *n = it->elem;
-
-			if (n->name) {
-				if (strlen(n->name) >= sizeof(eth))
-					goto err;
-				strncpy(eth, n->name, sizeof(eth));
-			} else
-				sprintf(eth, "eth%d", netnr);
-
-			veth = n->priv.veth_attr.pair;
-
-			ret = snprintf(buf, sizeof(buf), "%s=%s@%s", eth, veth, n->link);
-			if (ret < 0 || ret >= sizeof(buf))
-				goto err;
-
-			DECLARE_ARG("--veth-pair");
-			DECLARE_ARG(buf);
-		}
-
-	}
-
-	argv[argc] = NULL;
-
-#undef DECLARE_ARG
-	execv(argv[0], argv);
-err:
-	if (mnts)
-		fclose(mnts);
-	for (i = 0; argv[i]; i++)
-		free(argv[i]);
-	free(argv);
-}
-
-/* Check and make sure the container has a configuration that we know CRIU can
- * dump. */
-static bool criu_ok(struct lxc_container *c)
-{
-	struct lxc_list *it;
-	bool found_deny_rule = false;
-
-	if (geteuid()) {
-		ERROR("Must be root to checkpoint\n");
-		return false;
-	}
-
-	/* We only know how to restore containers with veth networks. */
-	lxc_list_for_each(it, &c->lxc_conf->network) {
-		struct lxc_netdev *n = it->elem;
-		if (n->type != LXC_NET_VETH && n->type != LXC_NET_NONE) {
-			ERROR("Found network that is not VETH or NONE\n");
-			return false;
-		}
-	}
-
-	// These requirements come from http://criu.org/LXC
-	if (c->lxc_conf->console.path &&
-			strcmp(c->lxc_conf->console.path, "none") != 0) {
-		ERROR("lxc.console must be none\n");
-		return false;
-	}
-
-	if (c->lxc_conf->tty != 0) {
-		ERROR("lxc.tty must be 0\n");
-		return false;
-	}
-
-	lxc_list_for_each(it, &c->lxc_conf->cgroup) {
-		struct lxc_cgroup *cg = it->elem;
-		if (strcmp(cg->subsystem, "devices.deny") == 0 &&
-				strcmp(cg->value, "c 5:1 rwm") == 0) {
-
-			found_deny_rule = true;
-			break;
-		}
-	}
-
-	if (!found_deny_rule) {
-		ERROR("couldn't find devices.deny = c 5:1 rwm");
-		return false;
-	}
-
-	return true;
-}
-
-static bool dump_net_info(struct lxc_container *c, char *directory)
-{
-	int netnr;
-	struct lxc_list *it;
-
-	netnr = 0;
-	lxc_list_for_each(it, &c->lxc_conf->network) {
-		char *veth = NULL, *bridge = NULL, veth_path[PATH_MAX], eth[128];
-		struct lxc_netdev *n = it->elem;
-		bool has_error = true;
-		int pret;
-
-		pret = snprintf(veth_path, PATH_MAX, "lxc.network.%d.veth.pair", netnr);
-		if (pret < 0 || pret >= PATH_MAX)
-			goto out;
-
-		veth = lxcapi_get_running_config_item(c, veth_path);
-		if (!veth) {
-			/* criu_ok() checks that all interfaces are
-			 * LXC_NET{VETH,NONE}, and VETHs should have this
-			 * config */
-			assert(n->type == LXC_NET_NONE);
-			break;
-		}
-
-		bridge = lxcapi_get_running_config_item(c, veth_path);
-		if (!bridge)
-			goto out;
-
-		pret = snprintf(veth_path, PATH_MAX, "%s/veth%d", directory, netnr);
-		if (pret < 0 || pret >= PATH_MAX || print_to_file(veth_path, veth) < 0)
-			goto out;
-
-		if (n->name) {
-			if (strlen(n->name) >= 128)
-				goto out;
-			strncpy(eth, n->name, 128);
-		} else
-			sprintf(eth, "eth%d", netnr);
-
-		has_error = false;
-out:
-		free(veth);
-		free(bridge);
-		if (has_error)
-			return false;
-	}
-
-	return true;
-}
-
 static bool lxcapi_checkpoint(struct lxc_container *c, char *directory, bool stop, bool verbose)
 {
 	pid_t pid;
@@ -3811,189 +3542,6 @@ static bool lxcapi_checkpoint(struct lxc_container *c, char *directory, bool sto
 	}
 }
 
-static bool restore_net_info(struct lxc_container *c)
-{
-	struct lxc_list *it;
-	bool has_error = true;
-
-	if (container_mem_lock(c))
-		return false;
-
-	lxc_list_for_each(it, &c->lxc_conf->network) {
-		struct lxc_netdev *netdev = it->elem;
-		char template[IFNAMSIZ];
-		snprintf(template, sizeof(template), "vethXXXXXX");
-
-		if (!netdev->priv.veth_attr.pair)
-			netdev->priv.veth_attr.pair = lxc_mkifname(template);
-
-		if (!netdev->priv.veth_attr.pair)
-			goto out_unlock;
-	}
-
-	has_error = false;
-
-out_unlock:
-	container_mem_unlock(c);
-	return !has_error;
-}
-
-// do_restore never returns, the calling process is used as the
-// monitor process. do_restore calls exit() if it fails.
-static void do_restore(struct lxc_container *c, int pipe, char *directory, bool verbose)
-{
-	pid_t pid;
-	char pidfile[L_tmpnam];
-	struct lxc_handler *handler;
-	int status;
-
-	if (!tmpnam(pidfile))
-		goto out;
-
-	handler = lxc_init(c->name, c->lxc_conf, c->config_path);
-	if (!handler)
-		goto out;
-
-	if (!cgroup_init(handler)) {
-		ERROR("failed initing cgroups");
-		goto out_fini_handler;
-	}
-
-	if (!cgroup_create(handler)) {
-		ERROR("failed creating groups");
-		goto out_fini_handler;
-	}
-
-	if (!restore_net_info(c)) {
-		ERROR("failed restoring network info");
-		goto out_fini_handler;
-	}
-
-	resolve_clone_flags(handler);
-
-	pid = fork();
-	if (pid < 0)
-		goto out_fini_handler;
-
-	if (pid == 0) {
-		struct criu_opts os;
-		struct lxc_rootfs *rootfs;
-
-		close(pipe);
-		pipe = -1;
-
-		if (unshare(CLONE_NEWNS))
-			goto out_fini_handler;
-
-		/* CRIU needs the lxc root bind mounted so that it is the root of some
-		 * mount. */
-		rootfs = &c->lxc_conf->rootfs;
-
-		if (rootfs_is_blockdev(c->lxc_conf)) {
-			if (do_rootfs_setup(c->lxc_conf, c->name, c->config_path) < 0)
-				goto out_fini_handler;
-		} else {
-			if (mkdir(rootfs->mount, 0755) < 0 && errno != EEXIST)
-				goto out_fini_handler;
-
-			if (mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL) < 0) {
-				SYSERROR("remount / to private failed");
-				goto out_fini_handler;
-			}
-
-			if (mount(rootfs->path, rootfs->mount, NULL, MS_BIND, NULL) < 0) {
-				rmdir(rootfs->mount);
-				goto out_fini_handler;
-			}
-		}
-
-		os.action = "restore";
-		os.directory = directory;
-		os.c = c;
-		os.pidfile = pidfile;
-		os.verbose = verbose;
-		os.cgroup_path = cgroup_canonical_path(handler);
-
-		/* exec_criu() returning is an error */
-		exec_criu(&os);
-		umount(rootfs->mount);
-		rmdir(rootfs->mount);
-		goto out_fini_handler;
-	} else {
-		int ret;
-		char title[2048];
-
-		pid_t w = waitpid(pid, &status, 0);
-		if (w == -1) {
-			SYSERROR("waitpid");
-			goto out_fini_handler;
-		}
-
-		ret = write(pipe, &status, sizeof(status));
-		close(pipe);
-		pipe = -1;
-
-		if (sizeof(status) != ret) {
-			SYSERROR("failed to write all of status");
-			goto out_fini_handler;
-		}
-
-		if (WIFEXITED(status)) {
-			if (WEXITSTATUS(status)) {
-				goto out_fini_handler;
-			} else {
-				int ret;
-				FILE *f = fopen(pidfile, "r");
-				if (!f) {
-					SYSERROR("couldn't read restore's init pidfile %s\n", pidfile);
-					goto out_fini_handler;
-				}
-
-				ret = fscanf(f, "%d", (int*) &handler->pid);
-				fclose(f);
-				if (ret != 1) {
-					ERROR("reading restore pid failed");
-					goto out_fini_handler;
-				}
-
-				if (lxc_set_state(c->name, handler, RUNNING))
-					goto out_fini_handler;
-			}
-		} else {
-			ERROR("CRIU was killed with signal %d\n", WTERMSIG(status));
-			goto out_fini_handler;
-		}
-
-		/*
-		 * See comment in lxcapi_start; we don't care if these
-		 * fail because it's just a beauty thing. We just
-		 * assign the return here to silence potential.
-		 */
-		ret = snprintf(title, sizeof(title), "[lxc monitor] %s %s", c->config_path, c->name);
-		ret = setproctitle(title);
-
-		ret = lxc_poll(c->name, handler);
-		if (ret)
-			lxc_abort(c->name, handler);
-		lxc_fini(c->name, handler);
-		exit(ret);
-	}
-
-out_fini_handler:
-	lxc_fini(c->name, handler);
-
-out:
-	if (pipe >= 0) {
-		status = 1;
-		if (write(pipe, &status, sizeof(status)) != sizeof(status)) {
-			SYSERROR("writing status failed");
-		}
-		close(pipe);
-	}
-
-	exit(1);
-}
-
 static bool lxcapi_restore(struct lxc_container *c, char *directory, bool verbose)
 {
 	pid_t pid;