lxc/debian/patches/0005-separate-the-limiting-from-the-namespaced-cgroup-roo.patch

From 6adbaea0d07553932f4cd78b5530cd5291c3b41f Mon Sep 17 00:00:00 2001
From: Wolfgang Bumiller <w.bumiller@proxmox.com>
Date: Tue, 15 Nov 2016 09:20:24 +0100
Subject: [PATCH 5/9] separate the limiting from the namespaced cgroup root

When cgroup namespaces are enabled a privileged container
with mixed cgroups has full write access to its own root
cgroup effectively allowing it to overwrite values written
from the outside or configured via lxc.cgroup.*.

This patch causes an additional 'ns/' directory to be
created in all cgroups if cgroup namespaces and cgfsng are
being used in order to combat this.

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
---
 src/lxc/cgroups/cgfs.c      |  19 ++++++--
 src/lxc/cgroups/cgfsng.c    |  81 +++++++++++++++++++++++++++-----
 src/lxc/cgroups/cgmanager.c |  19 ++++++--
 src/lxc/cgroups/cgroup.c    |  16 +++----
 src/lxc/cgroups/cgroup.h    |  22 +++++----
 src/lxc/commands.c          | 112 ++++++++++++++++++++++++++++++--------------
 src/lxc/commands.h          |   2 +
 src/lxc/criu.c              |   4 +-
 src/lxc/start.c             |  21 +++++++--
 9 files changed, 219 insertions(+), 77 deletions(-)

diff --git a/src/lxc/cgroups/cgfs.c b/src/lxc/cgroups/cgfs.c
index 8499200..b78b78d 100644
--- a/src/lxc/cgroups/cgfs.c
+++ b/src/lxc/cgroups/cgfs.c
@@ -2383,12 +2383,15 @@ static void cgfs_destroy(void *hdata, struct lxc_conf *conf)
 	free(d);
 }
 
-static inline bool cgfs_create(void *hdata)
+static inline bool cgfs_create(void *hdata, bool inner)
 {
 	struct cgfs_data *d = hdata;
 	struct cgroup_process_info *i;
 	struct cgroup_meta_data *md;
 
+	if (inner)
+		return true;
+
 	if (!d)
 		return false;
 	md = d->meta;
@@ -2399,12 +2402,15 @@ static inline bool cgfs_create(void *hdata)
 	return true;
 }
 
-static inline bool cgfs_enter(void *hdata, pid_t pid)
+static inline bool cgfs_enter(void *hdata, pid_t pid, bool inner)
 {
 	struct cgfs_data *d = hdata;
 	struct cgroup_process_info *i;
 	int ret;
 
+	if (inner)
+		return true;
+
 	if (!d)
 		return false;
 	i = d->info;
@@ -2428,10 +2434,12 @@ static inline bool cgfs_create_legacy(void *hdata, pid_t pid)
 	return true;
 }
 
-static const char *cgfs_get_cgroup(void *hdata, const char *subsystem)
+static const char *cgfs_get_cgroup(void *hdata, const char *subsystem, bool inner)
 {
 	struct cgfs_data *d = hdata;
 
+	(void)inner;
+
 	if (!d)
 		return NULL;
 	return lxc_cgroup_get_hierarchy_path_data(subsystem, d);
@@ -2646,13 +2654,16 @@ static bool do_cgfs_chown(char *cgroup_path, struct lxc_conf *conf)
 	return true;
 }
 
-static bool cgfs_chown(void *hdata, struct lxc_conf *conf)
+static bool cgfs_chown(void *hdata, struct lxc_conf *conf, bool inner)
 {
 	struct cgfs_data *d = hdata;
 	struct cgroup_process_info *info_ptr;
 	char *cgpath;
 	bool r = true;
 
+	if (inner)
+		return true;
+
 	if (!d)
 		return false;
 
diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
index 2b772e2..f7df3cf 100644
--- a/src/lxc/cgroups/cgfsng.c
+++ b/src/lxc/cgroups/cgfsng.c
@@ -72,6 +72,7 @@ struct hierarchy {
 	char *mountpoint;
 	char *base_cgroup;
 	char *fullcgpath;
+	char *innercgpath;
 };
 
 /*
@@ -814,6 +815,7 @@ static void add_controller(char **clist, char *mountpoint, char *base_cgroup)
 	new->mountpoint = mountpoint;
 	new->base_cgroup = base_cgroup;
 	new->fullcgpath = NULL;
+	new->innercgpath = false;
 
 	newentry = append_null_to_list((void ***)&hierarchies);
 	hierarchies[newentry] = new;
@@ -1286,6 +1288,8 @@ static void cgfsng_destroy(void *hdata, struct lxc_conf *conf)
 				free(h->fullcgpath);
 				h->fullcgpath = NULL;
 			}
+			free(h->innercgpath);
+			h->innercgpath = NULL;
 		}
 	}
 
@@ -1299,18 +1303,25 @@ struct cgroup_ops *cgfsng_ops_init(void)
 	return &cgfsng_ops;
 }
 
-static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname)
+static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname, bool inner)
 {
-	h->fullcgpath = must_make_path(h->mountpoint, h->base_cgroup, cgname, NULL);
-	if (dir_exists(h->fullcgpath)) { // it must not already exist
-		ERROR("Path \"%s\" already existed.", h->fullcgpath);
+	char *path;
+	if (inner) {
+		path = must_make_path(h->fullcgpath, CGROUP_NAMESPACE_SUBDIR, NULL);
+		h->innercgpath = path;
+	} else {
+		path = must_make_path(h->mountpoint, h->base_cgroup, cgname, NULL);
+		h->fullcgpath = path;
+	}
+	if (dir_exists(path)) { // it must not already exist
+		ERROR("Path \"%s\" already existed.", path);
 		return false;
 	}
-	if (!handle_cpuset_hierarchy(h, cgname)) {
+	if (!inner && !handle_cpuset_hierarchy(h, cgname)) {
 		ERROR("Failed to handle cgroupfs v1 cpuset controller.");
 		return false;
 	}
-	return mkdir_p(h->fullcgpath, 0755) == 0;
+	return mkdir_p(path, 0755) == 0;
 }
 
 static void remove_path_for_hierarchy(struct hierarchy *h, char *cgname)
@@ -1325,7 +1336,8 @@ static void remove_path_for_hierarchy(struct hierarchy *h, char *cgname)
  * Try to create the same cgroup in all hierarchies.
  * Start with cgroup_pattern; next cgroup_pattern-1, -2, ..., -999
  */
-static inline bool cgfsng_create(void *hdata)
+static inline bool cgfsng_create_inner(struct cgfsng_handler_data*);
+static inline bool cgfsng_create(void *hdata, bool inner)
 {
 	struct cgfsng_handler_data *d = hdata;
 	char *tmp, *cgname, *offset;
@@ -1335,9 +1347,15 @@ static inline bool cgfsng_create(void *hdata)
 	if (!d)
 		return false;
 	if (d->container_cgroup) {
+		if (inner)
+			return cgfsng_create_inner(d);
 		WARN("cgfsng_create called a second time");
 		return false;
 	}
+	if (inner) {
+		ERROR("cgfsng_create called twice for innner cgroup");
+		return false;
+	}
 
 	tmp = lxc_string_replace("%n", d->name, d->cgroup_pattern);
 	if (!tmp) {
@@ -1358,7 +1376,7 @@ again:
 	if (idx)
 		snprintf(offset, 5, "-%d", idx);
 	for (i = 0; hierarchies[i]; i++) {
-		if (!create_path_for_hierarchy(hierarchies[i], cgname)) {
+		if (!create_path_for_hierarchy(hierarchies[i], cgname, false)) {
 			int j;
 			SYSERROR("Failed to create %s: %s", hierarchies[i]->fullcgpath, strerror(errno));
 			free(hierarchies[i]->fullcgpath);
@@ -1378,7 +1396,24 @@ out_free:
 	return false;
 }
 
-static bool cgfsng_enter(void *hdata, pid_t pid)
+static inline bool cgfsng_create_inner(struct cgfsng_handler_data *d)
+{
+	size_t i;
+	bool ret = true;
+	char *cgname = must_make_path(d->container_cgroup, CGROUP_NAMESPACE_SUBDIR, NULL);
+	for (i = 0; hierarchies[i]; i++) {
+		if (!create_path_for_hierarchy(hierarchies[i], cgname, true)) {
+			SYSERROR("Failed to create %s namespace subdirectory: %s", hierarchies[i]->fullcgpath, strerror(errno));
+			ret = false;
+			break;
+		}
+	}
+	free(cgname);
+	return ret;
+}
+
+
+static bool cgfsng_enter(void *hdata, pid_t pid, bool inner)
 {
 	char pidstr[25];
 	int i, len;
@@ -1388,7 +1423,13 @@ static bool cgfsng_enter(void *hdata, pid_t pid)
 		return false;
 
 	for (i = 0; hierarchies[i]; i++) {
-		char *fullpath = must_make_path(hierarchies[i]->fullcgpath,
+		char *fullpath;
+		if (inner)
+			fullpath = must_make_path(hierarchies[i]->fullcgpath,
+						CGROUP_NAMESPACE_SUBDIR,
+						"cgroup.procs", NULL);
+		else
+			fullpath = must_make_path(hierarchies[i]->fullcgpath,
 						"cgroup.procs", NULL);
 		if (lxc_write_to_file(fullpath, pidstr, len, false) != 0) {
 			SYSERROR("Failed to enter %s", fullpath);
@@ -1404,6 +1445,7 @@ static bool cgfsng_enter(void *hdata, pid_t pid)
 struct chown_data {
 	struct cgfsng_handler_data *d;
 	uid_t origuid; // target uid in parent namespace
+	bool inner;
 };
 
 /*
@@ -1432,13 +1474,20 @@ static int chown_cgroup_wrapper(void *data)
 	for (i = 0; hierarchies[i]; i++) {
 		char *fullpath, *path = hierarchies[i]->fullcgpath;
 
+		if (arg->inner)
+			path = must_make_path(path, CGROUP_NAMESPACE_SUBDIR, NULL);
+
 		if (chown(path, destuid, 0) < 0) {
 			SYSERROR("Error chowning %s to %d", path, (int) destuid);
+			if (arg->inner)
+				free(path);
 			return -1;
 		}
 
 		if (chmod(path, 0775) < 0) {
 			SYSERROR("Error chmoding %s", path);
+			if (arg->inner)
+				free(path);
 			return -1;
 		}
 
@@ -1462,12 +1511,14 @@ static int chown_cgroup_wrapper(void *data)
 		if (chmod(fullpath, 0664) < 0)
 			WARN("Error chmoding %s: %m", path);
 		free(fullpath);
+
+		free(path);
 	}
 
 	return 0;
 }
 
-static bool cgfsns_chown(void *hdata, struct lxc_conf *conf)
+static bool cgfsns_chown(void *hdata, struct lxc_conf *conf, bool inner)
 {
 	struct cgfsng_handler_data *d = hdata;
 	struct chown_data wrap;
@@ -1480,6 +1531,7 @@ static bool cgfsns_chown(void *hdata, struct lxc_conf *conf)
 
 	wrap.d = d;
 	wrap.origuid = geteuid();
+	wrap.inner = inner;
 
 	if (userns_exec_1(conf, chown_cgroup_wrapper, &wrap) < 0) {
 		ERROR("Error requesting cgroup chown in new namespace");
@@ -1774,12 +1826,15 @@ static bool cgfsng_unfreeze(void *hdata)
 	return true;
 }
 
-static const char *cgfsng_get_cgroup(void *hdata, const char *subsystem)
+static const char *cgfsng_get_cgroup(void *hdata, const char *subsystem, bool inner)
 {
 	struct hierarchy *h = get_hierarchy(subsystem);
 	if (!h)
 		return NULL;
 
+	if (inner && h->innercgpath)
+		return h->innercgpath + strlen(h->mountpoint);
+
 	return h->fullcgpath ? h->fullcgpath + strlen(h->mountpoint) : NULL;
 }
 
@@ -1814,7 +1869,7 @@ static bool cgfsng_attach(const char *name, const char *lxcpath, pid_t pid)
 		char *path, *fullpath;
 		struct hierarchy *h = hierarchies[i];
 
-		path = lxc_cmd_get_cgroup_path(name, lxcpath, h->controllers[0]);
+		path = lxc_cmd_get_attach_cgroup_path(name, lxcpath, h->controllers[0]);
 		if (!path) // not running
 			continue;
 
diff --git a/src/lxc/cgroups/cgmanager.c b/src/lxc/cgroups/cgmanager.c
index f2756b0..ac966b6 100644
--- a/src/lxc/cgroups/cgmanager.c
+++ b/src/lxc/cgroups/cgmanager.c
@@ -609,7 +609,7 @@ static inline void cleanup_cgroups(char *path)
 		cgm_remove_cgroup(slist[i], path);
 }
 
-static inline bool cgm_create(void *hdata)
+static inline bool cgm_create(void *hdata, bool inner)
 {
 	struct cgm_data *d = hdata;
 	char **slist = subsystems;
@@ -617,6 +617,9 @@ static inline bool cgm_create(void *hdata)
 	int32_t existed;
 	char result[MAXPATHLEN], *tmp, *cgroup_path;
 
+	if (inner)
+		return true;
+
 	if (!d)
 		return false;
 // XXX we should send a hint to the cgmanager that when these
@@ -709,13 +712,16 @@ static bool lxc_cgmanager_enter(pid_t pid, const char *controller,
 	return true;
 }
 
-static inline bool cgm_enter(void *hdata, pid_t pid)
+static inline bool cgm_enter(void *hdata, pid_t pid, bool inner)
 {
 	struct cgm_data *d = hdata;
 	char **slist = subsystems;
 	bool ret = false;
 	int i;
 
+	if (inner)
+		return true;
+
 	if (!d || !d->cgroup_path)
 		return false;
 
@@ -737,10 +743,12 @@ out:
 	return ret;
 }
 
-static const char *cgm_get_cgroup(void *hdata, const char *subsystem)
+static const char *cgm_get_cgroup(void *hdata, const char *subsystem, bool inner)
 {
 	struct cgm_data *d = hdata;
 
+	(void)inner;
+
 	if (!d || !d->cgroup_path)
 		return NULL;
 	return d->cgroup_path;
@@ -1541,10 +1549,13 @@ out:
 	return ret;
 }
 
-static bool cgm_chown(void *hdata, struct lxc_conf *conf)
+static bool cgm_chown(void *hdata, struct lxc_conf *conf, bool inner)
 {
 	struct cgm_data *d = hdata;
 
+	if (inner)
+		return true;
+
 	if (!d || !d->cgroup_path)
 		return false;
 	if (!cgm_dbus_connect()) {
diff --git a/src/lxc/cgroups/cgroup.c b/src/lxc/cgroups/cgroup.c
index 78472d4..4d26e72 100644
--- a/src/lxc/cgroups/cgroup.c
+++ b/src/lxc/cgroups/cgroup.c
@@ -80,10 +80,10 @@ void cgroup_destroy(struct lxc_handler *handler)
 }
 
 /* Create the container cgroups for all requested controllers */
-bool cgroup_create(struct lxc_handler *handler)
+bool cgroup_create(struct lxc_handler *handler, bool inner)
 {
 	if (ops)
-		return ops->create(handler->cgroup_data);
+		return ops->create(handler->cgroup_data, inner);
 	return false;
 }
 
@@ -91,10 +91,10 @@ bool cgroup_create(struct lxc_handler *handler)
  * Enter the container init into its new cgroups for all
  * requested controllers
  */
-bool cgroup_enter(struct lxc_handler *handler)
+bool cgroup_enter(struct lxc_handler *handler, bool inner)
 {
 	if (ops)
-		return ops->enter(handler->cgroup_data, handler->pid);
+		return ops->enter(handler->cgroup_data, handler->pid, inner);
 	return false;
 }
 
@@ -105,10 +105,10 @@ bool cgroup_create_legacy(struct lxc_handler *handler)
 	return true;
 }
 
-const char *cgroup_get_cgroup(struct lxc_handler *handler, const char *subsystem)
+const char *cgroup_get_cgroup(struct lxc_handler *handler, const char *subsystem, bool inner)
 {
 	if (ops)
-		return ops->get_cgroup(handler->cgroup_data, subsystem);
+		return ops->get_cgroup(handler->cgroup_data, subsystem, inner);
 	return NULL;
 }
 
@@ -150,10 +150,10 @@ bool cgroup_setup_limits(struct lxc_handler *handler, bool with_devices)
 	return false;
 }
 
-bool cgroup_chown(struct lxc_handler *handler)
+bool cgroup_chown(struct lxc_handler *handler, bool inner)
 {
 	if (ops && ops->chown)
-		return ops->chown(handler->cgroup_data, handler->conf);
+		return ops->chown(handler->cgroup_data, handler->conf, inner);
 	return true;
 }
 
diff --git a/src/lxc/cgroups/cgroup.h b/src/lxc/cgroups/cgroup.h
index 11b251e..f36c6f0 100644
--- a/src/lxc/cgroups/cgroup.h
+++ b/src/lxc/cgroups/cgroup.h
@@ -28,6 +28,12 @@
 #include <stddef.h>
 #include <sys/types.h>
 
+/* When lxc.cgroup.protect_limits is in effect the container's cgroup namespace
+ * will be moved into an additional subdirectory "cgns/" inside the cgroup in
+ * order to prevent it from accessing the outer limiting cgroup.
+ */
+#define CGROUP_NAMESPACE_SUBDIR "cgns"
+
 struct lxc_handler;
 struct lxc_conf;
 struct lxc_list;
@@ -43,10 +49,10 @@ struct cgroup_ops {
 
 	void *(*init)(const char *name);
 	void (*destroy)(void *hdata, struct lxc_conf *conf);
-	bool (*create)(void *hdata);
-	bool (*enter)(void *hdata, pid_t pid);
+	bool (*create)(void *hdata, bool inner);
+	bool (*enter)(void *hdata, pid_t pid, bool inner);
 	bool (*create_legacy)(void *hdata, pid_t pid);
-	const char *(*get_cgroup)(void *hdata, const char *subsystem);
+	const char *(*get_cgroup)(void *hdata, const char *subsystem, bool inner);
 	bool (*escape)();
 	int (*num_hierarchies)();
 	bool (*get_hierarchies)(int n, char ***out);
@@ -54,7 +60,7 @@ struct cgroup_ops {
 	int (*get)(const char *filename, char *value, size_t len, const char *name, const char *lxcpath);
 	bool (*unfreeze)(void *hdata);
 	bool (*setup_limits)(void *hdata, struct lxc_list *cgroup_conf, bool with_devices);
-	bool (*chown)(void *hdata, struct lxc_conf *conf);
+	bool (*chown)(void *hdata, struct lxc_conf *conf, bool inner);
 	bool (*attach)(const char *name, const char *lxcpath, pid_t pid);
 	bool (*mount_cgroup)(void *hdata, const char *root, int type);
 	int (*nrtasks)(void *hdata);
@@ -66,14 +72,14 @@ extern bool cgroup_attach(const char *name, const char *lxcpath, pid_t pid);
 extern bool cgroup_mount(const char *root, struct lxc_handler *handler, int type);
 extern void cgroup_destroy(struct lxc_handler *handler);
 extern bool cgroup_init(struct lxc_handler *handler);
-extern bool cgroup_create(struct lxc_handler *handler);
+extern bool cgroup_create(struct lxc_handler *handler, bool inner);
 extern bool cgroup_setup_limits(struct lxc_handler *handler, bool with_devices);
-extern bool cgroup_chown(struct lxc_handler *handler);
-extern bool cgroup_enter(struct lxc_handler *handler);
+extern bool cgroup_chown(struct lxc_handler *handler, bool inner);
+extern bool cgroup_enter(struct lxc_handler *handler, bool inner);
 extern void cgroup_cleanup(struct lxc_handler *handler);
 extern bool cgroup_create_legacy(struct lxc_handler *handler);
 extern int cgroup_nrtasks(struct lxc_handler *handler);
-extern const char *cgroup_get_cgroup(struct lxc_handler *handler, const char *subsystem);
+extern const char *cgroup_get_cgroup(struct lxc_handler *handler, const char *subsystem, bool inner);
 extern bool cgroup_escape();
 extern int cgroup_num_hierarchies();
 extern bool cgroup_get_hierarchies(int i, char ***out);
diff --git a/src/lxc/commands.c b/src/lxc/commands.c
index b17879b..5ef682f 100644
--- a/src/lxc/commands.c
+++ b/src/lxc/commands.c
@@ -128,15 +128,15 @@ static int fill_sock_name(char *path, int len, const char *name,
 static const char *lxc_cmd_str(lxc_cmd_t cmd)
 {
 	static const char * const cmdname[LXC_CMD_MAX] = {
-		[LXC_CMD_CONSOLE]         = "console",
-		[LXC_CMD_STOP]            = "stop",
-		[LXC_CMD_GET_STATE]       = "get_state",
-		[LXC_CMD_GET_INIT_PID]    = "get_init_pid",
-		[LXC_CMD_GET_CLONE_FLAGS] = "get_clone_flags",
-		[LXC_CMD_GET_CGROUP]      = "get_cgroup",
-		[LXC_CMD_GET_CONFIG_ITEM] = "get_config_item",
-		[LXC_CMD_GET_NAME]        = "get_name",
-		[LXC_CMD_GET_LXCPATH]     = "get_lxcpath",
+		[LXC_CMD_CONSOLE]           = "console",
+		[LXC_CMD_STOP]              = "stop",
+		[LXC_CMD_GET_STATE]         = "get_state",
+		[LXC_CMD_GET_INIT_PID]      = "get_init_pid",
+		[LXC_CMD_GET_CLONE_FLAGS]   = "get_clone_flags",
+		[LXC_CMD_GET_CGROUP]        = "get_cgroup",
+		[LXC_CMD_GET_CONFIG_ITEM]   = "get_config_item",
+		[LXC_CMD_GET_NAME]          = "get_name",
+		[LXC_CMD_GET_LXCPATH]       = "get_lxcpath",
 	};
 
 	if (cmd >= LXC_CMD_MAX)
@@ -429,30 +429,28 @@ static int lxc_cmd_get_clone_flags_callback(int fd, struct lxc_cmd_req *req,
 	return lxc_cmd_rsp_send(fd, &rsp);
 }
 
-/*
- * lxc_cmd_get_cgroup_path: Calculate a container's cgroup path for a
- * particular subsystem. This is the cgroup path relative to the root
- * of the cgroup filesystem.
- *
- * @name      : name of container to connect to
- * @lxcpath   : the lxcpath in which the container is running
- * @subsystem : the subsystem being asked about
- *
- * Returns the path on success, NULL on failure. The caller must free() the
- * returned path.
- */
-char *lxc_cmd_get_cgroup_path(const char *name, const char *lxcpath,
-	const char *subsystem)
+static char *do_lxc_cmd_get_cgroup_path(const char *name, const char *lxcpath,
+	const char *subsystem, bool inner)
 {
 	int ret, stopped;
+	size_t subsyslen = strlen(subsystem);
+
 	struct lxc_cmd_rr cmd = {
 		.req = {
 			.cmd = LXC_CMD_GET_CGROUP,
-			.datalen = strlen(subsystem)+1,
+			.datalen = subsyslen+1,
 			.data = subsystem,
 		},
 	};
 
+	if (inner) {
+		char *data = alloca(subsyslen+2);
+		memcpy(data, subsystem, subsyslen+1);
+		data[subsyslen+1] = 1;
+		cmd.req.datalen = subsyslen+2,
+		cmd.req.data = data;
+	}
+
 	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
 	if (ret < 0)
 		return NULL;
@@ -471,16 +469,42 @@ char *lxc_cmd_get_cgroup_path(const char *name, const char *lxcpath,
 	return cmd.rsp.data;
 }
 
+/*
+ * lxc_cmd_get_cgroup_path: Calculate a container's cgroup path for a
+ * particular subsystem. This is the cgroup path relative to the root
+ * of the cgroup filesystem.
+ *
+ * @name      : name of container to connect to
+ * @lxcpath   : the lxcpath in which the container is running
+ * @subsystem : the subsystem being asked about
+ *
+ * Returns the path on success, NULL on failure. The caller must free() the
+ * returned path.
+ */
+char *lxc_cmd_get_cgroup_path(const char *name, const char *lxcpath,
+	const char *subsystem)
+{
+	return do_lxc_cmd_get_cgroup_path(name, lxcpath, subsystem, false);
+}
+
 static int lxc_cmd_get_cgroup_callback(int fd, struct lxc_cmd_req *req,
 				       struct lxc_handler *handler)
 {
 	struct lxc_cmd_rsp rsp;
 	const char *path;
+	const char *subsystem;
+	size_t subsyslen;
+	bool inner = false;
 
 	if (req->datalen < 1)
 		return -1;
 
-	path = cgroup_get_cgroup(handler, req->data);
+	subsystem = req->data;
+	subsyslen = strlen(subsystem);
+	if (req->datalen == subsyslen+2)
+		inner = (subsystem[subsyslen+1] == 1);
+
+	path = cgroup_get_cgroup(handler, req->data, inner);
 	if (!path)
 		return -1;
 	rsp.datalen = strlen(path) + 1,
@@ -491,6 +515,24 @@ static int lxc_cmd_get_cgroup_callback(int fd, struct lxc_cmd_req *req,
 }
 
 /*
+ * lxc_cmd_get_attach_cgroup_path: Calculate a container's inner cgroup path
+ * for a particular subsystem. This is the cgroup path relative to the root
+ * of the cgroup filesystem.
+ *
+ * @name      : name of container to connect to
+ * @lxcpath   : the lxcpath in which the container is running
+ * @subsystem : the subsystem being asked about
+ *
+ * Returns the path on success, NULL on failure. The caller must free() the
+ * returned path.
+ */
+char *lxc_cmd_get_attach_cgroup_path(const char *name, const char *lxcpath,
+	const char *subsystem)
+{
+	return do_lxc_cmd_get_cgroup_path(name, lxcpath, subsystem, true);
+}
+
+/*
  * lxc_cmd_get_config_item: Get config item the running container
  *
  * @name     : name of container to connect to
@@ -841,16 +883,16 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
 	typedef int (*callback)(int, struct lxc_cmd_req *, struct lxc_handler *);
 
 	callback cb[LXC_CMD_MAX] = {
-		[LXC_CMD_CONSOLE]         = lxc_cmd_console_callback,
-		[LXC_CMD_CONSOLE_WINCH]   = lxc_cmd_console_winch_callback,
-		[LXC_CMD_STOP]            = lxc_cmd_stop_callback,
-		[LXC_CMD_GET_STATE]       = lxc_cmd_get_state_callback,
-		[LXC_CMD_GET_INIT_PID]    = lxc_cmd_get_init_pid_callback,
-		[LXC_CMD_GET_CLONE_FLAGS] = lxc_cmd_get_clone_flags_callback,
-		[LXC_CMD_GET_CGROUP]      = lxc_cmd_get_cgroup_callback,
-		[LXC_CMD_GET_CONFIG_ITEM] = lxc_cmd_get_config_item_callback,
-		[LXC_CMD_GET_NAME]        = lxc_cmd_get_name_callback,
-		[LXC_CMD_GET_LXCPATH]     = lxc_cmd_get_lxcpath_callback,
+		[LXC_CMD_CONSOLE]           = lxc_cmd_console_callback,
+		[LXC_CMD_CONSOLE_WINCH]     = lxc_cmd_console_winch_callback,
+		[LXC_CMD_STOP]              = lxc_cmd_stop_callback,
+		[LXC_CMD_GET_STATE]         = lxc_cmd_get_state_callback,
+		[LXC_CMD_GET_INIT_PID]      = lxc_cmd_get_init_pid_callback,
+		[LXC_CMD_GET_CLONE_FLAGS]   = lxc_cmd_get_clone_flags_callback,
+		[LXC_CMD_GET_CGROUP]        = lxc_cmd_get_cgroup_callback,
+		[LXC_CMD_GET_CONFIG_ITEM]   = lxc_cmd_get_config_item_callback,
+		[LXC_CMD_GET_NAME]          = lxc_cmd_get_name_callback,
+		[LXC_CMD_GET_LXCPATH]       = lxc_cmd_get_lxcpath_callback,
 	};
 
 	if (req->cmd >= LXC_CMD_MAX) {
diff --git a/src/lxc/commands.h b/src/lxc/commands.h
index 184eefa..6430b33 100644
--- a/src/lxc/commands.h
+++ b/src/lxc/commands.h
@@ -77,6 +77,8 @@ extern int lxc_cmd_console(const char *name, int *ttynum, int *fd,
  */
 extern char *lxc_cmd_get_cgroup_path(const char *name, const char *lxcpath,
 			const char *subsystem);
+extern char *lxc_cmd_get_attach_cgroup_path(const char *name,
+			const char *lxcpath, const char *subsystem);
 extern int lxc_cmd_get_clone_flags(const char *name, const char *lxcpath);
 extern char *lxc_cmd_get_config_item(const char *name, const char *item, const char *lxcpath);
 extern char *lxc_cmd_get_name(const char *hashed_sock);
diff --git a/src/lxc/criu.c b/src/lxc/criu.c
index 8a0702f..5843f97 100644
--- a/src/lxc/criu.c
+++ b/src/lxc/criu.c
@@ -283,7 +283,7 @@ static void exec_criu(struct criu_opts *opts)
 		} else {
 			const char *p;
 
-			p = cgroup_get_cgroup(opts->handler, controllers[0]);
+			p = cgroup_get_cgroup(opts->handler, controllers[0], false);
 			if (!p) {
 				ERROR("failed to get cgroup path for %s", controllers[0]);
 				goto err;
@@ -795,7 +795,7 @@ static void do_restore(struct lxc_container *c, int status_pipe, struct migrate_
 		goto out_fini_handler;
 	}
 
-	if (!cgroup_create(handler)) {
+	if (!cgroup_create(handler, false)) {
 		ERROR("failed creating groups");
 		goto out_fini_handler;
 	}
diff --git a/src/lxc/start.c b/src/lxc/start.c
index c2c14a7..e889421 100644
--- a/src/lxc/start.c
+++ b/src/lxc/start.c
@@ -1104,7 +1104,7 @@ static int lxc_spawn(struct lxc_handler *handler)
 
 	cgroups_connected = true;
 
-	if (!cgroup_create(handler)) {
+	if (!cgroup_create(handler, false)) {
 		ERROR("Failed creating cgroups.");
 		goto out_delete_net;
 	}
@@ -1191,10 +1191,10 @@ static int lxc_spawn(struct lxc_handler *handler)
 		goto out_delete_net;
 	}
 
-	if (!cgroup_enter(handler))
+	if (!cgroup_enter(handler, false))
 		goto out_delete_net;
 
-	if (!cgroup_chown(handler))
+	if (!cgroup_chown(handler, false))
 		goto out_delete_net;
 
 	if (failed_before_rename)
@@ -1237,6 +1237,21 @@ static int lxc_spawn(struct lxc_handler *handler)
 		goto out_delete_net;
 	}
 
+	if (cgns_supported()) {
+		if (!cgroup_create(handler, true)) {
+			ERROR("failed to create inner cgroup separation layer");
+			goto out_delete_net;
+		}
+		if (!cgroup_enter(handler, true)) {
+			ERROR("failed to enter inner cgroup separation layer");
+			goto out_delete_net;
+		}
+		if (!cgroup_chown(handler, true)) {
+			ERROR("failed chown inner cgroup separation layer");
+			goto out_delete_net;
+		}
+	}
+
 	cgroup_disconnect();
 	cgroups_connected = false;
 
-- 
2.1.4