snapshots: move snapshot directory

Originally we kept snapshots under /var/lib/lxcsnaps.  If a
separate btrfs is mounted at /var/lib/lxc, then we can't
make btrfs snapshots under /var/lib/lxcsnaps.

This patch moves the default directory to /var/lib/lxc/c/snaps.
If /var/lib/lxcsnaps already exists, then we continue to use that.

add c->destroy_with_snapshots() and c->snapshot_destroy_all()
API methods.  c->snashot_destroy_all() can be triggered from
lxc-snapshot using '-d ALL'.  There is no command to call
c->destroy_with_snapshots(c) as of yet.

lxclock: use ".$lxcname" for container lock files
that way we can use /run/lock/lxc/$lxcpath/$lxcname/snaps as a
directory when locking snapshots without having to worry about
/run/lock//lxc/$lxcpath/$lxcname being a file.

destroy: split off a container_destroy
container_destroy() doesn't check for snapshots, so snapshot_rename can
use it.  api_destroy() now does check for snapshots (previously it only
checked for fs - i.e. overlayfs/aufs - snapshots).

Add destroy to the manpage, as it was previously undocumented.

Update snapshot testcase accordingly.

[ rebased in the face of commits 840f05df and 7e36f87e. ]

Signed-off-by: Serge Hallyn <serge.hallyn@ubuntu.com>
Acked-by: S.Çağlar Onur <caglar@10ur.org>
Acked-by: Stéphane Graber <stgraber@ubuntu.com>

doc/lxc-snapshot.sgml.in

@@ -52,6 +52,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
       <arg choice="req">-n, --name <replaceable>name</replaceable></arg>
       <arg choice="opt">-c, --comment <replaceable>file</replaceable></arg>
     </cmdsynopsis>
+    <cmdsynopsis>
+      <command>lxc-snapshot</command>
+      <arg choice="req">-n, --name <replaceable>name</replaceable></arg>
+      <arg choice="req">-d, -destroy <replaceable>snapshot-name</replaceable></arg>
+    </cmdsynopsis>
     <cmdsynopsis>
       <command>lxc-snapshot</command>
       <arg choice="req">-n, --name <replaceable>name</replaceable></arg>
@@ -91,6 +96,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 	   </listitem>
 	  </varlistentry>
 
+	  <varlistentry>
+	    <term> <option>-d,--destroy snapshot-name</option> </term>
+	   <listitem>
+	    <para> Destroy the named snapshot.  If the named snapshot is ALL, then all snapshots
+	    will be destroyed.</para>
+	   </listitem>
+	  </varlistentry>
+
 	  <varlistentry>
 	    <term> <option>-L,--list </option> </term>
 	   <listitem>

src/lxc/bdev.c

@@ -482,7 +482,7 @@ static int dir_clonepaths(struct bdev *orig, struct bdev *new, const char *oldna
 
 static int dir_destroy(struct bdev *orig)
 {
-	if (lxc_rmdir_onedev(orig->src) < 0)
+	if (lxc_rmdir_onedev(orig->src, NULL) < 0)
 		return -1;
 	return 0;
 }
@@ -2073,7 +2073,7 @@ static int overlayfs_destroy(struct bdev *orig)
 	if (!upper)
 		return -22;
 	upper++;
-	return lxc_rmdir_onedev(upper);
+	return lxc_rmdir_onedev(upper, NULL);
 }
 
 /*
@@ -2350,7 +2350,7 @@ static int aufs_destroy(struct bdev *orig)
 	if (!upper)
 		return -22;
 	upper++;
-	return lxc_rmdir_onedev(upper);
+	return lxc_rmdir_onedev(upper, NULL);
 }
 
 /*

src/lxc/lxc_snapshot.c

@@ -112,7 +112,13 @@ static int do_restore_snapshots(struct lxc_container *c)
 
 static int do_destroy_snapshots(struct lxc_container *c)
 {
-	if (c->snapshot_destroy(c, snapshot))
+	bool bret;
+	if (strcmp(snapshot, "ALL") == 0)
+		bret = c->snapshot_destroy_all(c);
+	else
+		bret = c->snapshot_destroy(c, snapshot);
+
+	if (bret)
 		return 0;
 
 	ERROR("Error destroying snapshot %s", snapshot);
@@ -154,7 +160,8 @@ Options :\n\
   -C, --showcomments  show snapshot comments in list\n\
   -c, --comment=file  add file as a comment\n\
   -r, --restore=name  restore snapshot name, i.e. 'snap0'\n\
-  -d, --destroy=name  destroy snapshot name, i.e. 'snap0'\n",
+  -d, --destroy=name  destroy snapshot name, i.e. 'snap0'\n\
+                      use ALL to destroy all snapshots\n",
 	.options  = my_longopts,
 	.parser   = my_parser,
 	.checker  = NULL,

src/lxc/lxccontainer.c

@@ -1220,6 +1220,8 @@ static void lxcapi_clear_config(struct lxc_container *c)
 }
 
 static bool lxcapi_destroy(struct lxc_container *c);
+static bool container_destroy(struct lxc_container *c);
+static bool get_snappath_dir(struct lxc_container *c, char *snappath);
 /*
  * lxcapi_create:
  * create a container with the given parameters.
@@ -1362,7 +1364,7 @@ out_unlock:
 		remove_partial(c, partial_fd);
 out:
 	if (!ret && c)
-		lxcapi_destroy(c);
+		container_destroy(c);
 free_tpath:
 	if (tpath)
 		free(tpath);
@@ -1968,7 +1970,7 @@ out:
 	fclose(f);
 }
 
-static bool has_snapshots(struct lxc_container *c)
+static bool has_fs_snapshots(struct lxc_container *c)
 {
 	char path[MAXPATHLEN];
 	int ret, v;
@@ -1992,10 +1994,38 @@ out:
 	return bret;
 }
 
+static bool has_snapshots(struct lxc_container *c)
+{
+	char path[MAXPATHLEN];
+	struct dirent dirent, *direntp;
+	int count=0;
+	DIR *dir;
+
+	if (!get_snappath_dir(c, path))
+		return false;
+	dir = opendir(path);
+	if (!dir)
+		return false;
+	while (!readdir_r(dir, &dirent, &direntp)) {
+		if (!direntp)
+			break;
+
+		if (!strcmp(direntp->d_name, "."))
+			continue;
+
+		if (!strcmp(direntp->d_name, ".."))
+			continue;
+		count++;
+		break;
+	}
+	closedir(dir);
+	return count > 0;
+}
+
 static int lxc_rmdir_onedev_wrapper(void *data)
 {
 	char *arg = (char *) data;
-	return lxc_rmdir_onedev(arg);
+	return lxc_rmdir_onedev(arg, "snaps");
 }
 
 static int do_bdev_destroy(struct lxc_conf *conf)
@@ -2030,8 +2060,7 @@ static int bdev_destroy_wrapper(void *data)
 	return do_bdev_destroy(conf);
 }
 
-// do we want the api to support --force, or leave that to the caller?
+static bool container_destroy(struct lxc_container *c)
-static bool lxcapi_destroy(struct lxc_container *c)
 {
 	bool bret = false;
 	int ret;
@@ -2048,11 +2077,6 @@ static bool lxcapi_destroy(struct lxc_container *c)
 		goto out;
 	}
 
-	if (c->lxc_conf && has_snapshots(c)) {
-		ERROR("container %s has dependent snapshots", c->name);
-		goto out;
-	}
-
 	if (c->lxc_conf && c->lxc_conf->rootfs.path && c->lxc_conf->rootfs.mount) {
 		if (am_unpriv())
 			ret = userns_exec_1(c->lxc_conf, bdev_destroy_wrapper, c->lxc_conf);
@@ -2072,7 +2096,7 @@ static bool lxcapi_destroy(struct lxc_container *c)
 	if (am_unpriv())
 		ret = userns_exec_1(c->lxc_conf, lxc_rmdir_onedev_wrapper, path);
 	else
-		ret = lxc_rmdir_onedev(path);
+		ret = lxc_rmdir_onedev(path, "snaps");
 	if (ret < 0) {
 		ERROR("Error destroying container directory for %s", c->name);
 		goto out;
@@ -2084,6 +2108,37 @@ out:
 	return bret;
 }
 
+static bool lxcapi_destroy(struct lxc_container *c)
+{
+	if (!c || !lxcapi_is_defined(c))
+		return false;
+	if (has_snapshots(c)) {
+		ERROR("Container %s has snapshots;  not removing", c->name);
+		return false;
+	}
+
+	if (has_fs_snapshots(c)) {
+		ERROR("container %s has snapshots on its rootfs", c->name);
+		return false;
+	}
+
+	return container_destroy(c);
+}
+
+static bool lxcapi_snapshot_destroy_all(struct lxc_container *c);
+
+static bool lxcapi_destroy_with_snapshots(struct lxc_container *c)
+{
+	if (!c || !lxcapi_is_defined(c))
+		return false;
+	if (!lxcapi_snapshot_destroy_all(c)) {
+		ERROR("Error deleting all snapshots");
+		return false;
+	}
+	return lxcapi_destroy(c);
+}
+
+
 static bool set_config_item_locked(struct lxc_container *c, const char *key, const char *v)
 {
 	struct lxc_config_t *config;
@@ -2804,6 +2859,10 @@ static bool lxcapi_rename(struct lxc_container *c, const char *newname)
 	if (!c || !c->name || !c->config_path || !c->lxc_conf)
 		return false;
 
+	if (has_fs_snapshots(c) || has_snapshots(c)) {
+		ERROR("Renaming a container with snapshots is not supported");
+		return false;
+	}
 	bdev = bdev_init(c->lxc_conf, c->lxc_conf->rootfs.path, c->lxc_conf->rootfs.mount, NULL);
 	if (!bdev) {
 		ERROR("Failed to find original backing store type");
@@ -2820,7 +2879,7 @@ static bool lxcapi_rename(struct lxc_container *c, const char *newname)
 	if (newc && lxcapi_is_defined(newc))
 		lxc_container_put(newc);
 
-	if (!lxcapi_destroy(c)) {
+	if (!container_destroy(c)) {
 		ERROR("Could not destroy existing container %s", c->name);
 		return false;
 	}
@@ -2870,6 +2929,33 @@ static int get_next_index(const char *lxcpath, char *cname)
 	}
 }
 
+static bool get_snappath_dir(struct lxc_container *c, char *snappath)
+{
+	int ret;
+	/*
+	 * If the old style snapshot path exists, use it
+	 * /var/lib/lxc -> /var/lib/lxcsnaps
+	 */
+	ret = snprintf(snappath, MAXPATHLEN, "%ssnaps", c->config_path);
+	if (ret < 0 || ret >= MAXPATHLEN)
+		return false;
+	if (dir_exists(snappath)) {
+		ret = snprintf(snappath, MAXPATHLEN, "%ssnaps/%s", c->config_path, c->name);
+		if (ret < 0 || ret >= MAXPATHLEN)
+			return false;
+		return true;
+	}
+
+	/*
+	 * Use the new style path
+	 * /var/lib/lxc -> /var/lib/lxc + c->name + /snaps + \0
+	 */
+	ret = snprintf(snappath, MAXPATHLEN, "%s/%s/snaps", c->config_path, c->name);
+	if (ret < 0 || ret >= MAXPATHLEN)
+		return false;
+	return true;
+}
+
 static int lxcapi_snapshot(struct lxc_container *c, const char *commentfile)
 {
 	int i, flags, ret;
@@ -2879,10 +2965,9 @@ static int lxcapi_snapshot(struct lxc_container *c, const char *commentfile)
 	if (!c || !lxcapi_is_defined(c))
 		return -1;
 
-	// /var/lib/lxc -> /var/lib/lxcsnaps \0
+	if (!get_snappath_dir(c, snappath))
-	ret = snprintf(snappath, MAXPATHLEN, "%ssnaps/%s", c->config_path, c->name);
-	if (ret < 0 || ret >= MAXPATHLEN)
 		return -1;
+
 	i = get_next_index(snappath, c->name);
 
 	if (mkdir_p(snappath, 0755) < 0) {
@@ -3016,7 +3101,7 @@ static char *get_timestamp(char* snappath, char *name)
 static int lxcapi_snapshot_list(struct lxc_container *c, struct lxc_snapshot **ret_snaps)
 {
 	char snappath[MAXPATHLEN], path2[MAXPATHLEN];
-	int dirlen, count = 0, ret;
+	int count = 0, ret;
 	struct dirent dirent, *direntp;
 	struct lxc_snapshot *snaps =NULL, *nsnaps;
 	DIR *dir;
@@ -3024,9 +3109,7 @@ static int lxcapi_snapshot_list(struct lxc_container *c, struct lxc_snapshot **r
 	if (!c || !lxcapi_is_defined(c))
 		return -1;
 
-	// snappath is ${lxcpath}snaps/${lxcname}/
+	if (!get_snappath_dir(c, snappath)) {
-	dirlen = snprintf(snappath, MAXPATHLEN, "%ssnaps/%s", c->config_path, c->name);
-	if (dirlen < 0 || dirlen >= MAXPATHLEN) {
 		ERROR("path name too long");
 		return -1;
 	}
@@ -3094,7 +3177,7 @@ out_free:
 static bool lxcapi_snapshot_restore(struct lxc_container *c, const char *snapname, const char *newname)
 {
 	char clonelxcpath[MAXPATHLEN];
-	int flags = 0,ret;
+	int flags = 0;
 	struct lxc_container *snap, *rest;
 	struct bdev *bdev;
 	bool b = false;
@@ -3102,6 +3185,11 @@ static bool lxcapi_snapshot_restore(struct lxc_container *c, const char *snapnam
 	if (!c || !c->name || !c->config_path)
 		return false;
 
+	if (has_fs_snapshots(c)) {
+		ERROR("container rootfs has dependent snapshots");
+		return false;
+	}
+
 	bdev = bdev_init(c->lxc_conf, c->lxc_conf->rootfs.path, c->lxc_conf->rootfs.mount, NULL);
 	if (!bdev) {
 		ERROR("Failed to find original backing store type");
@@ -3111,8 +3199,7 @@ static bool lxcapi_snapshot_restore(struct lxc_container *c, const char *snapnam
 	if (!newname)
 		newname = c->name;
 
-	ret = snprintf(clonelxcpath, MAXPATHLEN, "%ssnaps/%s", c->config_path, c->name);
+	if (!get_snappath_dir(c, clonelxcpath)) {
-	if (ret < 0 || ret >= MAXPATHLEN) {
 		bdev_put(bdev);
 		return false;
 	}
@@ -3127,7 +3214,7 @@ static bool lxcapi_snapshot_restore(struct lxc_container *c, const char *snapnam
 	}
 
 	if (strcmp(c->name, newname) == 0) {
-		if (!lxcapi_destroy(c)) {
+		if (!container_destroy(c)) {
 			ERROR("Could not destroy existing container %s", newname);
 			lxc_container_put(snap);
 			bdev_put(bdev);
@@ -3148,21 +3235,13 @@ static bool lxcapi_snapshot_restore(struct lxc_container *c, const char *snapnam
 	return b;
 }
 
-static bool lxcapi_snapshot_destroy(struct lxc_container *c, const char *snapname)
+static bool do_snapshot_destroy(const char *snapname, const char *clonelxcpath)
 {
-	int ret;
-	char clonelxcpath[MAXPATHLEN];
 	struct lxc_container *snap = NULL;
-
+	bool bret = false;
-	if (!c || !c->name || !c->config_path)
-		return false;
-
-	ret = snprintf(clonelxcpath, MAXPATHLEN, "%ssnaps/%s", c->config_path, c->name);
-	if (ret < 0 || ret >= MAXPATHLEN)
-		goto err;
 
 	snap = lxc_container_new(snapname, clonelxcpath);
-	if (!snap || !lxcapi_is_defined(snap)) {
+	if (!snap) {
 		ERROR("Could not find snapshot %s", snapname);
 		goto err;
 	}
@@ -3171,13 +3250,70 @@ static bool lxcapi_snapshot_destroy(struct lxc_container *c, const char *snapnam
 		ERROR("Could not destroy snapshot %s", snapname);
 		goto err;
 	}
-	lxc_container_put(snap);
+	bret = true;
 
-	return true;
 err:
 	if (snap)
 		lxc_container_put(snap);
-	return false;
+	return bret;
+}
+
+static bool remove_all_snapshots(const char *path)
+{
+	DIR *dir;
+	struct dirent dirent, *direntp;
+	bool bret = true;
+
+	dir = opendir(path);
+	if (!dir) {
+		SYSERROR("opendir on snapshot path %s", path);
+		return false;
+	}
+	while (!readdir_r(dir, &dirent, &direntp)) {
+		if (!direntp)
+			break;
+		if (!strcmp(direntp->d_name, "."))
+			continue;
+		if (!strcmp(direntp->d_name, ".."))
+			continue;
+		if (!do_snapshot_destroy(direntp->d_name, path)) {
+			bret = false;
+			continue;
+		}
+	}
+
+	closedir(dir);
+
+	if (rmdir(path))
+		SYSERROR("Error removing directory %s", path);
+
+	return bret;
+}
+
+static bool lxcapi_snapshot_destroy(struct lxc_container *c, const char *snapname)
+{
+	char clonelxcpath[MAXPATHLEN];
+
+	if (!c || !c->name || !c->config_path || !snapname)
+		return false;
+
+	if (!get_snappath_dir(c, clonelxcpath))
+		return false;
+
+	return do_snapshot_destroy(snapname, clonelxcpath);
+}
+
+static bool lxcapi_snapshot_destroy_all(struct lxc_container *c)
+{
+	char clonelxcpath[MAXPATHLEN];
+
+	if (!c || !c->name || !c->config_path)
+		return false;
+
+	if (!get_snappath_dir(c, clonelxcpath))
+		return false;
+
+	return remove_all_snapshots(clonelxcpath);
 }
 
 static bool lxcapi_may_control(struct lxc_container *c)
@@ -3337,6 +3473,9 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 {
 	struct lxc_container *c;
 
+	if (!name)
+		return NULL;
+
 	c = malloc(sizeof(*c));
 	if (!c) {
 		fprintf(stderr, "failed to malloc lxc_container\n");
@@ -3383,7 +3522,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 
 	if (ongoing_create(c) == 2) {
 		ERROR("Error: %s creation was not completed", c->name);
-		lxcapi_destroy(c);
+		container_destroy(c);
 		lxcapi_clear_config(c);
 	}
 	c->daemonize = true;
@@ -3408,6 +3547,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 	c->wait = lxcapi_wait;
 	c->set_config_item = lxcapi_set_config_item;
 	c->destroy = lxcapi_destroy;
+	c->destroy_with_snapshots = lxcapi_destroy_with_snapshots;
 	c->rename = lxcapi_rename;
 	c->save_config = lxcapi_save_config;
 	c->get_keys = lxcapi_get_keys;
@@ -3433,6 +3573,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 	c->snapshot_list = lxcapi_snapshot_list;
 	c->snapshot_restore = lxcapi_snapshot_restore;
 	c->snapshot_destroy = lxcapi_snapshot_destroy;
+	c->snapshot_destroy_all = lxcapi_snapshot_destroy_all;
 	c->may_control = lxcapi_may_control;
 	c->add_device_node = lxcapi_add_device_node;
 	c->remove_device_node = lxcapi_remove_device_node;

src/lxc/lxccontainer.h

@@ -295,6 +295,17 @@ struct lxc_container {
 	 */
 	bool (*destroy)(struct lxc_container *c);
 
+	/*!
+	 * \brief Delete the container and all its snapshots.
+	 *
+	 * \param c Container.
+	 *
+	 * \return \c true on success, else \c false.
+	 *
+	 * \note Container must be stopped.
+	 */
+	bool (*destroy_with_snapshots)(struct lxc_container *c);
+
 	/*!
 	 * \brief Save configuaration to a file.
 	 *
@@ -659,7 +670,7 @@ struct lxc_container {
 	 * \brief Create a container snapshot.
 	 *
 	 * Assuming default paths, snapshots will be created as
-	 * \c /var/lib/lxcsnaps/\<c\>/snap\<n\>
+	 * \c /var/lib/lxc/\<c\>/snaps/snap\<n\>
 	 * where \c \<c\> represents the container name and \c \<n\>
 	 * represents the zero-based snapshot number.
 	 *
@@ -701,7 +712,7 @@ struct lxc_container {
 	 *  fail if the  snapshot is overlay-based, since the snapshots
 	 *  will pin the original container.
 	 * \note As an example, if the container exists as \c /var/lib/lxc/c1, snapname might be \c 'snap0'
-	 *  (representing \c /var/lib/lxcsnaps/c1/snap0). If \p newname is \p c2,
+	 *  (representing \c /var/lib/lxc/c1/snaps/snap0). If \p newname is \p c2,
 	 *  then \c snap0 will be copied to \c /var/lib/lxc/c2.
 	 */
 	bool (*snapshot_restore)(struct lxc_container *c, const char *snapname, const char *newname);
@@ -716,6 +727,15 @@ struct lxc_container {
 	 */
 	bool (*snapshot_destroy)(struct lxc_container *c, const char *snapname);
 
+	/*!
+	 * \brief Destroy all the container's snapshot.
+	 *
+	 * \param c Container.
+	 *
+	 * \return \c true on success, else \c false.
+	 */
+	bool (*snapshot_destroy_all)(struct lxc_container *c);
+
 	/*!
 	 * \brief Determine if the caller may control the container.
 	 *

src/lxc/lxclock.c

@@ -108,8 +108,8 @@ static char *lxclock_name(const char *p, const char *n)
 	 * $XDG_RUNTIME_DIR + "/lock/lxc/$lxcpath/$lxcname + '\0' if non-root
 	 */
 
-	/* length of "/lock/lxc/" + $lxcpath + "/" + $lxcname + '\0' */
+	/* length of "/lock/lxc/" + $lxcpath + "/" + "." + $lxcname + '\0' */
-	len = strlen("/lock/lxc/") + strlen(n) + strlen(p) + 2;
+	len = strlen("/lock/lxc/") + strlen(n) + strlen(p) + 3;
 	rundir = get_rundir();
 	if (!rundir)
 		return NULL;
@@ -129,7 +129,7 @@ static char *lxclock_name(const char *p, const char *n)
 	ret = mkdir_p(dest, 0755);
 	if (ret < 0) {
 		/* fall back to "/tmp/" $(id -u) "/lxc/" $lxcpath / $lxcname + '\0' */
-		int l2 = 33 + strlen(n) + strlen(p);
+		int l2 = 34 + strlen(n) + strlen(p);
 		if (l2 > len) {
 			char *d;
 			d = realloc(dest, l2);
@@ -147,9 +147,9 @@ static char *lxclock_name(const char *p, const char *n)
 			free(rundir);
 			return NULL;
 		}
-		ret = snprintf(dest, len, "/tmp/%d/lxc/%s/%s", geteuid(), p, n);
+		ret = snprintf(dest, len, "/tmp/%d/lxc/%s/.%s", geteuid(), p, n);
 	} else
-		ret = snprintf(dest, len, "%s/lock/lxc/%s/%s", rundir, p, n);
+		ret = snprintf(dest, len, "%s/lock/lxc/%s/.%s", rundir, p, n);
 
 	free(rundir);
 

src/lxc/utils.c

@@ -46,12 +46,14 @@
 
 lxc_log_define(lxc_utils, lxc);
 
-static int _recursive_rmdir_onedev(char *dirname, dev_t pdev)
+static int _recursive_rmdir_onedev(char *dirname, dev_t pdev,
+				   const char *exclude, int level)
 {
 	struct dirent dirent, *direntp;
 	DIR *dir;
 	int ret, failed=0;
 	char pathname[MAXPATHLEN];
+	bool hadexclude = false;
 
 	dir = opendir(dirname);
 	if (!dir) {
@@ -76,6 +78,29 @@ static int _recursive_rmdir_onedev(char *dirname, dev_t pdev)
 			failed=1;
 			continue;
 		}
+
+		if (!level && exclude && !strcmp(direntp->d_name, exclude)) {
+			ret = rmdir(pathname);
+			if (ret < 0) {
+				switch(errno) {
+				case ENOTEMPTY:
+					INFO("Not deleting snapshots");
+					hadexclude = true;
+					break;
+				case ENOTDIR:
+					ret = unlink(pathname);
+					if (ret)
+						INFO("%s: failed to remove %s", __func__, pathname);
+					break;
+				default:
+					SYSERROR("%s: failed to rmdir %s", __func__, pathname);
+					failed = 1;
+					break;
+				}
+			}
+			continue;
+		}
+
 		ret = lstat(pathname, &mystat);
 		if (ret) {
 			ERROR("%s: failed to stat %s", __func__, pathname);
@@ -85,7 +110,7 @@ static int _recursive_rmdir_onedev(char *dirname, dev_t pdev)
 		if (mystat.st_dev != pdev)
 			continue;
 		if (S_ISDIR(mystat.st_mode)) {
-			if (_recursive_rmdir_onedev(pathname, pdev) < 0)
+			if (_recursive_rmdir_onedev(pathname, pdev, exclude, level+1) < 0)
 				failed=1;
 		} else {
 			if (unlink(pathname) < 0) {
@@ -96,8 +121,10 @@ static int _recursive_rmdir_onedev(char *dirname, dev_t pdev)
 	}
 
 	if (rmdir(dirname) < 0) {
-		ERROR("%s: failed to delete %s", __func__, dirname);
+		if (!hadexclude) {
-		failed=1;
+			ERROR("%s: failed to delete %s", __func__, dirname);
+			failed=1;
+		}
 	}
 
 	ret = closedir(dir);
@@ -110,7 +137,7 @@ static int _recursive_rmdir_onedev(char *dirname, dev_t pdev)
 }
 
 /* returns 0 on success, -1 if there were any failures */
-extern int lxc_rmdir_onedev(char *path)
+extern int lxc_rmdir_onedev(char *path, const char *exclude)
 {
 	struct stat mystat;
 
@@ -119,7 +146,7 @@ extern int lxc_rmdir_onedev(char *path)
 		return -1;
 	}
 
-	return _recursive_rmdir_onedev(path, mystat.st_dev);
+	return _recursive_rmdir_onedev(path, mystat.st_dev, exclude, 0);
 }
 
 static int mount_fs(const char *source, const char *target, const char *type)

src/lxc/utils.h

@@ -34,7 +34,7 @@
 #include "config.h"
 
 /* returns 1 on success, 0 if there were any failures */
-extern int lxc_rmdir_onedev(char *path);
+extern int lxc_rmdir_onedev(char *path, const char *exclude);
 extern void lxc_setup_fs(void);
 extern int get_u16(unsigned short *val, const char *arg, int base);
 extern int mkdir_p(const char *dir, mode_t mode);

src/tests/snapshot.c

@@ -32,28 +32,21 @@
 static void try_to_remove(void)
 {
 	struct lxc_container *c;
-	char snappath[1024];
 	c = lxc_container_new(RESTNAME, NULL);
 	if (c) {
-		if (c->is_defined(c))
+		c->snapshot_destroy_all(c);
-			c->destroy(c);
-		lxc_container_put(c);
-	}
-	snprintf(snappath, 1024, "%ssnaps/%s", lxc_get_global_config_item("lxc.lxcpath"), MYNAME);
-	c = lxc_container_new("snap0", snappath);
-	if (c) {
 		if (c->is_defined(c))
 			c->destroy(c);
 		lxc_container_put(c);
 	}
 	c = lxc_container_new(MYNAME2, NULL);
 	if (c) {
-		if (c->is_defined(c))
+		c->destroy_with_snapshots(c);
-			c->destroy(c);
 		lxc_container_put(c);
 	}
 	c = lxc_container_new(MYNAME, NULL);
 	if (c) {
+		c->snapshot_destroy_all(c);
 		if (c->is_defined(c))
 			c->destroy(c);
 		lxc_container_put(c);
@@ -77,7 +70,7 @@ int main(int argc, char *argv[])
 
 	if (c->is_defined(c)) {
 		fprintf(stderr, "%d: %s thought it was defined\n", __LINE__, MYNAME);
-		(void) c->destroy(c);
+		(void) c->destroy_with_snapshots(c);
 	}
 	if (!c->set_config_item(c, "lxc.network.type", "empty")) {
 		fprintf(stderr, "%s: %d: failed to set network type\n", __FILE__, __LINE__);
@@ -95,11 +88,11 @@ int main(int argc, char *argv[])
 		goto err;
 	}
 
-	// rootfs should be ${lxcpath}snaps/${lxcname}/snap0/rootfs
+	// rootfs should be ${lxcpath}${lxcname}/snaps/snap0/rootfs
 	struct stat sb;
 	int ret;
 	char path[1024];
-	snprintf(path, 1024, "%ssnaps/%s/snap0/rootfs", lxc_get_global_config_item("lxc.lxcpath"), MYNAME);
+	snprintf(path, 1024, "%s/%s/snaps/snap0/rootfs", lxc_get_global_config_item("lxc.lxcpath"), MYNAME);
 	ret = stat(path, &sb);
 	if (ret != 0) {
 		fprintf(stderr, "%s: %d: snapshot was not actually created\n", __FILE__, __LINE__);
@@ -169,17 +162,7 @@ int main(int argc, char *argv[])
 		goto err;
 	}
 
-	if (!c2->destroy(c2)) {
-		fprintf(stderr, "%s: %d: failed to destroy container\n", __FILE__, __LINE__);
-		goto err;
-	}
-
 good:
-	if (!c->destroy(c)) {
-		fprintf(stderr, "%s: %d: failed to destroy container\n", __FILE__, __LINE__);
-		goto err;
-	}
-
 	lxc_container_put(c);
 	try_to_remove();