Block patches:

- Minor fixes and tests from the freeze period (too minor to be included
   in 4.2)
 - Allow many bash iotests to test qcow2's external data file feature
 - Add compress filter driver
 - Fix Python iotests after 6f6e1698a6
 - Fix for the backup job
 -----BEGIN PGP SIGNATURE-----
 
 iQFGBAABCAAwFiEEkb62CjDbPohX0Rgp9AfbAGHVz0AFAl4TRSISHG1yZWl0ekBy
 ZWRoYXQuY29tAAoJEPQH2wBh1c9AOiUIAJKWyzc+d7Fe4mwYamrNq5kf3dadw0qp
 t4HGjQ5vn+u4I4FH7NwSlx7qpWcerqxijeOJUYPMzCAS2/cm6OG4ETcPzfM+TeZL
 sJupvu1aloUnJiQxs424KD5AClHftByu+KpDOYmt8HmuSG+PkXvrz3q5bfJEjBQ3
 ZVKp7Ye8XkSLMPFlOavJGiniEx9huzQ7d1PHxZjpq+GrowFN+vGBuRu2+GfYQSIl
 +F1XbojSA7X91DDLgLbgZbTrwA4ph9qUJF1ObzhCToL2JcTt8H4Wleujaah2czi5
 2tOt1xKGtXy0mSx8ToJSI9JYIy/R/cGp7J9RNWgWa9OvNftgCJ10mAs=
 =EHAE
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2020-01-06' into staging

Block patches:
- Minor fixes and tests from the freeze period (too minor to be included
  in 4.2)
- Allow many bash iotests to test qcow2's external data file feature
- Add compress filter driver
- Fix Python iotests after 6f6e1698a6
- Fix for the backup job

# gpg: Signature made Mon 06 Jan 2020 14:33:06 GMT
# gpg:                using RSA key 91BEB60A30DB3E8857D11829F407DB0061D5CF40
# gpg:                issuer "mreitz@redhat.com"
# gpg: Good signature from "Max Reitz <mreitz@redhat.com>" [full]
# Primary key fingerprint: 91BE B60A 30DB 3E88 57D1  1829 F407 DB00 61D5 CF40

* remotes/maxreitz/tags/pull-block-2020-01-06: (34 commits)
  backup-top: Begin drain earlier
  tests/qemu-iotests: Update tests to recent desugarized -accel option
  tests/qemu-iotests: add case to write compressed data of multiple clusters
  qcow2: Allow writing compressed data of multiple clusters
  block: introduce compress filter driver
  iotests: Allow check -o data_file
  iotests: Disable data_file where it cannot be used
  iotests: Make 198 work with data_file
  iotests: Make 137 work with data_file
  iotests: Make 110 work with data_file
  iotests: Make 091 work with data_file
  iotests: Avoid cp/mv of test images
  iotests: Use _rm_test_img for deleting test images
  iotests: Avoid qemu-img create
  iotests: Drop IMGOPTS use in 267
  iotests: Replace IMGOPTS='' by --no-opts
  iotests: Replace IMGOPTS= by -o
  iotests: Inject space into -ocompat=0.10 in 051
  iotests: Add -o and --no-opts to _make_test_img
  iotests: Let _make_test_img parse its parameters
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

block.c

@@ -2227,6 +2227,24 @@ void bdrv_format_default_perms(BlockDriverState *bs, BdrvChild *c,
     *nshared = shared;
 }
 
+uint64_t bdrv_qapi_perm_to_blk_perm(BlockPermission qapi_perm)
+{
+    static const uint64_t permissions[] = {
+        [BLOCK_PERMISSION_CONSISTENT_READ]  = BLK_PERM_CONSISTENT_READ,
+        [BLOCK_PERMISSION_WRITE]            = BLK_PERM_WRITE,
+        [BLOCK_PERMISSION_WRITE_UNCHANGED]  = BLK_PERM_WRITE_UNCHANGED,
+        [BLOCK_PERMISSION_RESIZE]           = BLK_PERM_RESIZE,
+        [BLOCK_PERMISSION_GRAPH_MOD]        = BLK_PERM_GRAPH_MOD,
+    };
+
+    QEMU_BUILD_BUG_ON(ARRAY_SIZE(permissions) != BLOCK_PERMISSION__MAX);
+    QEMU_BUILD_BUG_ON(1UL << ARRAY_SIZE(permissions) != BLK_PERM_ALL + 1);
+
+    assert(qapi_perm < BLOCK_PERMISSION__MAX);
+
+    return permissions[qapi_perm];
+}
+
 static void bdrv_replace_child_noperm(BdrvChild *child,
                                       BlockDriverState *new_bs)
 {
@@ -4854,36 +4872,23 @@ static void xdbg_graph_add_node(XDbgBlockGraphConstructor *gr, void *node,
 static void xdbg_graph_add_edge(XDbgBlockGraphConstructor *gr, void *parent,
                                 const BdrvChild *child)
 {
-    typedef struct {
+    BlockPermission qapi_perm;
-        unsigned int flag;
-        BlockPermission num;
-    } PermissionMap;
-
-    static const PermissionMap permissions[] = {
-        { BLK_PERM_CONSISTENT_READ, BLOCK_PERMISSION_CONSISTENT_READ },
-        { BLK_PERM_WRITE,           BLOCK_PERMISSION_WRITE },
-        { BLK_PERM_WRITE_UNCHANGED, BLOCK_PERMISSION_WRITE_UNCHANGED },
-        { BLK_PERM_RESIZE,          BLOCK_PERMISSION_RESIZE },
-        { BLK_PERM_GRAPH_MOD,       BLOCK_PERMISSION_GRAPH_MOD },
-        { 0, 0 }
-    };
-    const PermissionMap *p;
     XDbgBlockGraphEdge *edge;
 
-    QEMU_BUILD_BUG_ON(1UL << (ARRAY_SIZE(permissions) - 1) != BLK_PERM_ALL + 1);
-
     edge = g_new0(XDbgBlockGraphEdge, 1);
 
     edge->parent = xdbg_graph_node_num(gr, parent);
     edge->child = xdbg_graph_node_num(gr, child->bs);
     edge->name = g_strdup(child->name);
 
-    for (p = permissions; p->flag; p++) {
+    for (qapi_perm = 0; qapi_perm < BLOCK_PERMISSION__MAX; qapi_perm++) {
-        if (p->flag & child->perm) {
+        uint64_t flag = bdrv_qapi_perm_to_blk_perm(qapi_perm);
-            QAPI_LIST_ADD(edge->perm, p->num);
+
+        if (flag & child->perm) {
+            QAPI_LIST_ADD(edge->perm, qapi_perm);
         }
-        if (p->flag & child->shared_perm) {
+        if (flag & child->shared_perm) {
-            QAPI_LIST_ADD(edge->shared_perm, p->num);
+            QAPI_LIST_ADD(edge->shared_perm, qapi_perm);
         }
     }
 

block/Makefile.objs

@@ -43,6 +43,7 @@ block-obj-y += crypto.o
 
 block-obj-y += aio_task.o
 block-obj-y += backup-top.o
+block-obj-y += filter-compress.o
 
 common-obj-y += stream.o
 

block/backup-top.c

@@ -257,12 +257,12 @@ void bdrv_backup_top_drop(BlockDriverState *bs)
     BDRVBackupTopState *s = bs->opaque;
     AioContext *aio_context = bdrv_get_aio_context(bs);
 
-    block_copy_state_free(s->bcs);
-
     aio_context_acquire(aio_context);
 
     bdrv_drained_begin(bs);
 
+    block_copy_state_free(s->bcs);
+
     s->active = false;
     bdrv_child_refresh_perms(bs, bs->backing, &error_abort);
     bdrv_replace_node(bs, backing_bs(bs), &error_abort);

block/blkdebug.c

@@ -28,10 +28,14 @@
 #include "qemu/cutils.h"
 #include "qemu/config-file.h"
 #include "block/block_int.h"
+#include "block/qdict.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
 #include "qapi/qmp/qstring.h"
+#include "qapi/qobject-input-visitor.h"
 #include "sysemu/qtest.h"
 
 typedef struct BDRVBlkdebugState {
@@ -44,6 +48,9 @@ typedef struct BDRVBlkdebugState {
     uint64_t opt_discard;
     uint64_t max_discard;
 
+    uint64_t take_child_perms;
+    uint64_t unshare_child_perms;
+
     /* For blkdebug_refresh_filename() */
     char *config_file;
 
@@ -344,6 +351,69 @@ static void blkdebug_parse_filename(const char *filename, QDict *options,
     qdict_put_str(options, "x-image", filename);
 }
 
+static int blkdebug_parse_perm_list(uint64_t *dest, QDict *options,
+                                    const char *prefix, Error **errp)
+{
+    int ret = 0;
+    QDict *subqdict = NULL;
+    QObject *crumpled_subqdict = NULL;
+    Visitor *v = NULL;
+    BlockPermissionList *perm_list = NULL, *element;
+    Error *local_err = NULL;
+
+    *dest = 0;
+
+    qdict_extract_subqdict(options, &subqdict, prefix);
+    if (!qdict_size(subqdict)) {
+        goto out;
+    }
+
+    crumpled_subqdict = qdict_crumple(subqdict, errp);
+    if (!crumpled_subqdict) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    v = qobject_input_visitor_new(crumpled_subqdict);
+    visit_type_BlockPermissionList(v, NULL, &perm_list, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    for (element = perm_list; element; element = element->next) {
+        *dest |= bdrv_qapi_perm_to_blk_perm(element->value);
+    }
+
+out:
+    qapi_free_BlockPermissionList(perm_list);
+    visit_free(v);
+    qobject_unref(subqdict);
+    qobject_unref(crumpled_subqdict);
+    return ret;
+}
+
+static int blkdebug_parse_perms(BDRVBlkdebugState *s, QDict *options,
+                                Error **errp)
+{
+    int ret;
+
+    ret = blkdebug_parse_perm_list(&s->take_child_perms, options,
+                                   "take-child-perms.", errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = blkdebug_parse_perm_list(&s->unshare_child_perms, options,
+                                   "unshare-child-perms.", errp);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
+}
+
 static QemuOptsList runtime_opts = {
     .name = "blkdebug",
     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
@@ -419,6 +489,12 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,
     /* Set initial state */
     s->state = 1;
 
+    /* Parse permissions modifiers before opening the image file */
+    ret = blkdebug_parse_perms(s, options, errp);
+    if (ret < 0) {
+        goto out;
+    }
+
     /* Open the image file */
     bs->file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options, "image",
                                bs, &child_file, false, &local_err);
@@ -916,6 +992,21 @@ static int blkdebug_reopen_prepare(BDRVReopenState *reopen_state,
     return 0;
 }
 
+static void blkdebug_child_perm(BlockDriverState *bs, BdrvChild *c,
+                                const BdrvChildRole *role,
+                                BlockReopenQueue *reopen_queue,
+                                uint64_t perm, uint64_t shared,
+                                uint64_t *nperm, uint64_t *nshared)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+
+    bdrv_filter_default_perms(bs, c, role, reopen_queue, perm, shared,
+                              nperm, nshared);
+
+    *nperm |= s->take_child_perms;
+    *nshared &= ~s->unshare_child_perms;
+}
+
 static const char *const blkdebug_strong_runtime_opts[] = {
     "config",
     "inject-error.",
@@ -940,7 +1031,7 @@ static BlockDriver bdrv_blkdebug = {
     .bdrv_file_open         = blkdebug_open,
     .bdrv_close             = blkdebug_close,
     .bdrv_reopen_prepare    = blkdebug_reopen_prepare,
-    .bdrv_child_perm        = bdrv_filter_default_perms,
+    .bdrv_child_perm        = blkdebug_child_perm,
 
     .bdrv_getlength         = blkdebug_getlength,
     .bdrv_refresh_filename  = blkdebug_refresh_filename,

block/filter-compress.c

@@ -0,0 +1,168 @@
+/*
+ * Compress filter block driver
+ *
+ * Copyright (c) 2019 Virtuozzo International GmbH
+ *
+ * Author:
+ *   Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
+ *   (based on block/copy-on-read.c by Max Reitz)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) any later version of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "qapi/error.h"
+
+
+static int compress_open(BlockDriverState *bs, QDict *options, int flags,
+                         Error **errp)
+{
+    bs->file = bdrv_open_child(NULL, options, "file", bs, &child_file, false,
+                               errp);
+    if (!bs->file) {
+        return -EINVAL;
+    }
+
+    if (!bs->file->bs->drv || !block_driver_can_compress(bs->file->bs->drv)) {
+        error_setg(errp,
+                   "Compression is not supported for underlying format: %s",
+                   bdrv_get_format_name(bs->file->bs) ?: "(no format)");
+
+        return -ENOTSUP;
+    }
+
+    bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |
+        (BDRV_REQ_FUA & bs->file->bs->supported_write_flags);
+
+    bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
+        ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &
+            bs->file->bs->supported_zero_flags);
+
+    return 0;
+}
+
+
+static int64_t compress_getlength(BlockDriverState *bs)
+{
+    return bdrv_getlength(bs->file->bs);
+}
+
+
+static int coroutine_fn compress_co_preadv_part(BlockDriverState *bs,
+                                                uint64_t offset, uint64_t bytes,
+                                                QEMUIOVector *qiov,
+                                                size_t qiov_offset,
+                                                int flags)
+{
+    return bdrv_co_preadv_part(bs->file, offset, bytes, qiov, qiov_offset,
+                               flags);
+}
+
+
+static int coroutine_fn compress_co_pwritev_part(BlockDriverState *bs,
+                                                 uint64_t offset,
+                                                 uint64_t bytes,
+                                                 QEMUIOVector *qiov,
+                                                 size_t qiov_offset, int flags)
+{
+    return bdrv_co_pwritev_part(bs->file, offset, bytes, qiov, qiov_offset,
+                                flags | BDRV_REQ_WRITE_COMPRESSED);
+}
+
+
+static int coroutine_fn compress_co_pwrite_zeroes(BlockDriverState *bs,
+                                                  int64_t offset, int bytes,
+                                                  BdrvRequestFlags flags)
+{
+    return bdrv_co_pwrite_zeroes(bs->file, offset, bytes, flags);
+}
+
+
+static int coroutine_fn compress_co_pdiscard(BlockDriverState *bs,
+                                             int64_t offset, int bytes)
+{
+    return bdrv_co_pdiscard(bs->file, offset, bytes);
+}
+
+
+static void compress_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    BlockDriverInfo bdi;
+    int ret;
+
+    if (!bs->file) {
+        return;
+    }
+
+    ret = bdrv_get_info(bs->file->bs, &bdi);
+    if (ret < 0 || bdi.cluster_size == 0) {
+        return;
+    }
+
+    bs->bl.request_alignment = bdi.cluster_size;
+}
+
+
+static void compress_eject(BlockDriverState *bs, bool eject_flag)
+{
+    bdrv_eject(bs->file->bs, eject_flag);
+}
+
+
+static void compress_lock_medium(BlockDriverState *bs, bool locked)
+{
+    bdrv_lock_medium(bs->file->bs, locked);
+}
+
+
+static bool compress_recurse_is_first_non_filter(BlockDriverState *bs,
+                                                 BlockDriverState *candidate)
+{
+    return bdrv_recurse_is_first_non_filter(bs->file->bs, candidate);
+}
+
+
+static BlockDriver bdrv_compress = {
+    .format_name                        = "compress",
+
+    .bdrv_open                          = compress_open,
+    .bdrv_child_perm                    = bdrv_filter_default_perms,
+
+    .bdrv_getlength                     = compress_getlength,
+
+    .bdrv_co_preadv_part                = compress_co_preadv_part,
+    .bdrv_co_pwritev_part               = compress_co_pwritev_part,
+    .bdrv_co_pwrite_zeroes              = compress_co_pwrite_zeroes,
+    .bdrv_co_pdiscard                   = compress_co_pdiscard,
+    .bdrv_refresh_limits                = compress_refresh_limits,
+
+    .bdrv_eject                         = compress_eject,
+    .bdrv_lock_medium                   = compress_lock_medium,
+
+    .bdrv_co_block_status               = bdrv_co_block_status_from_file,
+
+    .bdrv_recurse_is_first_non_filter   = compress_recurse_is_first_non_filter,
+
+    .has_variable_length                = true,
+    .is_filter                          = true,
+};
+
+static void bdrv_compress_init(void)
+{
+    bdrv_register(&bdrv_compress);
+}
+
+block_init(bdrv_compress_init);

block/qcow2-bitmap.c

@@ -1703,8 +1703,14 @@ bool coroutine_fn qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
                                                       Error **errp)
 {
     BDRVQcow2State *s = bs->opaque;
-    bool found;
+    BdrvDirtyBitmap *bitmap;
-    Qcow2BitmapList *bm_list;
+    uint64_t bitmap_directory_size = 0;
+    uint32_t nb_bitmaps = 0;
+
+    if (bdrv_find_dirty_bitmap(bs, name)) {
+        error_setg(errp, "Bitmap already exists: %s", name);
+        return false;
+    }
 
     if (s->qcow_version < 3) {
         /* Without autoclear_features, we would always have to assume
@@ -1720,38 +1726,27 @@ bool coroutine_fn qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
         goto fail;
     }
 
-    if (s->nb_bitmaps == 0) {
+    FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
-        return true;
+        if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
+            nb_bitmaps++;
+            bitmap_directory_size +=
+                calc_dir_entry_size(strlen(bdrv_dirty_bitmap_name(bitmap)), 0);
         }
+    }
+    nb_bitmaps++;
+    bitmap_directory_size += calc_dir_entry_size(strlen(name), 0);
 
-    if (s->nb_bitmaps >= QCOW2_MAX_BITMAPS) {
+    if (nb_bitmaps > QCOW2_MAX_BITMAPS) {
         error_setg(errp,
                    "Maximum number of persistent bitmaps is already reached");
         goto fail;
     }
 
-    if (s->bitmap_directory_size + calc_dir_entry_size(strlen(name), 0) >
+    if (bitmap_directory_size > QCOW2_MAX_BITMAP_DIRECTORY_SIZE) {
-        QCOW2_MAX_BITMAP_DIRECTORY_SIZE)
-    {
         error_setg(errp, "Not enough space in the bitmap directory");
         goto fail;
     }
 
-    qemu_co_mutex_lock(&s->lock);
-    bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
-                               s->bitmap_directory_size, errp);
-    qemu_co_mutex_unlock(&s->lock);
-    if (bm_list == NULL) {
-        goto fail;
-    }
-
-    found = find_bitmap_by_name(bm_list, name);
-    bitmap_list_free(bm_list);
-    if (found) {
-        error_setg(errp, "Bitmap with the same name is already stored");
-        goto fail;
-    }
-
     return true;
 
 fail:

block/qcow2.c

@@ -4221,10 +4221,8 @@ fail:
     return ret;
 }
 
-/* XXX: put compressed sectors first, then all the cluster aligned
-   tables to avoid losing bytes in alignment */
 static coroutine_fn int
-qcow2_co_pwritev_compressed_part(BlockDriverState *bs,
+qcow2_co_pwritev_compressed_task(BlockDriverState *bs,
                                  uint64_t offset, uint64_t bytes,
                                  QEMUIOVector *qiov, size_t qiov_offset)
 {
@@ -4234,32 +4232,11 @@ qcow2_co_pwritev_compressed_part(BlockDriverState *bs,
     uint8_t *buf, *out_buf;
     uint64_t cluster_offset;
 
-    if (has_data_file(bs)) {
+    assert(bytes == s->cluster_size || (bytes < s->cluster_size &&
-        return -ENOTSUP;
+           (offset + bytes == bs->total_sectors << BDRV_SECTOR_BITS)));
-    }
-
-    if (bytes == 0) {
-        /* align end of file to a sector boundary to ease reading with
-           sector based I/Os */
-        int64_t len = bdrv_getlength(bs->file->bs);
-        if (len < 0) {
-            return len;
-        }
-        return bdrv_co_truncate(bs->file, len, false, PREALLOC_MODE_OFF, NULL);
-    }
-
-    if (offset_into_cluster(s, offset)) {
-        return -EINVAL;
-    }
 
     buf = qemu_blockalign(bs, s->cluster_size);
-    if (bytes != s->cluster_size) {
+    if (bytes < s->cluster_size) {
-        if (bytes > s->cluster_size ||
-            offset + bytes != bs->total_sectors << BDRV_SECTOR_BITS)
-        {
-            qemu_vfree(buf);
-            return -EINVAL;
-        }
         /* Zero-pad last write if image size is not cluster aligned */
         memset(buf + bytes, 0, s->cluster_size - bytes);
     }
@@ -4308,6 +4285,77 @@ fail:
     return ret;
 }
 
+static coroutine_fn int qcow2_co_pwritev_compressed_task_entry(AioTask *task)
+{
+    Qcow2AioTask *t = container_of(task, Qcow2AioTask, task);
+
+    assert(!t->cluster_type && !t->l2meta);
+
+    return qcow2_co_pwritev_compressed_task(t->bs, t->offset, t->bytes, t->qiov,
+                                            t->qiov_offset);
+}
+
+/*
+ * XXX: put compressed sectors first, then all the cluster aligned
+ * tables to avoid losing bytes in alignment
+ */
+static coroutine_fn int
+qcow2_co_pwritev_compressed_part(BlockDriverState *bs,
+                                 uint64_t offset, uint64_t bytes,
+                                 QEMUIOVector *qiov, size_t qiov_offset)
+{
+    BDRVQcow2State *s = bs->opaque;
+    AioTaskPool *aio = NULL;
+    int ret = 0;
+
+    if (has_data_file(bs)) {
+        return -ENOTSUP;
+    }
+
+    if (bytes == 0) {
+        /*
+         * align end of file to a sector boundary to ease reading with
+         * sector based I/Os
+         */
+        int64_t len = bdrv_getlength(bs->file->bs);
+        if (len < 0) {
+            return len;
+        }
+        return bdrv_co_truncate(bs->file, len, false, PREALLOC_MODE_OFF, NULL);
+    }
+
+    if (offset_into_cluster(s, offset)) {
+        return -EINVAL;
+    }
+
+    while (bytes && aio_task_pool_status(aio) == 0) {
+        uint64_t chunk_size = MIN(bytes, s->cluster_size);
+
+        if (!aio && chunk_size != bytes) {
+            aio = aio_task_pool_new(QCOW2_MAX_WORKERS);
+        }
+
+        ret = qcow2_add_task(bs, aio, qcow2_co_pwritev_compressed_task_entry,
+                             0, 0, offset, chunk_size, qiov, qiov_offset, NULL);
+        if (ret < 0) {
+            break;
+        }
+        qiov_offset += chunk_size;
+        offset += chunk_size;
+        bytes -= chunk_size;
+    }
+
+    if (aio) {
+        aio_task_pool_wait_all(aio);
+        if (ret == 0) {
+            ret = aio_task_pool_status(aio);
+        }
+        g_free(aio);
+    }
+
+    return ret;
+}
+
 static int coroutine_fn
 qcow2_co_preadv_compressed(BlockDriverState *bs,
                            uint64_t file_cluster_offset,

block/throttle-groups.c

@@ -893,8 +893,7 @@ static void throttle_group_set_limits(Object *obj, Visitor *v,
 {
     ThrottleGroup *tg = THROTTLE_GROUP(obj);
     ThrottleConfig cfg;
-    ThrottleLimits arg = { 0 };
+    ThrottleLimits *argp;
-    ThrottleLimits *argp = &arg;
     Error *local_err = NULL;
 
     visit_type_ThrottleLimits(v, name, &argp, &local_err);
@@ -912,6 +911,7 @@ static void throttle_group_set_limits(Object *obj, Visitor *v,
 unlock:
     qemu_mutex_unlock(&tg->lock);
 ret:
+    qapi_free_ThrottleLimits(argp);
     error_propagate(errp, local_err);
     return;
 }

include/block/block.h

@@ -280,6 +280,7 @@ enum {
 };
 
 char *bdrv_perm_names(uint64_t perm);
+uint64_t bdrv_qapi_perm_to_blk_perm(BlockPermission qapi_perm);
 
 /* disk I/O throttling */
 void bdrv_init(void);

qapi/block-core.json

@@ -2884,15 +2884,16 @@
 # @copy-on-read: Since 3.0
 # @blklogwrites: Since 3.0
 # @blkreplay: Since 4.2
+# @compress: Since 5.0
 #
 # Since: 2.9
 ##
 { 'enum': 'BlockdevDriver',
   'data': [ 'blkdebug', 'blklogwrites', 'blkreplay', 'blkverify', 'bochs',
-            'cloop', 'copy-on-read', 'dmg', 'file', 'ftp', 'ftps', 'gluster',
+            'cloop', 'compress', 'copy-on-read', 'dmg', 'file', 'ftp', 'ftps',
-            'host_cdrom', 'host_device', 'http', 'https', 'iscsi', 'luks',
+            'gluster', 'host_cdrom', 'host_device', 'http', 'https', 'iscsi',
-            'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow',
+            'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
-            'qcow2', 'qed', 'quorum', 'raw', 'rbd',
+            'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
             { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
             'sheepdog',
             'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
@@ -3454,6 +3455,16 @@
 #
 # @set-state:       array of state-change descriptions
 #
+# @take-child-perms: Permissions to take on @image in addition to what
+#                    is necessary anyway (which depends on how the
+#                    blkdebug node is used).  Defaults to none.
+#                    (since 5.0)
+#
+# @unshare-child-perms: Permissions not to share on @image in addition
+#                       to what cannot be shared anyway (which depends
+#                       on how the blkdebug node is used).  Defaults
+#                       to none.  (since 5.0)
+#
 # Since: 2.9
 ##
 { 'struct': 'BlockdevOptionsBlkdebug',
@@ -3463,7 +3474,9 @@
             '*opt-write-zero': 'int32', '*max-write-zero': 'int32',
             '*opt-discard': 'int32', '*max-discard': 'int32',
             '*inject-error': ['BlkdebugInjectErrorOptions'],
-            '*set-state': ['BlkdebugSetStateOptions'] } }
+            '*set-state': ['BlkdebugSetStateOptions'],
+            '*take-child-perms': ['BlockPermission'],
+            '*unshare-child-perms': ['BlockPermission'] } }
 
 ##
 # @BlockdevOptionsBlklogwrites:
@@ -4048,6 +4061,7 @@
       'blkreplay':  'BlockdevOptionsBlkreplay',
       'bochs':      'BlockdevOptionsGenericFormat',
       'cloop':      'BlockdevOptionsGenericFormat',
+      'compress':   'BlockdevOptionsGenericFormat',
       'copy-on-read':'BlockdevOptionsGenericFormat',
       'dmg':        'BlockdevOptionsGenericFormat',
       'file':       'BlockdevOptionsFile',

tests/qemu-iotests/007

@@ -41,8 +41,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto generic
 # refcount_bits must be at least 4 so we can create ten internal snapshots
-# (1 bit supports none, 2 bits support two, 4 bits support 14)
+# (1 bit supports none, 2 bits support two, 4 bits support 14);
-_unsupported_imgopts 'refcount_bits=\(1\|2\)[^0-9]'
+# snapshot are generally impossible with external data files
+_unsupported_imgopts 'refcount_bits=\(1\|2\)[^0-9]' data_file
 
 echo
 echo "creating image"

tests/qemu-iotests/014

@@ -43,6 +43,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# Compression and snapshots do not work with external data files
+_unsupported_imgopts data_file
 
 TEST_OFFSETS="0 4294967296"
 TEST_OPS="writev read write readv"

tests/qemu-iotests/015

@@ -40,8 +40,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # actually any format that supports snapshots
 _supported_fmt qcow2
 _supported_proto generic
-# Internal snapshots are (currently) impossible with refcount_bits=1
+# Internal snapshots are (currently) impossible with refcount_bits=1,
-_unsupported_imgopts 'refcount_bits=1[^0-9]'
+# and generally impossible with external data files
+_unsupported_imgopts 'refcount_bits=1[^0-9]' data_file
 
 echo
 echo "creating image"

tests/qemu-iotests/019

@@ -31,8 +31,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.base"
+    _rm_test_img "$TEST_IMG.base"
-    rm -f "$TEST_IMG.orig"
+    _rm_test_img "$TEST_IMG.orig"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/020

@@ -29,8 +29,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.base"
+    _rm_test_img "$TEST_IMG.base"
-    rm -f "$TEST_IMG.orig"
+    _rm_test_img "$TEST_IMG.orig"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/024

@@ -29,12 +29,12 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_DIR/t.$IMGFMT.base_old"
+    _rm_test_img "$TEST_DIR/t.$IMGFMT.base_old"
-    rm -f "$TEST_DIR/t.$IMGFMT.base_new"
+    _rm_test_img "$TEST_DIR/t.$IMGFMT.base_new"
 
-    rm -f "$TEST_DIR/subdir/t.$IMGFMT"
+    _rm_test_img "$TEST_DIR/subdir/t.$IMGFMT"
-    rm -f "$TEST_DIR/subdir/t.$IMGFMT.base_old"
+    _rm_test_img "$TEST_DIR/subdir/t.$IMGFMT.base_old"
-    rm -f "$TEST_DIR/subdir/t.$IMGFMT.base_new"
+    _rm_test_img "$TEST_DIR/subdir/t.$IMGFMT.base_new"
     rmdir "$TEST_DIR/subdir" 2> /dev/null
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/026

@@ -49,7 +49,10 @@ _supported_cache_modes writethrough none
 # 32 and 64 bits do not work either, however, due to different leaked cluster
 # count on error.
 # Thus, the only remaining option is refcount_bits=16.
-_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
+#
+# As for data_file, none of the refcount tests can work for it.
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' \
+    data_file
 
 echo "Errors while writing 128 kB"
 echo

tests/qemu-iotests/028

@@ -32,7 +32,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_qemu
-    rm -f "${TEST_IMG}.copy"
+    _rm_test_img "${TEST_IMG}.copy"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/029

@@ -28,7 +28,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-    rm -f $TEST_IMG.snap
+    _rm_test_img "$TEST_IMG.snap"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -42,8 +42,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto generic
 _unsupported_proto vxhs
-# Internal snapshots are (currently) impossible with refcount_bits=1
+# Internal snapshots are (currently) impossible with refcount_bits=1,
-_unsupported_imgopts 'refcount_bits=1[^0-9]'
+# and generally impossible with external data files
+_unsupported_imgopts 'refcount_bits=1[^0-9]' data_file
 
 offset_size=24
 offset_l1_size=36

tests/qemu-iotests/031

@@ -40,19 +40,22 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto file
+# We want to test compat=0.10, which does not support external data
+# files or refcount widths other than 16
+_unsupported_imgopts data_file 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 CLUSTER_SIZE=65536
 
 # qcow2.py output depends on the exact options used, so override the command
 # line here as an exception
-for IMGOPTS in "compat=0.10" "compat=1.1"; do
+for compat in "compat=0.10" "compat=1.1"; do
 
     echo
-    echo ===== Testing with -o $IMGOPTS =====
+    echo ===== Testing with -o $compat =====
     echo
     echo === Create image with unknown header extension ===
     echo
-    _make_test_img 64M
+    _make_test_img -o $compat 64M
     $PYTHON qcow2.py "$TEST_IMG" add-header-ext 0x12345678 "This is a test header extension"
     $PYTHON qcow2.py "$TEST_IMG" dump-header
     _check_test_img

tests/qemu-iotests/031.out

@@ -18,9 +18,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -46,9 +46,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -74,9 +74,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -109,9 +109,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -142,9 +142,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -175,9 +175,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 

tests/qemu-iotests/036

@@ -43,9 +43,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto file
-
+# Only qcow2v3 and later supports feature bits;
-# Only qcow2v3 and later supports feature bits
+# qcow2.py does not support external data files
-IMGOPTS="compat=1.1"
+_unsupported_imgopts 'compat=0.10' data_file
 
 echo
 echo === Image with unknown incompatible feature bit ===
@@ -55,7 +55,8 @@ $PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 63
 
 # Without feature table
 $PYTHON qcow2.py "$TEST_IMG" del-header-ext 0x6803f857
-$PYTHON qcow2.py "$TEST_IMG" dump-header
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep features
+$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
 _img_info
 
 # With feature table containing bit 63
@@ -103,14 +104,16 @@ echo === Create image with unknown autoclear feature bit ===
 echo
 _make_test_img 64M
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit autoclear 63
-$PYTHON qcow2.py "$TEST_IMG" dump-header
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep features
+$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
 
 echo
 echo === Repair image ===
 echo
 _check_test_img -r all
 
-$PYTHON qcow2.py "$TEST_IMG" dump-header
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep features
+$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
 
 # success, all done
 echo "*** done"

tests/qemu-iotests/036.out

@@ -3,25 +3,9 @@ QA output created by 036
 === Image with unknown incompatible feature bit ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-magic                     0x514649fb
+incompatible_features     [63]
-version                   3
+compatible_features       []
-backing_file_offset       0x0
+autoclear_features        []
-backing_file_size         0x0
-cluster_bits              16
-size                      67108864
-crypt_method              0
-l1_size                   1
-l1_table_offset           0x30000
-refcount_table_offset     0x10000
-refcount_table_clusters   1
-nb_snapshots              0
-snapshot_offset           0x0
-incompatible_features     0x8000000000000000
-compatible_features       0x0
-autoclear_features        0x0
-refcount_order            4
-header_length             104
-
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Unknown incompatible feature: 8000000000000000
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature
 
@@ -37,25 +21,9 @@ qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): tes
 === Create image with unknown autoclear feature bit ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-magic                     0x514649fb
+incompatible_features     []
-version                   3
+compatible_features       []
-backing_file_offset       0x0
+autoclear_features        [63]
-backing_file_size         0x0
-cluster_bits              16
-size                      67108864
-crypt_method              0
-l1_size                   1
-l1_table_offset           0x30000
-refcount_table_offset     0x10000
-refcount_table_clusters   1
-nb_snapshots              0
-snapshot_offset           0x0
-incompatible_features     0x0
-compatible_features       0x0
-autoclear_features        0x8000000000000000
-refcount_order            4
-header_length             104
-
 Header extension:
 magic                     0x6803f857
 length                    192
@@ -65,25 +33,9 @@ data                      <binary>
 === Repair image ===
 
 No errors were found on the image.
-magic                     0x514649fb
+incompatible_features     []
-version                   3
+compatible_features       []
-backing_file_offset       0x0
+autoclear_features        []
-backing_file_size         0x0
-cluster_bits              16
-size                      67108864
-crypt_method              0
-l1_size                   1
-l1_table_offset           0x30000
-refcount_table_offset     0x10000
-refcount_table_clusters   1
-nb_snapshots              0
-snapshot_offset           0x0
-incompatible_features     0x0
-compatible_features       0x0
-autoclear_features        0x0
-refcount_order            4
-header_length             104
-
 Header extension:
 magic                     0x6803f857
 length                    192

tests/qemu-iotests/039

@@ -44,14 +44,16 @@ _supported_proto file
 _supported_os Linux
 _default_cache_mode writethrough
 _supported_cache_modes writethrough
+# Some of these test cases expect no external data file so that all
+# clusters are part of the qcow2 image and refcounted
+_unsupported_imgopts data_file
 
 size=128M
 
 echo
 echo "== Checking that image is clean on shutdown =="
 
-IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img -o "compat=1.1,lazy_refcounts=on" $size
-_make_test_img $size
 
 $QEMU_IO -c "write -P 0x5a 0 512" "$TEST_IMG" | _filter_qemu_io
 
@@ -62,8 +64,7 @@ _check_test_img
 echo
 echo "== Creating a dirty image file =="
 
-IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img -o "compat=1.1,lazy_refcounts=on" $size
-_make_test_img $size
 
 _NO_VALGRIND \
 $QEMU_IO -c "write -P 0x5a 0 512" \
@@ -98,8 +99,7 @@ $QEMU_IO -c "read -P 0x5a 0 512" "$TEST_IMG" | _filter_qemu_io
 echo
 echo "== Opening a dirty image read/write should repair it =="
 
-IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img -o "compat=1.1,lazy_refcounts=on" $size
-_make_test_img $size
 
 _NO_VALGRIND \
 $QEMU_IO -c "write -P 0x5a 0 512" \
@@ -117,8 +117,7 @@ $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 echo
 echo "== Creating an image file with lazy_refcounts=off =="
 
-IMGOPTS="compat=1.1,lazy_refcounts=off"
+_make_test_img -o "compat=1.1,lazy_refcounts=off" $size
-_make_test_img $size
 
 _NO_VALGRIND \
 $QEMU_IO -c "write -P 0x5a 0 512" \
@@ -132,11 +131,9 @@ _check_test_img
 echo
 echo "== Committing to a backing file with lazy_refcounts=on =="
 
-IMGOPTS="compat=1.1,lazy_refcounts=on"
+TEST_IMG="$TEST_IMG".base _make_test_img -o "compat=1.1,lazy_refcounts=on" $size
-TEST_IMG="$TEST_IMG".base _make_test_img $size
 
-IMGOPTS="compat=1.1,lazy_refcounts=on,backing_file=$TEST_IMG.base"
+_make_test_img -o "compat=1.1,lazy_refcounts=on,backing_file=$TEST_IMG.base" $size
-_make_test_img $size
 
 $QEMU_IO -c "write 0 512" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG commit "$TEST_IMG"
@@ -151,8 +148,7 @@ TEST_IMG="$TEST_IMG".base _check_test_img
 echo
 echo "== Changing lazy_refcounts setting at runtime =="
 
-IMGOPTS="compat=1.1,lazy_refcounts=off"
+_make_test_img -o "compat=1.1,lazy_refcounts=off" $size
-_make_test_img $size
 
 _NO_VALGRIND \
 $QEMU_IO -c "reopen -o lazy-refcounts=on" \
@@ -164,8 +160,7 @@ $QEMU_IO -c "reopen -o lazy-refcounts=on" \
 $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 _check_test_img
 
-IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img -o "compat=1.1,lazy_refcounts=on" $size
-_make_test_img $size
 
 _NO_VALGRIND \
 $QEMU_IO -c "reopen -o lazy-refcounts=off" \

tests/qemu-iotests/039.out

@@ -4,7 +4,7 @@ QA output created by 039
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-incompatible_features     0x0
+incompatible_features     []
 No errors were found on the image.
 
 == Creating a dirty image file ==
@@ -12,7 +12,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./common.rc: Killed                  ( VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@" )
-incompatible_features     0x1
+incompatible_features     [0]
 ERROR cluster 5 refcount=0 reference=1
 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
 
@@ -22,7 +22,7 @@ Data may be corrupted, or further writes to the image may corrupt it.
 == Read-only access must still work ==
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-incompatible_features     0x1
+incompatible_features     [0]
 
 == Repairing the image file must succeed ==
 ERROR cluster 5 refcount=0 reference=1
@@ -36,7 +36,7 @@ The following inconsistencies were found and repaired:
 
 Double checking the fixed image now...
 No errors were found on the image.
-incompatible_features     0x0
+incompatible_features     []
 
 == Data should still be accessible after repair ==
 read 512/512 bytes at offset 0
@@ -47,21 +47,21 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./common.rc: Killed                  ( VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@" )
-incompatible_features     0x1
+incompatible_features     [0]
 ERROR cluster 5 refcount=0 reference=1
 Rebuilding refcount structure
 Repairing cluster 1 refcount=1 reference=0
 Repairing cluster 2 refcount=1 reference=0
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-incompatible_features     0x0
+incompatible_features     []
 
 == Creating an image file with lazy_refcounts=off ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./common.rc: Killed                  ( VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@" )
-incompatible_features     0x0
+incompatible_features     []
 No errors were found on the image.
 
 == Committing to a backing file with lazy_refcounts=on ==
@@ -70,8 +70,8 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Image committed.
-incompatible_features     0x0
+incompatible_features     []
-incompatible_features     0x0
+incompatible_features     []
 No errors were found on the image.
 No errors were found on the image.
 
@@ -80,7 +80,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./common.rc: Killed                  ( VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@" )
-incompatible_features     0x1
+incompatible_features     [0]
 ERROR cluster 5 refcount=0 reference=1
 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
 
@@ -90,6 +90,6 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./common.rc: Killed                  ( VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@" )
-incompatible_features     0x0
+incompatible_features     []
 No errors were found on the image.
 *** done

tests/qemu-iotests/041

@@ -1121,6 +1121,50 @@ class TestOrphanedSource(iotests.QMPTestCase):
                              target='dest-ro')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
+    def test_failing_permission_in_complete(self):
+        self.assert_no_active_block_jobs()
+
+        # Unshare consistent-read on the target
+        # (The mirror job does not care)
+        result = self.vm.qmp('blockdev-add',
+                             driver='blkdebug',
+                             node_name='dest-perm',
+                             image='dest',
+                             unshare_child_perms=['consistent-read'])
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-mirror', job_id='job', device='src',
+                             sync='full', target='dest',
+                             filter_node_name='mirror-filter')
+        self.assert_qmp(result, 'return', {})
+
+        # Require consistent-read on the source
+        # (We can only add this node once the job has started, or it
+        # will complain that it does not want to run on non-root nodes)
+        result = self.vm.qmp('blockdev-add',
+                             driver='blkdebug',
+                             node_name='src-perm',
+                             image='src',
+                             take_child_perms=['consistent-read'])
+        self.assert_qmp(result, 'return', {})
+
+        # While completing, mirror will attempt to replace src by
+        # dest, which must fail because src-perm requires
+        # consistent-read but dest-perm does not share it; thus
+        # aborting the job when it is supposed to complete
+        self.complete_and_wait('job',
+                               completion_error='Operation not permitted')
+
+        # Assert that all of our nodes are still there (except for the
+        # mirror filter, which should be gone despite the failure)
+        nodes = self.vm.qmp('query-named-block-nodes')['return']
+        nodes = [node['node-name'] for node in nodes]
+
+        for expect in ('src', 'src-perm', 'dest', 'dest-perm'):
+            self.assertTrue(expect in nodes, '%s disappeared' % expect)
+        self.assertFalse('mirror-filter' in nodes,
+                         'Mirror filter node did not disappear')
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'],
                  supported_protocols=['file'])

tests/qemu-iotests/041.out

@@ -1,5 +1,5 @@
-..........................................................................................
+...........................................................................................
 ----------------------------------------------------------------------
-Ran 90 tests
+Ran 91 tests
 
 OK

tests/qemu-iotests/043

@@ -29,7 +29,9 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG".[123].base
+    for img in "$TEST_IMG".[123].base; do
+        _rm_test_img "$img"
+    done
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/046

@@ -38,6 +38,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt qcow2
 _supported_proto file
+# data_file does not support compressed clusters
+_unsupported_imgopts data_file
 
 CLUSTER_SIZE=64k
 size=128M

tests/qemu-iotests/048

@@ -31,7 +31,7 @@ _cleanup()
 {
     echo "Cleanup"
     _cleanup_test_img
-    rm "${TEST_IMG_FILE2}"
+    _rm_test_img "${TEST_IMG_FILE2}"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -49,6 +49,8 @@ _compare()
 _supported_fmt raw qcow2 qed luks
 _supported_proto file
 _supported_os Linux
+# Using 'cp' is incompatible with external data files
+_unsupported_imgopts data_file
 
 # Remove once all tests are fixed to use TEST_IMG_FILE
 # correctly and common.rc sets it unconditionally

tests/qemu-iotests/050

@@ -29,8 +29,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.old"
+    _rm_test_img "$TEST_IMG.old"
-    rm -f "$TEST_IMG.new"
+    _rm_test_img "$TEST_IMG.new"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -41,10 +41,6 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2 qed
 _supported_proto file
 
-if test "$IMGFMT" = qcow2 && test $IMGOPTS = ""; then
-  IMGOPTS=compat=1.1
-fi
-
 echo
 echo "== Creating images =="
 

tests/qemu-iotests/051

@@ -39,8 +39,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 # A compat=0.10 image is created in this test which does not support anything
-# other than refcount_bits=16
+# other than refcount_bits=16;
-_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
+# it also will not support an external data file
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
 _require_drivers nbd
 
 do_run_qemu()
@@ -158,7 +159,7 @@ echo
 echo === With version 2 images enabling lazy refcounts must fail ===
 echo
 
-_make_test_img -ocompat=0.10 $size
+_make_test_img -o compat=0.10 $size
 
 run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=on
 run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=off

tests/qemu-iotests/053

@@ -28,7 +28,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-	rm -f "$TEST_IMG.orig"
+    _rm_test_img "$TEST_IMG.orig"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/058

@@ -42,7 +42,7 @@ _cleanup()
 {
     nbd_server_stop
     _cleanup_test_img
-    rm -f "$converted_image"
+    _rm_test_img "$converted_image"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -56,8 +56,9 @@ _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
 _require_command QEMU_NBD
-# Internal snapshots are (currently) impossible with refcount_bits=1
+# Internal snapshots are (currently) impossible with refcount_bits=1,
-_unsupported_imgopts 'refcount_bits=1[^0-9]'
+# and generally impossible with external data files
+_unsupported_imgopts 'refcount_bits=1[^0-9]' data_file
 
 nbd_snapshot_img="nbd:unix:$nbd_unix_socket"
 

tests/qemu-iotests/059

@@ -29,7 +29,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.qcow2"
+    IMGFMT=qcow2 _rm_test_img "$TEST_IMG.qcow2"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -70,18 +70,18 @@ poke_file "$TEST_IMG" "$grain_table_size_offset" "\x01\x00\x00\x00"
 
 echo
 echo "=== Testing monolithicFlat creation and opening ==="
-IMGOPTS="subformat=monolithicFlat" _make_test_img 2G
+_make_test_img -o "subformat=monolithicFlat" 2G
 _img_info
 _cleanup_test_img
 
 echo
 echo "=== Testing monolithicFlat with zeroed_grain ==="
-IMGOPTS="subformat=monolithicFlat,zeroed_grain=on" _make_test_img 2G
+_make_test_img -o "subformat=monolithicFlat,zeroed_grain=on" 2G
 _cleanup_test_img
 
 echo
 echo "=== Testing big twoGbMaxExtentFlat ==="
-IMGOPTS="subformat=twoGbMaxExtentFlat" _make_test_img 1000G
+_make_test_img -o "subformat=twoGbMaxExtentFlat" 1000G
 $QEMU_IMG info $TEST_IMG | _filter_testdir | sed -e 's/cid: [0-9]*/cid: XXXXXXXX/'
 _cleanup_test_img
 
@@ -101,13 +101,13 @@ _img_info
 
 echo
 echo "=== Testing truncated sparse ==="
-IMGOPTS="subformat=monolithicSparse" _make_test_img 100G
+_make_test_img -o "subformat=monolithicSparse" 100G
 truncate -s 10M $TEST_IMG
 _img_info
 
 echo
 echo "=== Converting to streamOptimized from image with small cluster size==="
-TEST_IMG="$TEST_IMG.qcow2" IMGFMT=qcow2 IMGOPTS="cluster_size=4096" _make_test_img 1G
+TEST_IMG="$TEST_IMG.qcow2" IMGFMT=qcow2 _make_test_img -o "cluster_size=4096" 1G
 $QEMU_IO -f qcow2 -c "write -P 0xa 0 512" "$TEST_IMG.qcow2" | _filter_qemu_io
 $QEMU_IO -f qcow2 -c "write -P 0xb 10240 512" "$TEST_IMG.qcow2" | _filter_qemu_io
 $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2" "$TEST_IMG" 2>&1
@@ -117,7 +117,7 @@ echo "=== Testing monolithicFlat with internally generated JSON file name ==="
 
 echo '--- blkdebug ---'
 # Should work, because bdrv_dirname() works fine with blkdebug
-IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
+_make_test_img -o "subformat=monolithicFlat" 64M
 $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
          -c info \
     2>&1 \
@@ -126,7 +126,7 @@ _cleanup_test_img
 
 echo '--- quorum ---'
 # Should not work, because bdrv_dirname() does not work with quorum
-IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
+_make_test_img -o "subformat=monolithicFlat" 64M
 cp "$TEST_IMG" "$TEST_IMG.orig"
 
 filename="json:{
@@ -161,7 +161,7 @@ _cleanup_test_img
 
 echo
 echo "=== Testing 4TB monolithicFlat creation and IO ==="
-IMGOPTS="subformat=monolithicFlat" _make_test_img 4T
+_make_test_img -o "subformat=monolithicFlat" 4T
 _img_info
 $QEMU_IO -c "write -P 0xa 900G 512" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "read -v 900G 1024" "$TEST_IMG" | _filter_qemu_io
@@ -170,7 +170,7 @@ _cleanup_test_img
 echo
 echo "=== Testing qemu-img map on extents ==="
 for fmt in monolithicSparse twoGbMaxExtentSparse; do
-    IMGOPTS="subformat=$fmt" _make_test_img 31G
+    _make_test_img -o "subformat=$fmt" 31G
     $QEMU_IO -c "write 65024 1k" "$TEST_IMG" | _filter_qemu_io
     $QEMU_IO -c "write 2147483136 1k" "$TEST_IMG" | _filter_qemu_io
     $QEMU_IO -c "write 5G 1k" "$TEST_IMG" | _filter_qemu_io

tests/qemu-iotests/060

@@ -44,10 +44,14 @@ _filter_io_error()
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# These tests only work for compat=1.1 images without an external
+# data file with refcount_bits=16
+_unsupported_imgopts 'compat=0.10' data_file \
+    'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 # The repair process will create a large file - so check for availability first
 _require_large_file 64G
@@ -58,8 +62,6 @@ l1_offset=196608 # 0x30000 (XXX: just an assumption)
 l2_offset=262144 # 0x40000 (XXX: just an assumption)
 l2_offset_after_snapshot=524288 # 0x80000 (XXX: just an assumption)
 
-IMGOPTS="compat=1.1"
-
 OPEN_RW="open -o overlap-check=all $TEST_IMG"
 # Overlap checks are done before write operations only, therefore opening an
 # image read-only makes the overlap-check option irrelevant
@@ -161,7 +163,7 @@ $QEMU_IO -c 'write 0k 64k' "$BACKING_IMG" | _filter_qemu_io
 # compat=0.10 is required in order to make the following discard actually
 # unallocate the sector rather than make it a zero sector - we want COW, after
 # all.
-IMGOPTS='compat=0.10' _make_test_img -b "$BACKING_IMG" 1G
+_make_test_img -o 'compat=0.10' -b "$BACKING_IMG" 1G
 # Write two clusters, the second one enforces creation of an L2 table after
 # the first data cluster.
 $QEMU_IO -c 'write 0k 64k' -c 'write 512M 64k' "$TEST_IMG" | _filter_qemu_io
@@ -401,7 +403,7 @@ echo
 echo "=== Discarding a non-covered in-bounds refblock ==="
 echo
 
-IMGOPTS='refcount_bits=1' _make_test_img 64M
+_make_test_img -o 'refcount_bits=1' 64M
 
 # Pretend there's a refblock somewhere where there is no refblock to
 # cover it (but the covering refblock has a valid index in the
@@ -425,7 +427,7 @@ echo
 echo "=== Discarding a refblock covered by an unaligned refblock ==="
 echo
 
-IMGOPTS='refcount_bits=1' _make_test_img 64M
+_make_test_img -o 'refcount_bits=1' 64M
 
 # Same as above
 poke_file "$TEST_IMG" "$(($rt_offset+8))" "\x00\x00\x00\x10\x00\x00\x00\x00"

tests/qemu-iotests/060.out

@@ -7,10 +7,10 @@ ERROR cluster 3 refcount=1 reference=3
 
 1 errors were found on the image.
 Data may be corrupted, or further writes to the image may corrupt it.
-incompatible_features     0x0
+incompatible_features     []
 qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with active L1 table); further corruption events will be suppressed
 write failed: Input/output error
-incompatible_features     0x2
+incompatible_features     [1]
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64 MiB (67108864 bytes)
@@ -33,10 +33,10 @@ ERROR cluster 2 refcount=1 reference=2
 
 2 errors were found on the image.
 Data may be corrupted, or further writes to the image may corrupt it.
-incompatible_features     0x0
+incompatible_features     []
 qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with refcount block); further corruption events will be suppressed
 write failed: Input/output error
-incompatible_features     0x2
+incompatible_features     [1]
 ERROR refcount block 0 refcount=2
 ERROR cluster 2 refcount=1 reference=2
 Rebuilding refcount structure
@@ -49,10 +49,10 @@ The following inconsistencies were found and repaired:
 
 Double checking the fixed image now...
 No errors were found on the image.
-incompatible_features     0x0
+incompatible_features     []
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-incompatible_features     0x0
+incompatible_features     []
 
 === Testing cluster data reference into inactive L2 table ===
 
@@ -69,10 +69,10 @@ Data may be corrupted, or further writes to the image may corrupt it.
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-incompatible_features     0x0
+incompatible_features     []
 qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with inactive L2 table); further corruption events will be suppressed
 write failed: Input/output error
-incompatible_features     0x2
+incompatible_features     [1]
 ERROR cluster 4 refcount=1 reference=2
 Leaked cluster 9 refcount=1 reference=0
 Repairing cluster 4 refcount=1 reference=2
@@ -85,10 +85,10 @@ The following inconsistencies were found and repaired:
 
 Double checking the fixed image now...
 No errors were found on the image.
-incompatible_features     0x0
+incompatible_features     []
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-incompatible_features     0x0
+incompatible_features     []
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.

tests/qemu-iotests/061

@@ -29,7 +29,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f $TEST_IMG.data
+    _rm_test_img "$TEST_IMG.data"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -37,15 +37,20 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# Conversion between different compat versions can only really work
+# with refcount_bits=16;
+# we have explicit tests for data_file here, but the whole test does
+# not work with it
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
 
 echo
 echo "=== Testing version downgrade with zero expansion ==="
 echo
-IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+_make_test_img -o "compat=1.1,lazy_refcounts=on" 64M
 $QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
@@ -56,7 +61,7 @@ _check_test_img
 echo
 echo "=== Testing version downgrade with zero expansion and 4K cache entries ==="
 echo
-IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+_make_test_img -o "compat=1.1,lazy_refcounts=on" 64M
 $QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "write -z 32M 128k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c map "$TEST_IMG" | _filter_qemu_io
@@ -72,7 +77,7 @@ _check_test_img
 echo
 echo "=== Testing dirty version downgrade ==="
 echo
-IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+_make_test_img -o "compat=1.1,lazy_refcounts=on" 64M
 _NO_VALGRIND \
 $QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
          -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
@@ -85,7 +90,7 @@ _check_test_img
 echo
 echo "=== Testing version downgrade with unknown compat/autoclear flags ==="
 echo
-IMGOPTS="compat=1.1" _make_test_img 64M
+_make_test_img -o "compat=1.1" 64M
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit compatible 42
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit autoclear 42
 $PYTHON qcow2.py "$TEST_IMG" dump-header
@@ -96,7 +101,7 @@ _check_test_img
 echo
 echo "=== Testing version upgrade and resize ==="
 echo
-IMGOPTS="compat=0.10" _make_test_img 64M
+_make_test_img -o "compat=0.10" 64M
 $QEMU_IO -c "write -P 0x2a 42M 64k" "$TEST_IMG" | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "compat=1.1,lazy_refcounts=on,size=128M" "$TEST_IMG"
@@ -107,7 +112,7 @@ _check_test_img
 echo
 echo "=== Testing dirty lazy_refcounts=off ==="
 echo
-IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+_make_test_img -o "compat=1.1,lazy_refcounts=on" 64M
 _NO_VALGRIND \
 $QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
          -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
@@ -120,8 +125,8 @@ _check_test_img
 echo
 echo "=== Testing backing file ==="
 echo
-IMGOPTS="compat=1.1" _make_test_img 64M
+_make_test_img -o "compat=1.1" 64M
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+TEST_IMG="$TEST_IMG.base" _make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
 $QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG amend -o "backing_file=$TEST_IMG.base,backing_fmt=qcow2" "$TEST_IMG"
@@ -131,7 +136,7 @@ _check_test_img
 echo
 echo "=== Testing invalid configurations ==="
 echo
-IMGOPTS="compat=0.10" _make_test_img 64M
+_make_test_img -o "compat=0.10" 64M
 $QEMU_IMG amend -o "lazy_refcounts=on" "$TEST_IMG"
 $QEMU_IMG amend -o "compat=1.1" "$TEST_IMG" # actually valid
 $QEMU_IMG amend -o "compat=0.10,lazy_refcounts=on" "$TEST_IMG"
@@ -144,7 +149,7 @@ $QEMU_IMG amend -o "preallocation=on" "$TEST_IMG"
 echo
 echo "=== Testing correct handling of unset value ==="
 echo
-IMGOPTS="compat=1.1,cluster_size=1k" _make_test_img 64M
+_make_test_img -o "compat=1.1,cluster_size=1k" 64M
 echo "Should work:"
 $QEMU_IMG amend -o "lazy_refcounts=on" "$TEST_IMG"
 echo "Should not work:" # Just to know which of these tests actually fails
@@ -153,7 +158,7 @@ $QEMU_IMG amend -o "cluster_size=64k" "$TEST_IMG"
 echo
 echo "=== Testing zero expansion on inactive clusters ==="
 echo
-IMGOPTS="compat=1.1" _make_test_img 64M
+_make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG snapshot -c foo "$TEST_IMG"
 $QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG" | _filter_qemu_io
@@ -167,7 +172,7 @@ $QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
 echo
 echo "=== Testing zero expansion on shared L2 table ==="
 echo
-IMGOPTS="compat=1.1" _make_test_img 64M
+_make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG snapshot -c foo "$TEST_IMG"
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
@@ -180,9 +185,9 @@ $QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
 echo
 echo "=== Testing zero expansion on backed image ==="
 echo
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+TEST_IMG="$TEST_IMG.base" _make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
-IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 64M
+_make_test_img -o "compat=1.1" -b "$TEST_IMG.base" 64M
 $QEMU_IO -c "read -P 0x2a 0 128k" -c "write -z 0 64k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 _check_test_img
@@ -191,9 +196,9 @@ $QEMU_IO -c "read -P 0 0 64k" -c "read -P 0x2a 64k 64k" "$TEST_IMG" | _filter_qe
 echo
 echo "=== Testing zero expansion on backed inactive clusters ==="
 echo
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+TEST_IMG="$TEST_IMG.base" _make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
-IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 64M
+_make_test_img -o "compat=1.1" -b "$TEST_IMG.base" 64M
 $QEMU_IO -c "write -z 0 64k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG snapshot -c foo "$TEST_IMG"
 $QEMU_IO -c "write -P 0x42 0 128k" "$TEST_IMG" | _filter_qemu_io
@@ -207,9 +212,9 @@ $QEMU_IO -c "read -P 0 0 64k" -c "read -P 0x2a 64k 64k" "$TEST_IMG" | _filter_qe
 echo
 echo "=== Testing zero expansion on backed image with shared L2 table ==="
 echo
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+TEST_IMG="$TEST_IMG.base" _make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
-IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 64M
+_make_test_img -o "compat=1.1" -b "$TEST_IMG.base" 64M
 $QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IMG snapshot -c foo "$TEST_IMG"
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
@@ -222,7 +227,7 @@ $QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
 echo
 echo "=== Testing preallocated zero expansion on full image ==="
 echo
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG" _make_test_img 64M
+TEST_IMG="$TEST_IMG" _make_test_img -o "compat=1.1" 64M
 $QEMU_IO -c "write -P 0x2a 0 64M" "$TEST_IMG" -c "write -z 0 64M" | _filter_qemu_io
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 _check_test_img
@@ -231,8 +236,8 @@ $QEMU_IO -c "read -P 0 0 64M" "$TEST_IMG" | _filter_qemu_io
 echo
 echo "=== Testing progress report without snapshot ==="
 echo
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 4G
+TEST_IMG="$TEST_IMG.base" _make_test_img -o "compat=1.1" 4G
-IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 4G
+_make_test_img -o "compat=1.1" -b "$TEST_IMG.base" 4G
 $QEMU_IO -c "write -z 0  64k" \
          -c "write -z 1G 64k" \
          -c "write -z 2G 64k" \
@@ -243,8 +248,8 @@ _check_test_img
 echo
 echo "=== Testing progress report with snapshot ==="
 echo
-IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 4G
+TEST_IMG="$TEST_IMG.base" _make_test_img -o "compat=1.1" 4G
-IMGOPTS="compat=1.1" _make_test_img -b "$TEST_IMG.base" 4G
+_make_test_img -o "compat=1.1" -b "$TEST_IMG.base" 4G
 $QEMU_IO -c "write -z 0  64k" \
          -c "write -z 1G 64k" \
          -c "write -z 2G 64k" \
@@ -256,7 +261,7 @@ _check_test_img
 echo
 echo "=== Testing version downgrade with external data file ==="
 echo
-IMGOPTS="compat=1.1,data_file=$TEST_IMG.data" _make_test_img 64M
+_make_test_img -o "compat=1.1,data_file=$TEST_IMG.data" 64M
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 _img_info --format-specific
 _check_test_img
@@ -264,11 +269,11 @@ _check_test_img
 echo
 echo "=== Try changing the external data file ==="
 echo
-IMGOPTS="compat=1.1" _make_test_img 64M
+_make_test_img -o "compat=1.1" 64M
 $QEMU_IMG amend -o "data_file=foo" "$TEST_IMG"
 
 echo
-IMGOPTS="compat=1.1,data_file=$TEST_IMG.data" _make_test_img 64M
+_make_test_img -o "compat=1.1,data_file=$TEST_IMG.data" 64M
 $QEMU_IMG amend -o "data_file=foo" "$TEST_IMG"
 _img_info --format-specific
 TEST_IMG="data-file.filename=$TEST_IMG.data,file.filename=$TEST_IMG" _img_info --format-specific --image-opts
@@ -281,7 +286,7 @@ TEST_IMG="data-file.filename=$TEST_IMG.data,file.filename=$TEST_IMG" _img_info -
 echo
 echo "=== Clearing and setting data-file-raw ==="
 echo
-IMGOPTS="compat=1.1,data_file=$TEST_IMG.data,data_file_raw=on" _make_test_img 64M
+_make_test_img -o "compat=1.1,data_file=$TEST_IMG.data,data_file_raw=on" 64M
 $QEMU_IMG amend -o "data_file_raw=on" "$TEST_IMG"
 _img_info --format-specific
 _check_test_img

tests/qemu-iotests/061.out

@@ -18,9 +18,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x1
+compatible_features       [0]
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -42,9 +42,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -76,9 +76,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x1
+compatible_features       [0]
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -100,9 +100,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -132,9 +132,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x1
+incompatible_features     [0]
-compatible_features       0x1
+compatible_features       [0]
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -161,9 +161,9 @@ refcount_table_offset     0x80000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -187,9 +187,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x40000000000
+compatible_features       [42]
-autoclear_features        0x40000000000
+autoclear_features        [42]
 refcount_order            4
 header_length             104
 
@@ -211,9 +211,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -237,9 +237,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             72
 
@@ -256,9 +256,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x1
+compatible_features       [0]
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -290,9 +290,9 @@ refcount_table_offset     0x10000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x1
+incompatible_features     [0]
-compatible_features       0x1
+compatible_features       [0]
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 
@@ -319,9 +319,9 @@ refcount_table_offset     0x80000
 refcount_table_clusters   1
 nb_snapshots              0
 snapshot_offset           0x0
-incompatible_features     0x0
+incompatible_features     []
-compatible_features       0x0
+compatible_features       []
-autoclear_features        0x0
+autoclear_features        []
 refcount_order            4
 header_length             104
 

tests/qemu-iotests/062

@@ -37,11 +37,12 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto generic
+# We need zero clusters and snapshots
+_unsupported_imgopts 'compat=0.10' 'refcount_bits=1[^0-9]' data_file
 
-IMGOPTS="compat=1.1"
 IMG_SIZE=64M
 
 echo

tests/qemu-iotests/063

@@ -30,7 +30,9 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-	rm -f "$TEST_IMG.orig" "$TEST_IMG.raw1" "$TEST_IMG.raw2"
+    for img in "$TEST_IMG".{orig,raw1,raw2,target}; do
+        _rm_test_img "$img"
+    done
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -49,15 +51,13 @@ _unsupported_imgopts "subformat=monolithicFlat" \
 _make_test_img 4M
 
 echo "== Testing conversion with -n fails with no target file =="
-# check .orig file does not exist
-rm -f "$TEST_IMG.orig"
 if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG" "$TEST_IMG.orig" >/dev/null 2>&1; then
     exit 1
 fi
 
 echo "== Testing conversion with -n succeeds with a target file =="
-rm -f "$TEST_IMG.orig"
+_rm_test_img "$TEST_IMG.orig"
-cp "$TEST_IMG" "$TEST_IMG.orig"
+TEST_IMG="$TEST_IMG.orig" _make_test_img 4M
 if ! $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG" "$TEST_IMG.orig" ; then
     exit 1
 fi
@@ -83,10 +83,8 @@ fi
 _check_test_img
 
 echo "== Testing conversion to a smaller file fails =="
-rm -f "$TEST_IMG.orig"
+TEST_IMG="$TEST_IMG.target" _make_test_img 2M
-mv "$TEST_IMG" "$TEST_IMG.orig"
+if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG" "$TEST_IMG.target" >/dev/null 2>&1; then
-_make_test_img 2M
-if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG.orig" "$TEST_IMG" >/dev/null 2>&1; then
     exit 1
 fi
 

tests/qemu-iotests/063.out

@@ -2,11 +2,12 @@ QA output created by 063
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304
 == Testing conversion with -n fails with no target file ==
 == Testing conversion with -n succeeds with a target file ==
+Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=4194304
 == Testing conversion to raw is the same after conversion with -n ==
 == Testing conversion back to original format ==
 No errors were found on the image.
 == Testing conversion to a smaller file fails ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2097152
+Formatting 'TEST_DIR/t.IMGFMT.target', fmt=IMGFMT size=2097152
 == Regression testing for copy offloading bug ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
 Formatting 'TEST_DIR/t.IMGFMT.target', fmt=IMGFMT size=1048576

tests/qemu-iotests/066

@@ -36,12 +36,15 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto generic
+# We need zero clusters and snapshots
+# (TODO: Consider splitting the snapshot part into a separate test
+#        file, so this one runs with refcount_bits=1 and data_file)
+_unsupported_imgopts 'compat=0.10' 'refcount_bits=1[^0-9]' data_file
 
 # Intentionally create an unaligned image
-IMGOPTS="compat=1.1"
 IMG_SIZE=$((64 * 1024 * 1024 + 512))
 
 echo

tests/qemu-iotests/067

@@ -32,8 +32,10 @@ status=1	# failure is the default!
 
 _supported_fmt qcow2
 _supported_proto file
-# Because anything other than 16 would change the output of query-block
+# Because anything other than 16 would change the output of query-block,
-_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
+# and external data files would change the output of
+# query-named-block-nodes
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
 
 do_run_qemu()
 {

tests/qemu-iotests/068

@@ -36,11 +36,13 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto generic
+# Internal snapshots are (currently) impossible with refcount_bits=1,
+# and generally impossible with external data files
+_unsupported_imgopts 'compat=0.10' 'refcount_bits=1[^0-9]' data_file
 
-IMGOPTS="compat=1.1"
 IMG_SIZE=128K
 
 case "$QEMU_DEFAULT_MACHINE" in

tests/qemu-iotests/069

@@ -47,7 +47,7 @@ echo "=== Creating an image with a backing file and deleting that file ==="
 echo
 TEST_IMG="$TEST_IMG.base" _make_test_img $IMG_SIZE
 _make_test_img -b "$TEST_IMG.base" $IMG_SIZE
-rm -f "$TEST_IMG.base"
+_rm_test_img "$TEST_IMG.base"
 # Just open the image and close it right again (this should print an error message)
 $QEMU_IO -c quit "$TEST_IMG" 2>&1 | _filter_testdir | _filter_imgfmt
 

tests/qemu-iotests/071

@@ -39,6 +39,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _require_drivers blkdebug blkverify
+# blkdebug can only inject errors on bs->file, not on the data_file,
+# so thie test does not work with external data files
+_unsupported_imgopts data_file
 
 do_run_qemu()
 {
@@ -58,7 +61,7 @@ echo
 echo "=== Testing blkverify through filename ==="
 echo
 
-TEST_IMG="$TEST_IMG.base" IMGOPTS="" IMGFMT="raw" _make_test_img $IMG_SIZE |\
+TEST_IMG="$TEST_IMG.base" IMGFMT="raw" _make_test_img --no-opts $IMG_SIZE |\
     _filter_imgfmt
 _make_test_img $IMG_SIZE
 $QEMU_IO -c "open -o driver=raw,file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
@@ -73,7 +76,7 @@ echo
 echo "=== Testing blkverify through file blockref ==="
 echo
 
-TEST_IMG="$TEST_IMG.base" IMGOPTS="" IMGFMT="raw" _make_test_img $IMG_SIZE |\
+TEST_IMG="$TEST_IMG.base" IMGFMT="raw" _make_test_img --no-opts $IMG_SIZE |\
     _filter_imgfmt
 _make_test_img $IMG_SIZE
 $QEMU_IO -c "open -o driver=raw,file.driver=blkverify,file.raw.filename=$TEST_IMG.base,file.test.driver=$IMGFMT,file.test.file.filename=$TEST_IMG" \

tests/qemu-iotests/073

@@ -39,6 +39,10 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto generic
 _unsupported_proto vxhs
+# External data files do not support compressed clusters
+# (TODO: Consider writing a version for external data files that does
+#        not test compressed clusters)
+_unsupported_imgopts data_file
 
 CLUSTER_SIZE=64k
 size=128M

tests/qemu-iotests/074

@@ -31,7 +31,7 @@ _cleanup()
 {
     echo "Cleanup"
     _cleanup_test_img
-    rm "${TEST_IMG2}"
+    _rm_test_img "${TEST_IMG2}"
     rm -f "$TEST_DIR/blkdebug.conf"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -50,6 +50,8 @@ _compare()
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# blkdebug can only inject errors on bs->file
+_unsupported_imgopts data_file
 
 # Setup test basic parameters
 TEST_IMG2=$TEST_IMG.2

tests/qemu-iotests/079

@@ -47,8 +47,7 @@ echo
 cluster_sizes="16384 32768 65536 131072 262144 524288 1048576 2097152 4194304"
 
 for s in $cluster_sizes; do
-    IMGOPTS=$(_optstr_add "$IMGOPTS" "preallocation=metadata,cluster_size=$s") \
+    _make_test_img -o "preallocation=metadata,cluster_size=$s" 4G
-        _make_test_img 4G
 done
 
 # success, all done

tests/qemu-iotests/080

@@ -28,7 +28,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-    rm -f $TEST_IMG.snap
+    _rm_test_img "$TEST_IMG.snap"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -40,9 +40,10 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
-# - Internal snapshots are (currently) impossible with refcount_bits=1
+# - Internal snapshots are (currently) impossible with refcount_bits=1,
+#   and generally impossible with external data files
 # - This is generally a test for compat=1.1 images
-_unsupported_imgopts 'refcount_bits=1[^0-9]' 'compat=0.10'
+_unsupported_imgopts 'refcount_bits=1[^0-9]' data_file 'compat=0.10'
 
 header_size=104
 

tests/qemu-iotests/081

@@ -28,9 +28,9 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-    rm -rf $TEST_DIR/1.raw
+    _rm_test_img "$TEST_DIR/1.raw"
-    rm -rf $TEST_DIR/2.raw
+    _rm_test_img "$TEST_DIR/2.raw"
-    rm -rf $TEST_DIR/3.raw
+    _rm_test_img "$TEST_DIR/3.raw"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/085

@@ -41,10 +41,13 @@ _cleanup()
     _cleanup_qemu
     for i in $(seq 1 ${SNAPSHOTS})
     do
-        rm -f "${TEST_DIR}/${i}-${snapshot_virt0}"
+        _rm_test_img "${TEST_DIR}/${i}-${snapshot_virt0}"
-        rm -f "${TEST_DIR}/${i}-${snapshot_virt1}"
+        _rm_test_img "${TEST_DIR}/${i}-${snapshot_virt1}"
+    done
+    for img in "${TEST_IMG}".{1,2,base}
+    do
+        _rm_test_img "$img"
     done
-    rm -f "${TEST_IMG}" "${TEST_IMG}.1" "${TEST_IMG}.2" "${TEST_IMG}.base"
 
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -102,8 +105,7 @@ add_snapshot_image()
 {
     base_image="${TEST_DIR}/$((${1}-1))-${snapshot_virt0}"
     snapshot_file="${TEST_DIR}/${1}-${snapshot_virt0}"
-    _make_test_img -u -b "${base_image}" "$size"
+    TEST_IMG=$snapshot_file _make_test_img -u -b "${base_image}" "$size"
-    mv "${TEST_IMG}" "${snapshot_file}"
     do_blockdev_add "$1" "'backing': null, " "${snapshot_file}"
 }
 
@@ -119,10 +121,8 @@ blockdev_snapshot()
 
 size=128M
 
-_make_test_img $size
+TEST_IMG="$TEST_IMG.1" _make_test_img $size
-mv "${TEST_IMG}" "${TEST_IMG}.1"
+TEST_IMG="$TEST_IMG.2" _make_test_img $size
-_make_test_img $size
-mv "${TEST_IMG}" "${TEST_IMG}.2"
 
 echo
 echo === Running QEMU ===

tests/qemu-iotests/085.out

@@ -1,6 +1,6 @@
 QA output created by 085
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.1', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.2', fmt=IMGFMT size=134217728
 
 === Running QEMU ===
 
@@ -68,12 +68,12 @@ Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_fil
 
 === Create a couple of snapshots using blockdev-snapshot ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/10-snapshot-v0.IMGFMT
+Formatting 'TEST_DIR/11-snapshot-v0.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/10-snapshot-v0.IMGFMT
 { 'execute': 'blockdev-add', 'arguments': { 'driver': 'IMGFMT', 'node-name': 'snap_11', 'backing': null, 'file': { 'driver': 'file', 'filename': 'TEST_DIR/11-snapshot-v0.IMGFMT', 'node-name': 'file_11' } } }
 {"return": {}}
 { 'execute': 'blockdev-snapshot', 'arguments': { 'node': 'virtio0', 'overlay':'snap_11' } }
 {"return": {}}
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/11-snapshot-v0.IMGFMT
+Formatting 'TEST_DIR/12-snapshot-v0.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/11-snapshot-v0.IMGFMT
 { 'execute': 'blockdev-add', 'arguments': { 'driver': 'IMGFMT', 'node-name': 'snap_12', 'backing': null, 'file': { 'driver': 'file', 'filename': 'TEST_DIR/12-snapshot-v0.IMGFMT', 'node-name': 'file_12' } } }
 {"return": {}}
 { 'execute': 'blockdev-snapshot', 'arguments': { 'node': 'virtio0', 'overlay':'snap_12' } }

tests/qemu-iotests/088

@@ -28,7 +28,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-    rm -f $TEST_IMG.snap
+    _rm_test_img "$TEST_IMG.snap"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/090

@@ -38,6 +38,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt qcow2
 _supported_proto file nfs
+# External data files do not support compressed clusters
+_unsupported_imgopts data_file
 
 IMG_SIZE=128K
 

tests/qemu-iotests/091

@@ -101,7 +101,7 @@ echo "Check image pattern"
 ${QEMU_IO} -c "read -P 0x22 0 4M" "${TEST_IMG}" | _filter_testdir | _filter_qemu_io
 
 echo "Running 'qemu-img check -r all \$TEST_IMG'"
-"${QEMU_IMG}" check -r all "${TEST_IMG}" 2>&1 | _filter_testdir | _filter_qemu
+_check_test_img -r all
 
 echo "*** done"
 rm -f $seq.full

tests/qemu-iotests/091.out

@@ -23,6 +23,4 @@ read 4194304/4194304 bytes at offset 0
 4 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Running 'qemu-img check -r all $TEST_IMG'
 No errors were found on the image.
-80/16384 = 0.49% allocated, 0.00% fragmented, 0.00% compressed clusters
-Image end offset: 5570560
 *** done

tests/qemu-iotests/092

@@ -28,7 +28,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-    rm -f $TEST_IMG.snap
+    _rm_test_img "$TEST_IMG.snap"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/094

@@ -30,7 +30,7 @@ _cleanup()
 {
     _cleanup_qemu
     _cleanup_test_img
-    rm -f "$TEST_DIR/source.$IMGFMT"
+    _rm_test_img "$TEST_DIR/source.$IMGFMT"
 }
 
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -45,7 +45,7 @@ _supported_proto nbd
 _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
 
 _make_test_img 64M
-$QEMU_IMG create -f $IMGFMT "$TEST_DIR/source.$IMGFMT" 64M | _filter_img_create
+TEST_IMG_FILE="$TEST_DIR/source.$IMGFMT" IMGPROTO=file _make_test_img 64M
 
 _launch_qemu -drive if=none,id=src,file="$TEST_DIR/source.$IMGFMT",format=raw \
              -nodefaults

tests/qemu-iotests/095

@@ -32,7 +32,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_qemu
-    rm  -f "${TEST_IMG}.base" "${TEST_IMG}.snp1"
+    _rm_test_img "${TEST_IMG}.base"
+    _rm_test_img "${TEST_IMG}.snp1"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/098

@@ -40,8 +40,10 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt qcow2
 _supported_proto file
-
+# The code path we want to test here only works for compat=1.1 images;
-IMGOPTS="compat=1.1"
+# blkdebug can only inject errors on bs->file, so external data files
+# do not work with this test
+_unsupported_imgopts 'compat=0.10' data_file
 
 for event in l1_update empty_image_prepare reftable_update refblock_alloc; do
 

tests/qemu-iotests/099

@@ -30,6 +30,9 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
+    _rm_test_img "$TEST_IMG.compare"
+    rm -f "$TEST_DIR/blkdebug.conf"
+
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -43,8 +46,9 @@ _supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc
 _supported_proto file
 _supported_os Linux
 _require_drivers blkdebug blkverify
+# data_file would change the json:{} filenames
 _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
-    "subformat=twoGbMaxExtentSparse"
+    "subformat=twoGbMaxExtentSparse" data_file
 
 do_run_qemu()
 {
@@ -121,8 +125,6 @@ echo
 test_qemu "file.driver=blkdebug,file.image.filename=$TEST_IMG"
 
 
-rm -f "$TEST_IMG.compare" "$TEST_DIR/blkdebug.conf"
-
 # success, all done
 echo "*** done"
 rm -f $seq.full

tests/qemu-iotests/103

@@ -38,8 +38,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt qcow2
 _supported_proto file nfs
-# Internal snapshots are (currently) impossible with refcount_bits=1
+# Internal snapshots are (currently) impossible with refcount_bits=1,
-_unsupported_imgopts 'refcount_bits=1[^0-9]'
+# and generally impossible with external data files
+_unsupported_imgopts 'refcount_bits=1[^0-9]' data_file
 
 IMG_SIZE=64K
 

tests/qemu-iotests/106

@@ -51,7 +51,7 @@ for create_mode in off falloc full; do
         echo
         echo "--- create_mode=$create_mode growth_mode=$growth_mode ---"
 
-        IMGOPTS="preallocation=$create_mode" _make_test_img ${CREATION_SIZE}K
+        _make_test_img -o "preallocation=$create_mode" ${CREATION_SIZE}K
         $QEMU_IMG resize -f "$IMGFMT" --preallocation=$growth_mode "$TEST_IMG" +${GROWTH_SIZE}K
 
         expected_size=0

tests/qemu-iotests/108

@@ -37,12 +37,14 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
-# This test directly modifies a refblock so it relies on refcount_bits being 16
+# This test directly modifies a refblock so it relies on refcount_bits being 16;
-_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
+# and the low-level modification it performs are not tuned for external data
+# files
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
 
 echo
 echo '=== Repairing an image without any refcount table ==='
@@ -65,7 +67,7 @@ echo
 echo '=== Repairing unreferenced data cluster in new refblock area ==='
 echo
 
-IMGOPTS='cluster_size=512' _make_test_img 64M
+_make_test_img -o 'cluster_size=512' 64M
 # Allocate the first 128 kB in the image (first refblock)
 $QEMU_IO -c 'write 0 0x1b200' "$TEST_IMG" | _filter_qemu_io
 # should be 131072 == 0x20000

tests/qemu-iotests/109

@@ -29,7 +29,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_qemu
-    rm -f $TEST_IMG.src
+    _rm_test_img "$TEST_IMG.src"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/110

@@ -29,7 +29,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-        rm -f "$TEST_IMG.copy"
+    _rm_test_img "$TEST_IMG.copy"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -67,6 +67,7 @@ echo
 # Across blkdebug without a config file, you cannot reconstruct filenames, so
 # qemu is incapable of knowing the directory of the top image from the filename
 # alone. However, using bdrv_dirname(), it should still work.
+# (Filter out the json:{} filename so this test works with external data files)
 TEST_IMG="json:{
     'driver': '$IMGFMT',
     'file': {
@@ -82,7 +83,8 @@ TEST_IMG="json:{
             }
         ]
     }
-}" _img_info | _filter_img_info | grep -v 'backing file format'
+}" _img_info | _filter_img_info | grep -v 'backing file format' \
+    | _filter_json_filename
 
 echo
 echo '=== Backing name is always relative to the backed image ==='
@@ -114,7 +116,8 @@ TEST_IMG="json:{
             }
         ]
     }
-}" _img_info | _filter_img_info | grep -v 'backing file format'
+}" _img_info | _filter_img_info | grep -v 'backing file format' \
+    | _filter_json_filename
 
 
 # success, all done

tests/qemu-iotests/110.out

@@ -11,7 +11,7 @@ backing file: t.IMGFMT.base (actual path: TEST_DIR/t.IMGFMT.base)
 
 === Non-reconstructable filename ===
 
-image: json:{"driver": "IMGFMT", "file": {"set-state.0.event": "read_aio", "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "set-state.0.new_state": 42}}
+image: json:{ /* filtered */ }
 file format: IMGFMT
 virtual size: 64 MiB (67108864 bytes)
 backing file: t.IMGFMT.base (actual path: TEST_DIR/t.IMGFMT.base)
@@ -22,7 +22,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.b
 
 === Nodes without a common directory ===
 
-image: json:{"driver": "IMGFMT", "file": {"children": [{"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, {"driver": "file", "filename": "TEST_DIR/t.IMGFMT.copy"}], "driver": "quorum", "vote-threshold": 1}}
+image: json:{ /* filtered */ }
 file format: IMGFMT
 virtual size: 64 MiB (67108864 bytes)
 backing file: t.IMGFMT.base (cannot determine actual path)

tests/qemu-iotests/111

@@ -41,8 +41,7 @@ _supported_fmt qed qcow qcow2 vmdk
 _supported_proto file
 _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
 
-$QEMU_IMG create -f $IMGFMT -b "$TEST_IMG.inexistent" "$TEST_IMG" 2>&1 \
+_make_test_img -b "$TEST_IMG.inexistent"
-    | _filter_testdir | _filter_imgfmt
 
 # success, all done
 echo '*** done'

tests/qemu-iotests/112

@@ -40,8 +40,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 # This test will set refcount_bits on its own which would conflict with the
-# manual setting; compat will be overridden as well
+# manual setting; compat will be overridden as well;
-_unsupported_imgopts refcount_bits 'compat=0.10'
+# and external data files do not work well with our refcount testing
+_unsupported_imgopts refcount_bits 'compat=0.10' data_file
 
 print_refcount_bits()
 {
@@ -53,20 +54,20 @@ echo '=== refcount_bits limits ==='
 echo
 
 # Must be positive (non-zero)
-IMGOPTS="$IMGOPTS,refcount_bits=0" _make_test_img 64M
+_make_test_img -o "refcount_bits=0" 64M
 # Must be positive (non-negative)
-IMGOPTS="$IMGOPTS,refcount_bits=-1" _make_test_img 64M
+_make_test_img -o "refcount_bits=-1" 64M
 # May not exceed 64
-IMGOPTS="$IMGOPTS,refcount_bits=128" _make_test_img 64M
+_make_test_img -o "refcount_bits=128" 64M
 # Must be a power of two
-IMGOPTS="$IMGOPTS,refcount_bits=42" _make_test_img 64M
+_make_test_img -o "refcount_bits=42" 64M
 
 # 1 is the minimum
-IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+_make_test_img -o "refcount_bits=1" 64M
 print_refcount_bits
 
 # 64 is the maximum
-IMGOPTS="$IMGOPTS,refcount_bits=64" _make_test_img 64M
+_make_test_img -o "refcount_bits=64" 64M
 print_refcount_bits
 
 # 16 is the default
@@ -78,19 +79,19 @@ echo '=== refcount_bits and compat=0.10 ==='
 echo
 
 # Should work
-IMGOPTS="$IMGOPTS,compat=0.10,refcount_bits=16" _make_test_img 64M
+_make_test_img -o "compat=0.10,refcount_bits=16" 64M
 print_refcount_bits
 
 # Should not work
-IMGOPTS="$IMGOPTS,compat=0.10,refcount_bits=1" _make_test_img 64M
+_make_test_img -o "compat=0.10,refcount_bits=1" 64M
-IMGOPTS="$IMGOPTS,compat=0.10,refcount_bits=64" _make_test_img 64M
+_make_test_img -o "compat=0.10,refcount_bits=64" 64M
 
 
 echo
 echo '=== Snapshot limit on refcount_bits=1 ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+_make_test_img -o "refcount_bits=1" 64M
 print_refcount_bits
 
 $QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
@@ -106,7 +107,7 @@ echo
 echo '=== Snapshot limit on refcount_bits=2 ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=2" _make_test_img 64M
+_make_test_img -o "refcount_bits=2" 64M
 print_refcount_bits
 
 $QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
@@ -124,7 +125,7 @@ echo
 echo '=== Compressed clusters with refcount_bits=1 ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+_make_test_img -o "refcount_bits=1" 64M
 print_refcount_bits
 
 # Both should fit into a single host cluster; instead of failing to increase the
@@ -140,7 +141,7 @@ echo
 echo '=== MSb set in 64 bit refcount ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=64" _make_test_img 64M
+_make_test_img -o "refcount_bits=64" 64M
 print_refcount_bits
 
 $QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
@@ -158,7 +159,7 @@ echo
 echo '=== Snapshot on maximum 64 bit refcount value ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=64" _make_test_img 64M
+_make_test_img -o "refcount_bits=64" 64M
 print_refcount_bits
 
 $QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
@@ -239,7 +240,7 @@ echo
 echo '=== Testing too many references for check ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+_make_test_img -o "refcount_bits=1" 64M
 print_refcount_bits
 
 # This cluster should be created at 0x50000
@@ -262,7 +263,7 @@ echo
 echo '=== Multiple walks necessary during amend ==='
 echo
 
-IMGOPTS="$IMGOPTS,refcount_bits=1,cluster_size=512" _make_test_img 64k
+_make_test_img -o "refcount_bits=1,cluster_size=512" 64k
 
 # Cluster 0 is the image header, clusters 1 to 4 are used by the L1 table, a
 # single L2 table, the reftable and a single refblock. This creates 58 data

tests/qemu-iotests/114

@@ -39,6 +39,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto generic
 _unsupported_proto vxhs
+# qcow2.py does not work too well with external data files
+_unsupported_imgopts data_file
 
 
 TEST_IMG="$TEST_IMG.base" _make_test_img 64M

tests/qemu-iotests/115

@@ -64,8 +64,7 @@ echo
 # least 256 MB. We can achieve that by using preallocation=metadata for an image
 # which has a guest disk size of 256 MB.
 
-IMGOPTS="$IMGOPTS,refcount_bits=64,cluster_size=512,preallocation=metadata" \
+_make_test_img -o "refcount_bits=64,cluster_size=512,preallocation=metadata" 256M
-    _make_test_img 256M
 
 # We know for sure that the L1 and refcount tables do not overlap with any other
 # structure because the metadata overlap checks would have caught that case.

tests/qemu-iotests/121

@@ -39,6 +39,9 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# Refcount structures are used much differently with external data
+# files
+_unsupported_imgopts data_file
 
 echo
 echo '=== New refcount structures may not conflict with existing structures ==='
@@ -50,7 +53,7 @@ echo
 # Preallocation speeds up the write operation, but preallocating everything will
 # destroy the purpose of the write; so preallocate one KB less than what would
 # cause a reftable growth...
-IMGOPTS='preallocation=metadata,cluster_size=1k' _make_test_img 64512K
+_make_test_img -o 'preallocation=metadata,cluster_size=1k' 64512K
 # ...and make the image the desired size afterwards.
 $QEMU_IMG resize "$TEST_IMG" 65M
 
@@ -73,7 +76,7 @@ echo
 echo '--- Test 2 ---'
 echo
 
-IMGOPTS='preallocation=metadata,cluster_size=1k' _make_test_img 64513K
+_make_test_img -o 'preallocation=metadata,cluster_size=1k' 64513K
 # This results in an L1 table growth which in turn results in some clusters at
 # the start of the image becoming free
 $QEMU_IMG resize "$TEST_IMG" 65M
@@ -96,7 +99,7 @@ echo
 echo '=== Allocating a new refcount block must not leave holes in the image ==='
 echo
 
-IMGOPTS='cluster_size=512,refcount_bits=16' _make_test_img 1M
+_make_test_img -o 'cluster_size=512,refcount_bits=16' 1M
 
 # This results in an image with 256 used clusters: the qcow2 header,
 # the refcount table, one refcount block, the L1 table, four L2 tables

tests/qemu-iotests/122

@@ -28,7 +28,9 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-    rm -f "$TEST_IMG".[123]
+    for img in "$TEST_IMG".[123]; do
+        _rm_test_img "$img"
+    done
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/123

@@ -29,7 +29,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$SRC_IMG"
+    _rm_test_img "$SRC_IMG"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -44,7 +44,7 @@ _supported_os Linux
 SRC_IMG="$TEST_DIR/source.$IMGFMT"
 
 _make_test_img 1M
-$QEMU_IMG create -f $IMGFMT "$SRC_IMG" 1M | _filter_img_create
+TEST_IMG_FILE=$SRC_IMG IMGPROTO=file _make_test_img 1M
 
 $QEMU_IO -c 'write -P 42 0 1M' "$SRC_IMG" | _filter_qemu_io
 

tests/qemu-iotests/125

@@ -114,7 +114,7 @@ for GROWTH_SIZE in 16 48 80; do
         for growth_mode in off metadata falloc full; do
             echo "--- cluster_size=$cluster_size growth_size=$GROWTH_SIZE create_mode=$create_mode growth_mode=$growth_mode ---"
 
-            IMGOPTS="preallocation=$create_mode,cluster_size=$cluster_size" _make_test_img ${CREATION_SIZE}
+            _make_test_img -o "preallocation=$create_mode,cluster_size=$cluster_size" ${CREATION_SIZE}
             $QEMU_IMG resize -f "$IMGFMT" --preallocation=$growth_mode "$TEST_IMG" +${GROWTH_SIZE}K
 
             host_size_0=$(get_image_size_on_host)

tests/qemu-iotests/137

@@ -117,7 +117,7 @@ $QEMU_IO \
     -c "reopen -o cache-clean-interval=-1" \
     "$TEST_IMG" | _filter_qemu_io
 
-IMGOPTS="cluster_size=256k" _make_test_img 32P
+_make_test_img -o "cluster_size=256k" 32P
 $QEMU_IO \
     -c "reopen -o l2-cache-entry-size=512,l2-cache-size=1T" \
     "$TEST_IMG" | _filter_qemu_io
@@ -138,14 +138,21 @@ $QEMU_IO \
     "$TEST_IMG" 2>&1 | _filter_qemu_io
 
 # The dirty bit must not be set
-$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+# (Filter the external data file bit)
+if $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features \
+    | grep -q '\<0\>'
+then
+    echo 'ERROR: Dirty bit set'
+else
+    echo 'OK: Dirty bit not set'
+fi
 
 # Similarly we can test whether corruption detection has been enabled:
-# Create L1/L2, overwrite first entry in refcount block, allocate something.
+# Create L1, overwrite refcounts, force allocation of L2 by writing
+# data.
 # Disabling the checks should fail, so the corruption must be detected.
 _make_test_img 64M
-$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$((0x20000))" "\x00\x00\x00\x00\x00\x00\x00\x00"
-poke_file "$TEST_IMG" "$((0x20000))" "\x00\x00"
 $QEMU_IO \
     -c "reopen -o overlap-check=none,lazy-refcounts=42" \
     -c "write 64k 64k" \

tests/qemu-iotests/137.out

@@ -36,11 +36,9 @@ qemu-io: Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ./common.rc: Killed                  ( VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@" )
-incompatible_features     0x0
+OK: Dirty bit not set
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-wrote 65536/65536 bytes at offset 0
-64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-io: Parameter 'lazy-refcounts' expects 'on' or 'off'
-qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
+qcow2: Marking image as corrupt: Preventing invalid allocation of L2 table at offset 0; further corruption events will be suppressed
 write failed: Input/output error
 *** done

tests/qemu-iotests/138

@@ -36,17 +36,19 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-# This tests qocw2-specific low-level functionality
+# This tests qcow2-specific low-level functionality
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# With an external data file, data clusters are not refcounted
+# (and so qemu-img check does not check their refcount)
+_unsupported_imgopts data_file
 
 echo
 echo '=== Check on an image with a multiple of 2^32 clusters ==='
 echo
 
-IMGOPTS=$(_optstr_add "$IMGOPTS" "cluster_size=512") \
+_make_test_img -o "cluster_size=512" 512
-    _make_test_img 512
 
 # Allocate L2 table
 $QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io

tests/qemu-iotests/141

@@ -30,7 +30,9 @@ _cleanup()
 {
     _cleanup_qemu
     _cleanup_test_img
-    rm -f "$TEST_DIR"/{b,m,o}.$IMGFMT
+    for img in "$TEST_DIR"/{b,m,o}.$IMGFMT; do
+        _rm_test_img "$img"
+    done
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/142

@@ -29,7 +29,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f $TEST_IMG.snap
+    _rm_test_img "$TEST_IMG.snap"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/144

@@ -34,7 +34,9 @@ TMP_SNAP2=${TEST_DIR}/tmp2.qcow2
 _cleanup()
 {
     _cleanup_qemu
-    rm -f "${TEST_IMG}" "${TMP_SNAP1}" "${TMP_SNAP2}"
+    for img in "${TEST_IMG}" "${TMP_SNAP1}" "${TMP_SNAP2}"; do
+        _rm_test_img "$img"
+    done
 }
 
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/153

@@ -30,13 +30,9 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "${TEST_IMG}.base"
+    for img in "${TEST_IMG}".{base,overlay,convert,a,b,c,lnk}; do
-    rm -f "${TEST_IMG}.overlay"
+        _rm_test_img "$img"
-    rm -f "${TEST_IMG}.convert"
+    done
-    rm -f "${TEST_IMG}.a"
-    rm -f "${TEST_IMG}.b"
-    rm -f "${TEST_IMG}.c"
-    rm -f "${TEST_IMG}.lnk"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -98,7 +94,7 @@ for opts1 in "" "read-only=on" "read-only=on,force-share=on"; do
 
     echo
     echo "== Creating test image =="
-    $QEMU_IMG create -f $IMGFMT "${TEST_IMG}" -b ${TEST_IMG}.base | _filter_img_create
+    _make_test_img -b "${TEST_IMG}.base"
 
     echo
     echo "== Launching QEMU, opts: '$opts1' =="

tests/qemu-iotests/156

@@ -37,7 +37,9 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_qemu
-    rm -f "$TEST_IMG"{,.target}{,.backing,.overlay}
+    for img in "$TEST_IMG"{,.target}{,.backing,.overlay}; do
+        _rm_test_img "$img"
+    done
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -49,6 +51,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2 qed
 _supported_proto generic
 _unsupported_proto vxhs
+# Copying files around with cp does not work with external data files
+_unsupported_imgopts data_file
 
 # Create source disk
 TEST_IMG="$TEST_IMG.backing" _make_test_img 1M
@@ -120,7 +124,9 @@ _send_qemu_cmd $QEMU_HANDLE \
     '"status": "null"'
 
 # Remove the source images
-rm -f "$TEST_IMG{,.backing,.overlay}"
+for img in "$TEST_IMG{,.backing,.overlay}"; do
+    _rm_test_img "$img"
+done
 
 echo
 

tests/qemu-iotests/159

@@ -28,7 +28,7 @@ status=1
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.out"
+    _rm_test_img "$TEST_IMG.out"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/160

@@ -28,7 +28,8 @@ status=1
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.out" "$TEST_IMG.out.dd"
+    _rm_test_img "$TEST_IMG.out"
+    _rm_test_img "$TEST_IMG.out.dd"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/161

@@ -30,8 +30,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.base"
+    _rm_test_img "$TEST_IMG.base"
-    rm -f "$TEST_IMG.int"
+    _rm_test_img "$TEST_IMG.int"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/170

@@ -28,7 +28,7 @@ status=1
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.out"
+    _rm_test_img  "$TEST_IMG.out"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/172

@@ -29,8 +29,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.2"
+    _rm_test_img "$TEST_IMG.2"
-    rm -f "$TEST_IMG.3"
+    _rm_test_img "$TEST_IMG.3"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/173

@@ -29,7 +29,8 @@ status=1    # failure is the default!
 _cleanup()
 {
     _cleanup_qemu
-    rm  -f "${QEMU_TEST_DIR}/image.base" "${QEMU_TEST_DIR}/image.snp1"
+    _rm_test_img "${TEST_DIR}/image.base"
+    _rm_test_img "${TEST_DIR}/image.snp1"
     _cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/174

@@ -40,7 +40,7 @@ _unsupported_fmt raw
 
 
 size=256K
-IMGFMT=raw IMGKEYSECRET= IMGOPTS= _make_test_img $size | _filter_imgfmt
+IMGFMT=raw IMGKEYSECRET= _make_test_img --no-opts $size | _filter_imgfmt
 
 echo
 echo "== reading wrong format should fail =="

tests/qemu-iotests/175

@@ -95,7 +95,7 @@ stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_block
 for mode in off full falloc; do
     echo
     echo "== creating image with preallocation $mode =="
-    IMGOPTS=preallocation=$mode _make_test_img $size | _filter_imgfmt
+    _make_test_img -o preallocation=$mode $size | _filter_imgfmt
     stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
 done
 

tests/qemu-iotests/176

@@ -47,8 +47,11 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
-# Persistent dirty bitmaps require compat=1.1
+# Persistent dirty bitmaps require compat=1.1;
-_unsupported_imgopts 'compat=0.10'
+# Internal snapshots forbid using an external data file
+# (they work with refcount_bits=1 here, though, because there actually
+# is no data when creating the snapshot)
+_unsupported_imgopts 'compat=0.10' data_file
 
 run_qemu()
 {

tests/qemu-iotests/178

@@ -29,7 +29,7 @@ status=1    # failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.converted"
+    _rm_test_img "$TEST_IMG.converted"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -62,8 +62,8 @@ $QEMU_IMG measure -O foo "$TEST_IMG" # unknown image file format
 
 make_test_img_with_fmt() {
     # Shadow global variables within this function
-    local IMGFMT="$1" IMGOPTS=""
+    local IMGFMT="$1"
-    _make_test_img "$2"
+    _make_test_img --no-opts "$2"
 }
 
 qemu_io_with_fmt() {

tests/qemu-iotests/182

@@ -30,7 +30,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.overlay"
+    _rm_test_img "$TEST_IMG.overlay"
     rm -f "$SOCK_DIR/nbd.socket"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/183

@@ -31,7 +31,7 @@ MIG_SOCKET="${SOCK_DIR}/migrate"
 _cleanup()
 {
     rm -f "${MIG_SOCKET}"
-    rm -f "${TEST_IMG}.dest"
+    _rm_test_img "${TEST_IMG}.dest"
     _cleanup_test_img
     _cleanup_qemu
 }

tests/qemu-iotests/185

@@ -28,8 +28,8 @@ status=1 # failure is the default!
 
 _cleanup()
 {
-    rm -f "${TEST_IMG}.mid"
+    _rm_test_img "${TEST_IMG}.mid"
-    rm -f "${TEST_IMG}.copy"
+    _rm_test_img "${TEST_IMG}.copy"
     _cleanup_test_img
     _cleanup_qemu
 }

tests/qemu-iotests/187

@@ -29,8 +29,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.2"
+    _rm_test_img "$TEST_IMG.2"
-    rm -f "$TEST_IMG.3"
+    _rm_test_img "$TEST_IMG.3"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/190

@@ -29,7 +29,7 @@ status=1    # failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f "$TEST_IMG.converted"
+    _rm_test_img "$TEST_IMG.converted"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -45,7 +45,7 @@ _supported_proto file
 echo "== Huge file =="
 echo
 
-IMGOPTS='cluster_size=2M' _make_test_img 2T
+_make_test_img -o 'cluster_size=2M' 2T
 
 $QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG"
 $QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG"