diff --git a/block/file-posix.c b/block/file-posix.c index ad299beb38..fe83cbf0eb 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -648,7 +648,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options, } #endif - bs->supported_zero_flags = s->discard_zeroes ? BDRV_REQ_MAY_UNMAP : 0; + bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP; ret = 0; fail: if (filename && (bdrv_flags & BDRV_O_TEMPORARY)) { @@ -1487,6 +1487,35 @@ static ssize_t handle_aiocb_write_zeroes(RawPosixAIOData *aiocb) return -ENOTSUP; } +static ssize_t handle_aiocb_write_zeroes_unmap(RawPosixAIOData *aiocb) +{ + BDRVRawState *s G_GNUC_UNUSED = aiocb->bs->opaque; + int ret; + + /* First try to write zeros and unmap at the same time */ + +#ifdef CONFIG_FALLOCATE_PUNCH_HOLE + ret = do_fallocate(s->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + aiocb->aio_offset, aiocb->aio_nbytes); + if (ret != -ENOTSUP) { + return ret; + } +#endif + +#ifdef CONFIG_XFS + if (s->is_xfs) { + /* xfs_discard() guarantees that the discarded area reads as all-zero + * afterwards, so we can use it here. */ + return xfs_discard(s, aiocb->aio_offset, aiocb->aio_nbytes); + } +#endif + + /* If we couldn't manage to unmap while guaranteed that the area reads as + * all-zero afterwards, just write zeroes without unmapping */ + ret = handle_aiocb_write_zeroes(aiocb); + return ret; +} + #ifndef HAVE_COPY_FILE_RANGE static off_t copy_file_range(int in_fd, off_t *in_off, int out_fd, off_t *out_off, size_t len, unsigned int flags) @@ -1646,6 +1675,9 @@ static int handle_aiocb_truncate(RawPosixAIOData *aiocb) num = MIN(left, 65536); result = write(fd, buf, num); if (result < 0) { + if (errno == EINTR) { + continue; + } result = -errno; error_setg_errno(errp, -result, "Could not write zeros for preallocation"); @@ -1729,6 +1761,9 @@ static int aio_worker(void *arg) case QEMU_AIO_WRITE_ZEROES: ret = handle_aiocb_write_zeroes(aiocb); break; + case QEMU_AIO_WRITE_ZEROES | QEMU_AIO_DISCARD: + ret = handle_aiocb_write_zeroes_unmap(aiocb); + break; case QEMU_AIO_COPY_RANGE: ret = handle_aiocb_copy_range(aiocb); break; @@ -2553,15 +2588,13 @@ static int coroutine_fn raw_co_pwrite_zeroes( int bytes, BdrvRequestFlags flags) { BDRVRawState *s = bs->opaque; + int operation = QEMU_AIO_WRITE_ZEROES; - if (!(flags & BDRV_REQ_MAY_UNMAP)) { - return paio_submit_co(bs, s->fd, offset, NULL, bytes, - QEMU_AIO_WRITE_ZEROES); - } else if (s->discard_zeroes) { - return paio_submit_co(bs, s->fd, offset, NULL, bytes, - QEMU_AIO_DISCARD); + if (flags & BDRV_REQ_MAY_UNMAP) { + operation |= QEMU_AIO_DISCARD; } - return -ENOTSUP; + + return paio_submit_co(bs, s->fd, offset, NULL, bytes, operation); } static int raw_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) @@ -3054,20 +3087,19 @@ static coroutine_fn int hdev_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes, BdrvRequestFlags flags) { BDRVRawState *s = bs->opaque; + int operation = QEMU_AIO_WRITE_ZEROES | QEMU_AIO_BLKDEV; int rc; rc = fd_open(bs); if (rc < 0) { return rc; } - if (!(flags & BDRV_REQ_MAY_UNMAP)) { - return paio_submit_co(bs, s->fd, offset, NULL, bytes, - QEMU_AIO_WRITE_ZEROES|QEMU_AIO_BLKDEV); - } else if (s->discard_zeroes) { - return paio_submit_co(bs, s->fd, offset, NULL, bytes, - QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV); + + if (flags & BDRV_REQ_MAY_UNMAP) { + operation |= QEMU_AIO_DISCARD; } - return -ENOTSUP; + + return paio_submit_co(bs, s->fd, offset, NULL, bytes, operation); } static int coroutine_fn hdev_co_create_opts(const char *filename, QemuOpts *opts, diff --git a/block/qapi.c b/block/qapi.c index e12968fec8..339727f0f4 100644 --- a/block/qapi.c +++ b/block/qapi.c @@ -593,15 +593,29 @@ BlockStatsList *qmp_query_blockstats(bool has_query_nodes, p_next = &info->next; } } else { - for (blk = blk_next(NULL); blk; blk = blk_next(blk)) { + for (blk = blk_all_next(NULL); blk; blk = blk_all_next(blk)) { BlockStatsList *info = g_malloc0(sizeof(*info)); AioContext *ctx = blk_get_aio_context(blk); BlockStats *s; + char *qdev; + + if (!*blk_name(blk) && !blk_get_attached_dev(blk)) { + continue; + } aio_context_acquire(ctx); s = bdrv_query_bds_stats(blk_bs(blk), true); s->has_device = true; s->device = g_strdup(blk_name(blk)); + + qdev = blk_get_attached_dev_id(blk); + if (qdev && *qdev) { + s->has_qdev = true; + s->qdev = qdev; + } else { + g_free(qdev); + } + bdrv_query_blk_stats(s->stats, blk); aio_context_release(ctx); diff --git a/block/qcow.c b/block/qcow.c index 102d058d1c..385d935258 100644 --- a/block/qcow.c +++ b/block/qcow.c @@ -938,6 +938,7 @@ static int coroutine_fn qcow_co_create(BlockdevCreateOptions *opts, ret = 0; exit: blk_unref(qcow_blk); + bdrv_unref(bs); qcrypto_block_free(crypto); return ret; } diff --git a/block/qcow2.c b/block/qcow2.c index 6162ed8be2..ec9e6238a0 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -797,7 +797,7 @@ static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts, if (l2_cache_size_set && refcount_cache_size_set) { error_setg(errp, QCOW2_OPT_CACHE_SIZE ", " QCOW2_OPT_L2_CACHE_SIZE " and " QCOW2_OPT_REFCOUNT_CACHE_SIZE " may not be set " - "the same time"); + "at the same time"); return; } else if (*l2_cache_size > combined_cache_size) { error_setg(errp, QCOW2_OPT_L2_CACHE_SIZE " may not exceed " diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index 5e19cd50da..8e1fa3a66e 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -255,6 +255,17 @@ comparable library support for invoking and interacting with QEMU programs. If you opt for Python, it is strongly recommended to write Python 3 compatible code. +Both Python and Bash frameworks in iotests provide helpers to manage test +images. They can be used to create and clean up images under the test +directory. If no I/O or any protocol specific feature is needed, it is often +more convenient to use the pseudo block driver, ``null-co://``, as the test +image, which doesn't require image creation or cleaning up. Avoid system-wide +devices or files whenever possible, such as ``/dev/null`` or ``/dev/zero``. +Otherwise, image locking implications have to be considered. For example, +another application on the host may have locked the file, possibly leading to a +test failure. If using such devices are explicitly desired, consider adding +``locking=off`` option to disable image locking. + Docker based tests ================== diff --git a/include/block/block.h b/include/block/block.h index f85e3a6ed3..4e0871aaf9 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -43,11 +43,12 @@ typedef struct BlockFragInfo { typedef enum { BDRV_REQ_COPY_ON_READ = 0x1, BDRV_REQ_ZERO_WRITE = 0x2, - /* The BDRV_REQ_MAY_UNMAP flag is used to indicate that the block driver - * is allowed to optimize a write zeroes request by unmapping (discarding) - * blocks if it is guaranteed that the result will read back as - * zeroes. The flag is only passed to the driver if the block device is - * opened with BDRV_O_UNMAP. + + /* + * The BDRV_REQ_MAY_UNMAP flag is used in write_zeroes requests to indicate + * that the block driver should unmap (discard) blocks if it is guaranteed + * that the result will read back as zeroes. The flag is only passed to the + * driver if the block device is opened with BDRV_O_UNMAP. */ BDRV_REQ_MAY_UNMAP = 0x4, diff --git a/qapi/block-core.json b/qapi/block-core.json index d40d5ecc3b..5b9084a394 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -866,6 +866,9 @@ # # @node-name: The node name of the device. (Since 2.3) # +# @qdev: The qdev ID, or if no ID is assigned, the QOM path of the block +# device. (since 3.0) +# # @stats: A @BlockDeviceStats for the device. # # @parent: This describes the file block device if it has one. @@ -879,7 +882,7 @@ # Since: 0.14.0 ## { 'struct': 'BlockStats', - 'data': {'*device': 'str', '*node-name': 'str', + 'data': {'*device': 'str', '*qdev': 'str', '*node-name': 'str', 'stats': 'BlockDeviceStats', '*parent': 'BlockStats', '*backing': 'BlockStats'} } @@ -941,7 +944,8 @@ # "idle_time_ns":2953431879, # "account_invalid":true, # "account_failed":false -# } +# }, +# "qdev": "/machine/unattached/device[23]" # }, # { # "device":"ide1-cd0", @@ -959,7 +963,8 @@ # "wr_merged":0, # "account_invalid":false, # "account_failed":false -# } +# }, +# "qdev": "/machine/unattached/device[24]" # }, # { # "device":"floppy0", @@ -977,7 +982,8 @@ # "wr_merged":0, # "account_invalid":false, # "account_failed":false -# } +# }, +# "qdev": "/machine/unattached/device[16]" # }, # { # "device":"sd0", diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx index 69758fb6e8..1526f327a5 100644 --- a/qemu-img-cmds.hx +++ b/qemu-img-cmds.hx @@ -44,7 +44,7 @@ STEXI ETEXI DEF("convert", img_convert, - "convert [--object objectdef] [--image-opts] [--target-image-opts] [-U] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] filename [filename2 [...]] output_filename") + "convert [--object objectdef] [--image-opts] [--target-image-opts] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] filename [filename2 [...]] output_filename") STEXI @item convert [--object @var{objectdef}] [--image-opts] [--target-image-opts] [-U] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-B @var{backing_file}] [-o @var{options}] [-l @var{snapshot_param}] [-S @var{sparse_size}] [-m @var{num_coroutines}] [-W] @var{filename} [@var{filename2} [...]] @var{output_filename} ETEXI diff --git a/qemu-img.c b/qemu-img.c index 9b7506b8ae..1acddf693c 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -2024,11 +2024,12 @@ static int img_convert(int argc, char **argv) skip_create = false, progress = false, tgt_image_opts = false; int64_t ret = -EINVAL; bool force_share = false; + bool explict_min_sparse = false; ImgConvertState s = (ImgConvertState) { /* Need at least 4k of zeros for sparse detection */ .min_sparse = 8, - .copy_range = true, + .copy_range = false, .buf_sectors = IO_BUF_SIZE / BDRV_SECTOR_SIZE, .wr_in_order = true, .num_coroutines = 8, @@ -2043,7 +2044,7 @@ static int img_convert(int argc, char **argv) {"target-image-opts", no_argument, 0, OPTION_TARGET_IMAGE_OPTS}, {0, 0, 0, 0} }; - c = getopt_long(argc, argv, ":hf:O:B:co:l:S:pt:T:qnm:WU", + c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WU", long_options, NULL); if (c == -1) { break; @@ -2067,9 +2068,11 @@ static int img_convert(int argc, char **argv) case 'B': out_baseimg = optarg; break; + case 'C': + s.copy_range = true; + break; case 'c': s.compressed = true; - s.copy_range = false; break; case 'o': if (!is_valid_option_list(optarg)) { @@ -2112,7 +2115,7 @@ static int img_convert(int argc, char **argv) } s.min_sparse = sval / BDRV_SECTOR_SIZE; - s.copy_range = false; + explict_min_sparse = true; break; } case 'p': @@ -2172,6 +2175,16 @@ static int img_convert(int argc, char **argv) goto fail_getopt; } + if (s.compressed && s.copy_range) { + error_report("Cannot enable copy offloading when -c is used"); + goto fail_getopt; + } + + if (explict_min_sparse && s.copy_range) { + error_report("Cannot enable copy offloading when -S is used"); + goto fail_getopt; + } + if (tgt_image_opts && !skip_create) { error_report("--target-image-opts requires use of -n flag"); goto fail_getopt; diff --git a/qemu-img.texi b/qemu-img.texi index 5853cd18d1..3b6710a580 100644 --- a/qemu-img.texi +++ b/qemu-img.texi @@ -96,8 +96,7 @@ will enumerate information about backing files in a disk image chain. Refer below for further description. @item -c -indicates that target image must be compressed (qcow format only). If this -option is used, copy offloading will not be attempted. +indicates that target image must be compressed (qcow format only) @item -h with or without a command shows help and lists the supported formats @@ -116,8 +115,7 @@ in case both @var{-q} and @var{-p} options are used. indicates the consecutive number of bytes that must contain only zeros for qemu-img to create a sparse image during conversion. This value is rounded down to the nearest 512 bytes. You may use the common size suffixes like -@code{k} for kilobytes. If this option is used, copy offloading will not be -attempted. +@code{k} for kilobytes. @item -t @var{cache} specifies the cache mode that should be used with the (destination) file. See @@ -171,6 +169,12 @@ Number of parallel coroutines for the convert process Allow out-of-order writes to the destination. This option improves performance, but is only recommended for preallocated devices like host devices or other raw block devices. +@item -C +Try to use copy offloading to move data from source image to target. This may +improve performance if the data is remote, such as with NFS or iSCSI backends, +but will not automatically sparsify zero sectors, and may result in a fully +allocated target image depending on the host support for getting allocation +information. @end table Parameters to dd subcommand: @@ -321,7 +325,7 @@ Error on reading data @end table -@item convert [--object @var{objectdef}] [--image-opts] [--target-image-opts] [-U] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-B @var{backing_file}] [-o @var{options}] [-l @var{snapshot_param}] [-S @var{sparse_size}] [-m @var{num_coroutines}] [-W] @var{filename} [@var{filename2} [...]] @var{output_filename} +@item convert [--object @var{objectdef}] [--image-opts] [--target-image-opts] [-U] [-C] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-B @var{backing_file}] [-o @var{options}] [-l @var{snapshot_param}] [-S @var{sparse_size}] [-m @var{num_coroutines}] [-W] @var{filename} [@var{filename2} [...]] @var{output_filename} Convert the disk image @var{filename} or a snapshot @var{snapshot_param} to disk image @var{output_filename} using format @var{output_fmt}. It can be optionally compressed (@code{-c} diff --git a/tests/qemu-iotests/082 b/tests/qemu-iotests/082 index a872f771a6..3e605d52d1 100755 --- a/tests/qemu-iotests/082 +++ b/tests/qemu-iotests/082 @@ -157,6 +157,14 @@ run_qemu_img convert -o help # Try help option for a format that does not support creation run_qemu_img convert -O bochs -o help +echo +echo === convert: -C and other options === + +# Adding the help option to a command without other -o options +run_qemu_img convert -C -S 4k -O $IMGFMT "$TEST_IMG" "$TEST_IMG".target +run_qemu_img convert -C -S 8k -O $IMGFMT "$TEST_IMG" "$TEST_IMG".target +run_qemu_img convert -C -c -O $IMGFMT "$TEST_IMG" "$TEST_IMG".target + echo echo === amend: Options specified more than once === diff --git a/tests/qemu-iotests/082.out b/tests/qemu-iotests/082.out index 60ef87c276..19e9fb13ff 100644 --- a/tests/qemu-iotests/082.out +++ b/tests/qemu-iotests/082.out @@ -508,6 +508,17 @@ size Virtual disk size Testing: convert -O bochs -o help qemu-img: Format driver 'bochs' does not support image creation +=== convert: -C and other options === + +Testing: convert -C -S 4k -O qcow2 TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.target +qemu-img: Cannot enable copy offloading when -S is used + +Testing: convert -C -S 8k -O qcow2 TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.target +qemu-img: Cannot enable copy offloading when -S is used + +Testing: convert -C -c -O qcow2 TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.target +qemu-img: Cannot enable copy offloading when -c is used + === amend: Options specified more than once === Testing: amend -f foo -f qcow2 -o lazy_refcounts=on TEST_DIR/t.qcow2 diff --git a/tests/qemu-iotests/103.out b/tests/qemu-iotests/103.out index bd45d3875a..bd9eec3250 100644 --- a/tests/qemu-iotests/103.out +++ b/tests/qemu-iotests/103.out @@ -5,10 +5,10 @@ wrote 65536/65536 bytes at offset 0 === Testing invalid option combinations === -can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time +can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set at the same time can't open device TEST_DIR/t.IMGFMT: l2-cache-size may not exceed cache-size can't open device TEST_DIR/t.IMGFMT: refcount-cache-size may not exceed cache-size -can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time +can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set at the same time can't open device TEST_DIR/t.IMGFMT: L2 cache entry size must be a power of two between 512 and the cluster size (65536) can't open device TEST_DIR/t.IMGFMT: L2 cache entry size must be a power of two between 512 and the cluster size (65536) can't open device TEST_DIR/t.IMGFMT: L2 cache entry size must be a power of two between 512 and the cluster size (65536) diff --git a/tests/qemu-iotests/137.out b/tests/qemu-iotests/137.out index 96724a6c33..6a2ffc71fd 100644 --- a/tests/qemu-iotests/137.out +++ b/tests/qemu-iotests/137.out @@ -16,7 +16,7 @@ read 33554432/33554432 bytes at offset 0 === Try setting some invalid values === Parameter 'lazy-refcounts' expects 'on' or 'off' -cache-size, l2-cache-size and refcount-cache-size may not be set the same time +cache-size, l2-cache-size and refcount-cache-size may not be set at the same time l2-cache-size may not exceed cache-size refcount-cache-size may not exceed cache-size L2 cache size too big diff --git a/tests/qemu-iotests/226 b/tests/qemu-iotests/226 index 211ea9888a..8ec3e612dd 100755 --- a/tests/qemu-iotests/226 +++ b/tests/qemu-iotests/226 @@ -56,10 +56,10 @@ for PROTO in "file" "host_device" "host_cdrom"; do echo echo "== Testing RO ==" $QEMU_IO -c "open -r -o driver=$PROTO,filename=$TEST_IMG" 2>&1 | _filter_testdir | _filter_imgfmt - $QEMU_IO -c "open -r -o driver=$PROTO,filename=/dev/null" 2>&1 | _filter_imgfmt + $QEMU_IO -c "open -r -o driver=$PROTO,filename=/dev/null,locking=off" 2>&1 | _filter_imgfmt echo "== Testing RW ==" $QEMU_IO -c "open -o driver=$PROTO,filename=$TEST_IMG" 2>&1 | _filter_testdir | _filter_imgfmt - $QEMU_IO -c "open -o driver=$PROTO,filename=/dev/null" 2>&1 | _filter_imgfmt + $QEMU_IO -c "open -o driver=$PROTO,filename=/dev/null,locking=off" 2>&1 | _filter_imgfmt done # success, all done diff --git a/tests/qemu-iotests/227 b/tests/qemu-iotests/227 new file mode 100755 index 0000000000..9a5f7f9f14 --- /dev/null +++ b/tests/qemu-iotests/227 @@ -0,0 +1,101 @@ +#!/bin/bash +# +# Test query-blockstats with different ways to create a BB +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# creator +owner=kwolf@redhat.com + +seq=$(basename $0) +echo "QA output created by $seq" + +here=$PWD +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +_supported_fmt generic +_supported_proto file +_supported_os Linux + +function do_run_qemu() +{ + echo Testing: "$@" + $QEMU -nographic -qmp-pretty stdio -serial none "$@" + echo +} + +function run_qemu() +{ + do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \ + | _filter_qemu | _filter_imgfmt \ + | _filter_generated_node_ids +} + +echo +echo '=== blockstats with -drive if=virtio ===' +echo + +run_qemu -drive driver=null-co,if=virtio <