cli/dd: Keep in-order output going after ignored errors #133

Merged
vitalif merged 1 commits from koolkhel/vitastor:dd-noerror-in-order into master 2026-11-20 19:33:14 +03:00
2 changed files with 58 additions and 28 deletions

View File

@@ -399,7 +399,7 @@ struct cli_dd_t
bool write_zero = false;
uint64_t in_iodepth = 0, out_iodepth = 0;
uint64_t read_offset = 0, read_end = 0;
uint64_t read_offset = 0, read_end = 0, next_write = 0;
std::vector<dd_buf_t*> read_buffers, short_reads, short_writes;
std::vector<uint8_t*> zero_buf;
bool in_eof = false;
@@ -458,11 +458,8 @@ struct cli_dd_t
read_op->len, iinfo.iimg.c_str(), read_op->offset,
strerror(read_op->retval < 0 ? -read_op->retval : EIO), read_op->retval
);
if (!ignore_errors)
{
copy_error = read_op->retval < 0 ? -read_op->retval : EIO;
}
delete cur_read;
cur_read->len = 0;
add_failed_read(cur_read, read_op->retval < 0 ? -read_op->retval : EIO);
}
else if (!is_zero(read_op->bitmap_buf, (read_op->len/iinfo.in_granularity+7)/8))
{
@@ -503,11 +500,8 @@ struct cli_dd_t
read_op->len, iinfo.iimg.c_str(), read_op->offset,
strerror(read_op->retval < 0 ? -read_op->retval : EIO), read_op->retval
);
if (!ignore_errors)
{
copy_error = read_op->retval < 0 ? -read_op->retval : EIO;
}
delete cur_read;
cur_read->len = 0;
add_failed_read(cur_read, read_op->retval < 0 ? -read_op->retval : EIO);
}
else
{
@@ -574,17 +568,11 @@ struct cli_dd_t
data->iov.iov_len, iinfo.ifile == "" ? "stdin" : iinfo.ifile.c_str(), cur_read->offset,
strerror(-data->res), data->res
);
if (!ignore_errors)
{
copy_error = -data->res;
}
add_failed_read(cur_read, -data->res);
}
else if (data->res == 0)
{
in_eof = true;
}
if (data->res <= 0)
{
if (cur_read->len > 0)
add_finished_read(cur_read);
else
@@ -611,6 +599,27 @@ struct cli_dd_t
return true;
}
// an ignored read error must not leave a gap: in-order output would wait for the block forever
void add_failed_read(dd_buf_t *cur_read, int err)
{
if (!ignore_errors || (!iinfo.iwatch && !iinfo.in_seekable))
{
// a stream has no block to skip: its read error ends the copy, also with conv=noerror
copy_error = err;
in_eof = true;
if (cur_read->len > 0)
add_finished_read(cur_read);
else
delete cur_read;
}
else
{
memset((uint8_t*)cur_read->buf + cur_read->len, 0, cur_read->max - cur_read->len);
cur_read->len = cur_read->max;
add_finished_read(cur_read);
}
}
void add_finished_read(dd_buf_t *cur_read)
{
if (!write_zero && is_zero(cur_read->buf, cur_read->max))
@@ -628,6 +637,17 @@ struct cli_dd_t
bool add_write_op()
{
if (!short_writes.size() && !oinfo.out_seekable && read_buffers[0]->offset > next_write)
{
// can't write - input buffers are out of order
return false;
}
// take the sqe before the buffer, or a full ring drops the block; an unused sqe would still be submitted
io_uring_sqe *sqe = NULL;
if (!oinfo.owatch && !(sqe = parent->ringloop->get_sqe()))
{
return false;
}
dd_buf_t *cur_read;
if (short_writes.size())
{
@@ -637,11 +657,8 @@ struct cli_dd_t
else
{
cur_read = read_buffers[0];
if (!oinfo.out_seekable && cur_read->offset > written_size)
{
// can't write - input buffers are out of order
return false;
}
// by blocks taken, not bytes written: an ignored write error must not stop in-order output
next_write = cur_read->offset + cur_read->len;
cur_read->max = cur_read->len;
cur_read->len = 0;
read_buffers.erase(read_buffers.begin(), read_buffers.begin()+1);
@@ -688,11 +705,6 @@ struct cli_dd_t
}
else
{
io_uring_sqe *sqe = parent->ringloop->get_sqe();
if (!sqe)
{
return false;
}
ring_data_t *data = ((ring_data_t*)sqe->user_data);
data->iov = (iovec){ .iov_base = (uint8_t*)cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
io_uring_prep_writev(sqe, oinfo.ofd, &data->iov, 1, oinfo.out_seekable ? cur_read->offset+cur_read->len+oseek : -1);

View File

@@ -76,6 +76,24 @@ $VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=8 of
[[ $rc = 22 ]] || format_error "dd conv=trunc oflag=append returned $rc instead of EINVAL"
cmp ./testdata/over ./testdata/testfile1 || format_error "dd conv=trunc oflag=append changed the output file"
rm ./testdata/testfile1
# conv=noerror gives zeros for an unreadable block, also on in-order output; reading lo/speed fails with EINVAL
if [[ -r /sys/class/net/lo/speed ]] && ! cat /sys/class/net/lo/speed >/dev/null 2>&1; then
cp ./testdata/over ./testdata/testfile1
$VITASTOR_CLI dd if=/sys/class/net/lo/speed of=./testdata/testfile1 bs=4k conv=noerror oflag=append
(cat ./testdata/over; head -c 4k /dev/zero) | cmp - ./testdata/testfile1 || format_error "dd conv=noerror did not replace an unreadable block with zeros"
rm ./testdata/testfile1
else
echo "SKIP: dd conv=noerror with an unreadable block - /sys/class/net/lo/speed is missing or readable here"
fi
# a stream has no block to skip: its read error ends the copy, also with conv=noerror; reading an unattached tun fails with EBADFD
if [[ -c /dev/net/tun && -r /dev/net/tun ]]; then
rc=0
timeout 60 $VITASTOR_CLI dd if=/dev/net/tun bs=4k count=2 conv=noerror > ./testdata/testfile1 || rc=$?
[[ $rc != 0 && $rc != 124 && ! -s ./testdata/testfile1 ]] || format_error "dd conv=noerror from a failing stream returned $rc and wrote $(stat -c %s ./testdata/testfile1) bytes"
rm ./testdata/testfile1
else
echo "SKIP: dd conv=noerror with a failing stream - /dev/net/tun is missing or not readable here"
fi
# pipe in without count - copy to the end of input
$VITASTOR_CLI dd iimg=testimg iodepth=4 | $VITASTOR_CLI dd of=./testdata/testfile1 iodepth=4