vitalif/vitastor
5
6
Fork 12
Code Issues 4 Pull Requests 6 Actions Releases 112 Wiki Activity

cli/dd: Fix size=, status=, oflag=append, pipe and character device inputs #94

Closed
koolkhel wants to merge 8 commits from koolkhel:cli-dd into master
pull from: koolkhel:cli-dd
merge into: vitalif:master
Conversation 1 Commits 8 Files Changed 2 +149 -42
2 changed files with 149 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/cmd/cli_dd.cpp
View File

@@ -37,7 +37,6 @@ struct dd_in_info_t
// in
std::string iimg, ifile;
bool in_direct = false;
bool detect_size = true;
// out
cli_result_t result;
@@ -68,10 +67,7 @@ struct dd_in_info_t
return;
}
in_granularity = pool_it->second.bitmap_granularity;
if (detect_size)
{
in_size = iwatch->cfg.size;
}
in_size = iwatch->cfg.size;
in_seekable = true;
return;
}
@@ -92,33 +88,31 @@ struct dd_in_info_t
{
ifd = 0;
}
if (detect_size)
struct stat st;
if (fstat(ifd, &st) < 0)
{
struct stat st;
if (fstat(ifd, &st) < 0)
result = (cli_result_t){ .err = errno, .text = "Failed to stat "+ifile+": "+std::string(strerror(errno)) };
close(ifd);
ifd = -1;
return;
}
if (S_ISREG(st.st_mode))
{
in_size = st.st_size;
}
else if (S_ISBLK(st.st_mode))
{
if (ioctl(ifd, BLKGETSIZE64, &in_size) < 0)
{
result = (cli_result_t){ .err = errno, .text = "Failed to stat "+ifile+": "+std::string(strerror(errno)) };
result = (cli_result_t){ .err = errno, .text = "Failed to get "+ifile+" size: "+std::string(strerror(errno)) };
close(ifd);
ifd = -1;
return;
}
if (S_ISREG(st.st_mode))
{
in_size = st.st_size;
}
else if (S_ISBLK(st.st_mode))
{
if (ioctl(ifd, BLKGETSIZE64, &in_size) < 0)
{
result = (cli_result_t){ .err = errno, .text = "Failed to get "+ifile+" size: "+std::string(strerror(errno)) };
close(ifd);
ifd = -1;
return;
}
}
}
in_seekable = false;
if (lseek(ifd, 1, SEEK_SET) == (off_t)1)
// a seekable character device has no size (st_size 0), read it like a pipe
if (!S_ISCHR(st.st_mode) && lseek(ifd, 1, SEEK_SET) == (off_t)1)
{
in_seekable = true;
lseek(ifd, 0, SEEK_SET);
@@ -175,7 +169,7 @@ struct dd_out_info_t
return NULL;
}
bool open_output(cli_tool_t *parent, int & state, int base_state)
bool open_output(cli_tool_t *parent, int & state, int base_state, uint64_t oseek, uint64_t blocksize)
{
if (state == base_state)
goto resume_1;
@@ -212,6 +206,13 @@ struct dd_out_info_t
return true;
}
}
if ((oseek % out_granularity) || (blocksize % out_granularity))
{
result = (cli_result_t){ .err = EINVAL, .text = "Unaligned write to Vitastor is not supported" };
parent->cli->st_cli->close_watch(owatch);
owatch = NULL;
return true;
}
if (out_size % 4096)
{
out_size += (4096 - (out_size % 4096));
@@ -288,13 +289,13 @@ resume_2:
}
if (ofile != "")
{
ofd = open(ofile.c_str(), (out_direct ? O_DIRECT : 0) | (out_append ? O_APPEND : O_RDWR) | (out_create ? O_CREAT : 0), 0666);
ofd = open(ofile.c_str(), (out_direct ? O_DIRECT : 0) | (out_append ? O_WRONLY|O_APPEND : O_RDWR) | (out_create ? O_CREAT : 0), 0666);
if (ofd < 0)
{
result = (cli_result_t){ .err = errno, .text = "Failed to open "+ofile+": "+std::string(strerror(errno)) };
return true;
}
if (out_trunc && ftruncate(ofd, out_size) < 0)
if (out_trunc && ftruncate(ofd, out_size ? out_size : oseek) < 0)
{
result = (cli_result_t){ .err = errno, .text = "Failed to truncate "+ofile+": "+std::string(strerror(errno)) };
return true;
@@ -314,6 +315,16 @@ resume_2:
out_seekable = true;
lseek(ofd, 0, SEEK_SET);
}
struct stat st;
if (fstat(ofd, &st) < 0)
{
result = (cli_result_t){ .err = errno, .text = "Failed to stat "+(ofile != "" ? ofile : "stdout")+": "+std::string(strerror(errno)) };
return true;
}
if (!S_ISREG(st.st_mode) && !S_ISBLK(st.st_mode))
{
end_fsync = false;
}
return true;
}
@@ -395,6 +406,7 @@ struct cli_dd_t
uint64_t written_size = 0;
uint64_t written_progress = 0;
timespec tv_begin = {}, tv_progress = {};
bool progress_shown = false;
int state = 0;
int copy_error = 0;
int in_waiting = 0, out_waiting = 0;
@@ -413,6 +425,7 @@ struct cli_dd_t
auto res = read(fd, buf, blocksize < to_skip ? blocksize : to_skip);
if (res <= 0)
{
free(buf);
return res == 0 ? -EPIPE : -errno;
}
to_skip -= res;
@@ -722,6 +735,12 @@ struct cli_dd_t
{
return;
}
if (end && !end_status && !parent->json_output)
{
if (progress_shown)
fprintf(stderr, "\r\033[K");
return;
}
timespec tv_now;
clock_gettime(CLOCK_REALTIME, &tv_now);
double sec_delta = ((tv_now.tv_sec - tv_progress.tv_sec) + (double)(tv_now.tv_nsec - tv_progress.tv_nsec)/1000000000.0);
@@ -764,6 +783,7 @@ struct cli_dd_t
format_size((uint64_t)(delta/sec_delta), true).c_str(),
format_size((uint64_t)(written_size/sec_total), true).c_str()
);
progress_shown = true;
}
}
@@ -826,6 +846,12 @@ struct cli_dd_t
state = 100;
return;
}
if (oinfo.ofile != "" && oinfo.out_append && oinfo.out_trunc)
{
result = (cli_result_t){ .err = EINVAL, .text = "conv=trunc and oflag=append can't be used together" };
state = 100;
return;
}
if ((iinfo.iimg != "" ? "i"+iinfo.iimg : "f"+iinfo.ifile) == (oinfo.oimg != "" ? "i"+oinfo.oimg : "f"+oinfo.ofile))
{
result = (cli_result_t){ .err = EINVAL, .text = "Input and output image/file can't be equal" };
@@ -848,13 +874,20 @@ struct cli_dd_t
state = 100;
return;
}
if (!oinfo.out_size)
if (iinfo.in_seekable && iseek >= iinfo.in_size)
{
iinfo.close_input(parent);
result = (cli_result_t){ .err = EINVAL, .text = "Input seek position is beyond end of input" };
state = 100;
return;
}
if (!oinfo.out_size && (iinfo.in_seekable || bytelimit))
{
oinfo.out_size = oseek + (iinfo.in_seekable && (!bytelimit || iinfo.in_size-iseek < bytelimit) ? iinfo.in_size-iseek : bytelimit);
}
resume_1:
resume_2:
if (!oinfo.open_output(parent, state, 1))
if (!oinfo.open_output(parent, state, 1, oseek, blocksize))
{
return;
}
@@ -865,17 +898,7 @@ resume_2:
state = 100;
return;
}
if (oinfo.owatch && ((oseek % oinfo.out_granularity) || (blocksize % oinfo.out_granularity)))
{
result = (cli_result_t){ .err = EINVAL, .text = "Unaligned write to Vitastor is not supported" };
goto close_end;
}
// Copy data
if (iinfo.in_seekable && iseek >= iinfo.in_size)
{
result = (cli_result_t){ .err = EINVAL, .text = "Input seek position is beyond end of input" };
goto close_end;
}
if (!iinfo.iwatch && !iinfo.in_seekable && iseek)
{
// Read and ignore some data from input
@@ -889,9 +912,8 @@ resume_2:
in_iodepth = iinfo.in_seekable ? iodepth : 1;
out_iodepth = oinfo.out_seekable ? iodepth : 1;
write_zero = write_zero || !oinfo.out_seekable;
oinfo.end_fsync = oinfo.end_fsync && oinfo.out_seekable;
read_offset = 0;
read_end = iinfo.in_seekable ? iinfo.in_size-iseek : 0;
read_end = iinfo.in_seekable ? iinfo.in_size-iseek : UINT64_MAX;
if (oinfo.out_size && (!read_end || read_end > oinfo.out_size-oseek))
read_end = oinfo.out_size-oseek;
if (bytelimit && (!read_end || read_end > bytelimit))
@@ -1000,10 +1022,14 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_dd(json11::Json cfg)
if (!dd->iodepth)
dd->iodepth = 4;
if (cfg["status"] == "none")
{
dd->end_status = false;
progress = false;
}
else if (cfg["status"] == "noxfer")
dd->end_status = false;
else if (cfg["status"] == "progress")
progress = true;
dd->iinfo.detect_size = cfg["size"].is_null();
dd->oinfo.out_size = parse_size(cfg["size"].as_string());
dd->conv = explode_json(",", cfg["conv"]);
dd->iflag = explode_json(",", cfg["iflag"]);

81
tests/test_dd.sh
View File

@@ -24,4 +24,85 @@ $VITASTOR_CLI dd iimg=testimg2 iodepth=4 bs=1M count=128 > ./testdata/testfile1
diff ./testdata/testfile ./testdata/testfile1
rm ./testdata/testfile1
# size= sets the output size, the input size is still detected
$VITASTOR_CLI dd iimg=testimg oimg=testimg3 iodepth=4 size=192M
$VITASTOR_CLI dd if=./testdata/testfile oimg=testimg4 iodepth=4 size=192M
for img in testimg3 testimg4; do
size=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "'$img'") | .size')
[[ "$size" = $((192*1024*1024)) ]] || format_error "dd size=192M created $img with size $size"
$VITASTOR_CLI dd iimg=$img iodepth=4 bs=1M count=128 > ./testdata/testfile1
diff ./testdata/testfile ./testdata/testfile1
rm ./testdata/testfile1
done
# skip beyond the end of input fails before the output image is created
rc=0
$VITASTOR_CLI dd iimg=testimg oimg=testimg5 bs=1M skip=200 count=1 || rc=$?
[[ $rc = 22 ]] || format_error "dd skip beyond the end of input returned $rc instead of EINVAL"
if $VITASTOR_CLI ls --json | jq -e '.[] | select(.name == "testimg5")'; then
format_error "dd skip beyond the end of input created the output image"
fi
# status=none prints nothing, also with the default --progress 1
$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 status=none 2>./testdata/dd.err
[[ ! -s ./testdata/dd.err ]] || format_error "dd status=none printed: $(cat ./testdata/dd.err)"
diff ./testdata/testfile ./testdata/testfile1
rm ./testdata/testfile1
# status=noxfer drops the final statistics line
$VITASTOR_CLI --progress 0 dd iimg=testimg of=./testdata/testfile1 iodepth=4 status=noxfer 2>./testdata/dd.err
[[ ! -s ./testdata/dd.err ]] || format_error "dd status=noxfer printed: $(cat ./testdata/dd.err)"
rm ./testdata/testfile1
# with --progress 1 the progress line, if it was shown at all, is cleared at the end
clear_line=$'\r\033[K'
for count in 128 4; do
$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=$count status=noxfer 2>./testdata/dd.err
if [[ $(wc -l < ./testdata/dd.err) != 0 || ( -s ./testdata/dd.err && "$(cat ./testdata/dd.err)" != *copied*"$clear_line" ) ]]; then
format_error "dd status=noxfer count=$count printed: $(cat -v ./testdata/dd.err)"
fi
head -c ${count}M ./testdata/testfile | cmp - ./testdata/testfile1
rm ./testdata/testfile1
done
# oflag=append appends to the output file
cp ./testdata/over ./testdata/testfile1
$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=8 oflag=append
(cat ./testdata/over; head -c 8M ./testdata/testfile) | cmp - ./testdata/testfile1 || format_error "dd oflag=append did not append to the output file"
rm ./testdata/testfile1
# conv=trunc contradicts oflag=append: fail without touching the output file
cp ./testdata/over ./testdata/testfile1
rc=0
$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=8 oflag=append conv=trunc || rc=$?
[[ $rc = 22 ]] || format_error "dd conv=trunc oflag=append returned $rc instead of EINVAL"
cmp ./testdata/over ./testdata/testfile1 || format_error "dd conv=trunc oflag=append changed the output file"
rm ./testdata/testfile1
# pipe in without count - copy to the end of input
$VITASTOR_CLI dd iimg=testimg iodepth=4 | $VITASTOR_CLI dd of=./testdata/testfile1 iodepth=4
diff ./testdata/testfile ./testdata/testfile1
rm ./testdata/testfile1
$VITASTOR_CLI dd iimg=testimg iodepth=4 | $VITASTOR_CLI dd of=./testdata/testfile1 iodepth=4 bs=1M seek=4
(head -c 4M /dev/zero; cat ./testdata/testfile) | cmp - ./testdata/testfile1 || format_error "dd from a pipe with seek= did not copy the whole input"
rm ./testdata/testfile1
# unaligned image I/O fails with EINVAL before the output image is created or resized
for args in "if=./testdata/testfile oimg=testimg6 seek=512B" "if=./testdata/testfile oimg=testimg2 seek=512B" "iimg=testimg oimg=testimg7 skip=512B"; do
rc=0
$VITASTOR_CLI dd iodepth=4 $args || rc=$?
[[ $rc = 22 ]] || format_error "dd $args returned $rc instead of EINVAL"
done
if $VITASTOR_CLI ls --json | jq -e '.[] | select(.name == "testimg6" or .name == "testimg7")'; then
format_error "dd with unaligned image I/O created the output image"
fi
size=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "testimg2") | .size')
[[ "$size" = $((128*1024*1024)) ]] || format_error "dd with unaligned seek resized testimg2 to $size"
# a character device is read like a pipe, up to count= or size=
$VITASTOR_CLI dd if=/dev/zero oimg=testimg8 size=32M
size=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "testimg8") | .size')
[[ "$size" = $((32*1024*1024)) ]] || format_error "dd if=/dev/zero size=32M created testimg8 with size $size"
$VITASTOR_CLI dd if=/dev/zero of=./testdata/testfile1 bs=1M count=4 conv=nosparse
head -c 4M /dev/zero | cmp - ./testdata/testfile1 || format_error "dd if=/dev/zero count=4 did not write 4 MiB of zeros"
rm ./testdata/testfile1
format_green OK