diff --git a/src/cmd/cli_dd.cpp b/src/cmd/cli_dd.cpp index e43a494d..fd052d6f 100644 --- a/src/cmd/cli_dd.cpp +++ b/src/cmd/cli_dd.cpp @@ -37,7 +37,6 @@ struct dd_in_info_t // in std::string iimg, ifile; bool in_direct = false; - bool detect_size = true; // out cli_result_t result; @@ -68,10 +67,7 @@ struct dd_in_info_t return; } in_granularity = pool_it->second.bitmap_granularity; - if (detect_size) - { - in_size = iwatch->cfg.size; - } + in_size = iwatch->cfg.size; in_seekable = true; return; } @@ -92,33 +88,31 @@ struct dd_in_info_t { ifd = 0; } - if (detect_size) + struct stat st; + if (fstat(ifd, &st) < 0) { - struct stat st; - if (fstat(ifd, &st) < 0) + result = (cli_result_t){ .err = errno, .text = "Failed to stat "+ifile+": "+std::string(strerror(errno)) }; + close(ifd); + ifd = -1; + return; + } + if (S_ISREG(st.st_mode)) + { + in_size = st.st_size; + } + else if (S_ISBLK(st.st_mode)) + { + if (ioctl(ifd, BLKGETSIZE64, &in_size) < 0) { - result = (cli_result_t){ .err = errno, .text = "Failed to stat "+ifile+": "+std::string(strerror(errno)) }; + result = (cli_result_t){ .err = errno, .text = "Failed to get "+ifile+" size: "+std::string(strerror(errno)) }; close(ifd); ifd = -1; return; } - if (S_ISREG(st.st_mode)) - { - in_size = st.st_size; - } - else if (S_ISBLK(st.st_mode)) - { - if (ioctl(ifd, BLKGETSIZE64, &in_size) < 0) - { - result = (cli_result_t){ .err = errno, .text = "Failed to get "+ifile+" size: "+std::string(strerror(errno)) }; - close(ifd); - ifd = -1; - return; - } - } } in_seekable = false; - if (lseek(ifd, 1, SEEK_SET) == (off_t)1) + // a seekable character device has no size (st_size 0), read it like a pipe + if (!S_ISCHR(st.st_mode) && lseek(ifd, 1, SEEK_SET) == (off_t)1) { in_seekable = true; lseek(ifd, 0, SEEK_SET); @@ -175,7 +169,7 @@ struct dd_out_info_t return NULL; } - bool open_output(cli_tool_t *parent, int & state, int base_state) + bool open_output(cli_tool_t *parent, int & state, int base_state, uint64_t oseek, uint64_t blocksize) { if (state == base_state) goto resume_1; @@ -212,6 +206,13 @@ struct dd_out_info_t return true; } } + if ((oseek % out_granularity) || (blocksize % out_granularity)) + { + result = (cli_result_t){ .err = EINVAL, .text = "Unaligned write to Vitastor is not supported" }; + parent->cli->st_cli->close_watch(owatch); + owatch = NULL; + return true; + } if (out_size % 4096) { out_size += (4096 - (out_size % 4096)); @@ -288,13 +289,13 @@ resume_2: } if (ofile != "") { - ofd = open(ofile.c_str(), (out_direct ? O_DIRECT : 0) | (out_append ? O_APPEND : O_RDWR) | (out_create ? O_CREAT : 0), 0666); + ofd = open(ofile.c_str(), (out_direct ? O_DIRECT : 0) | (out_append ? O_WRONLY|O_APPEND : O_RDWR) | (out_create ? O_CREAT : 0), 0666); if (ofd < 0) { result = (cli_result_t){ .err = errno, .text = "Failed to open "+ofile+": "+std::string(strerror(errno)) }; return true; } - if (out_trunc && ftruncate(ofd, out_size) < 0) + if (out_trunc && ftruncate(ofd, out_size ? out_size : oseek) < 0) { result = (cli_result_t){ .err = errno, .text = "Failed to truncate "+ofile+": "+std::string(strerror(errno)) }; return true; @@ -314,6 +315,16 @@ resume_2: out_seekable = true; lseek(ofd, 0, SEEK_SET); } + struct stat st; + if (fstat(ofd, &st) < 0) + { + result = (cli_result_t){ .err = errno, .text = "Failed to stat "+(ofile != "" ? ofile : "stdout")+": "+std::string(strerror(errno)) }; + return true; + } + if (!S_ISREG(st.st_mode) && !S_ISBLK(st.st_mode)) + { + end_fsync = false; + } return true; } @@ -395,6 +406,7 @@ struct cli_dd_t uint64_t written_size = 0; uint64_t written_progress = 0; timespec tv_begin = {}, tv_progress = {}; + bool progress_shown = false; int state = 0; int copy_error = 0; int in_waiting = 0, out_waiting = 0; @@ -413,6 +425,7 @@ struct cli_dd_t auto res = read(fd, buf, blocksize < to_skip ? blocksize : to_skip); if (res <= 0) { + free(buf); return res == 0 ? -EPIPE : -errno; } to_skip -= res; @@ -722,6 +735,12 @@ struct cli_dd_t { return; } + if (end && !end_status && !parent->json_output) + { + if (progress_shown) + fprintf(stderr, "\r\033[K"); + return; + } timespec tv_now; clock_gettime(CLOCK_REALTIME, &tv_now); double sec_delta = ((tv_now.tv_sec - tv_progress.tv_sec) + (double)(tv_now.tv_nsec - tv_progress.tv_nsec)/1000000000.0); @@ -764,6 +783,7 @@ struct cli_dd_t format_size((uint64_t)(delta/sec_delta), true).c_str(), format_size((uint64_t)(written_size/sec_total), true).c_str() ); + progress_shown = true; } } @@ -826,6 +846,12 @@ struct cli_dd_t state = 100; return; } + if (oinfo.ofile != "" && oinfo.out_append && oinfo.out_trunc) + { + result = (cli_result_t){ .err = EINVAL, .text = "conv=trunc and oflag=append can't be used together" }; + state = 100; + return; + } if ((iinfo.iimg != "" ? "i"+iinfo.iimg : "f"+iinfo.ifile) == (oinfo.oimg != "" ? "i"+oinfo.oimg : "f"+oinfo.ofile)) { result = (cli_result_t){ .err = EINVAL, .text = "Input and output image/file can't be equal" }; @@ -848,13 +874,20 @@ struct cli_dd_t state = 100; return; } - if (!oinfo.out_size) + if (iinfo.in_seekable && iseek >= iinfo.in_size) + { + iinfo.close_input(parent); + result = (cli_result_t){ .err = EINVAL, .text = "Input seek position is beyond end of input" }; + state = 100; + return; + } + if (!oinfo.out_size && (iinfo.in_seekable || bytelimit)) { oinfo.out_size = oseek + (iinfo.in_seekable && (!bytelimit || iinfo.in_size-iseek < bytelimit) ? iinfo.in_size-iseek : bytelimit); } resume_1: resume_2: - if (!oinfo.open_output(parent, state, 1)) + if (!oinfo.open_output(parent, state, 1, oseek, blocksize)) { return; } @@ -865,17 +898,7 @@ resume_2: state = 100; return; } - if (oinfo.owatch && ((oseek % oinfo.out_granularity) || (blocksize % oinfo.out_granularity))) - { - result = (cli_result_t){ .err = EINVAL, .text = "Unaligned write to Vitastor is not supported" }; - goto close_end; - } // Copy data - if (iinfo.in_seekable && iseek >= iinfo.in_size) - { - result = (cli_result_t){ .err = EINVAL, .text = "Input seek position is beyond end of input" }; - goto close_end; - } if (!iinfo.iwatch && !iinfo.in_seekable && iseek) { // Read and ignore some data from input @@ -889,9 +912,8 @@ resume_2: in_iodepth = iinfo.in_seekable ? iodepth : 1; out_iodepth = oinfo.out_seekable ? iodepth : 1; write_zero = write_zero || !oinfo.out_seekable; - oinfo.end_fsync = oinfo.end_fsync && oinfo.out_seekable; read_offset = 0; - read_end = iinfo.in_seekable ? iinfo.in_size-iseek : 0; + read_end = iinfo.in_seekable ? iinfo.in_size-iseek : UINT64_MAX; if (oinfo.out_size && (!read_end || read_end > oinfo.out_size-oseek)) read_end = oinfo.out_size-oseek; if (bytelimit && (!read_end || read_end > bytelimit)) @@ -1000,10 +1022,14 @@ std::function cli_tool_t::start_dd(json11::Json cfg) if (!dd->iodepth) dd->iodepth = 4; if (cfg["status"] == "none") + { + dd->end_status = false; + progress = false; + } + else if (cfg["status"] == "noxfer") dd->end_status = false; else if (cfg["status"] == "progress") progress = true; - dd->iinfo.detect_size = cfg["size"].is_null(); dd->oinfo.out_size = parse_size(cfg["size"].as_string()); dd->conv = explode_json(",", cfg["conv"]); dd->iflag = explode_json(",", cfg["iflag"]); diff --git a/tests/test_dd.sh b/tests/test_dd.sh index 36366e28..75ed2647 100755 --- a/tests/test_dd.sh +++ b/tests/test_dd.sh @@ -24,4 +24,85 @@ $VITASTOR_CLI dd iimg=testimg2 iodepth=4 bs=1M count=128 > ./testdata/testfile1 diff ./testdata/testfile ./testdata/testfile1 rm ./testdata/testfile1 +# size= sets the output size, the input size is still detected +$VITASTOR_CLI dd iimg=testimg oimg=testimg3 iodepth=4 size=192M +$VITASTOR_CLI dd if=./testdata/testfile oimg=testimg4 iodepth=4 size=192M +for img in testimg3 testimg4; do + size=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "'$img'") | .size') + [[ "$size" = $((192*1024*1024)) ]] || format_error "dd size=192M created $img with size $size" + $VITASTOR_CLI dd iimg=$img iodepth=4 bs=1M count=128 > ./testdata/testfile1 + diff ./testdata/testfile ./testdata/testfile1 + rm ./testdata/testfile1 +done + +# skip beyond the end of input fails before the output image is created +rc=0 +$VITASTOR_CLI dd iimg=testimg oimg=testimg5 bs=1M skip=200 count=1 || rc=$? +[[ $rc = 22 ]] || format_error "dd skip beyond the end of input returned $rc instead of EINVAL" +if $VITASTOR_CLI ls --json | jq -e '.[] | select(.name == "testimg5")'; then + format_error "dd skip beyond the end of input created the output image" +fi + +# status=none prints nothing, also with the default --progress 1 +$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 status=none 2>./testdata/dd.err +[[ ! -s ./testdata/dd.err ]] || format_error "dd status=none printed: $(cat ./testdata/dd.err)" +diff ./testdata/testfile ./testdata/testfile1 +rm ./testdata/testfile1 + +# status=noxfer drops the final statistics line +$VITASTOR_CLI --progress 0 dd iimg=testimg of=./testdata/testfile1 iodepth=4 status=noxfer 2>./testdata/dd.err +[[ ! -s ./testdata/dd.err ]] || format_error "dd status=noxfer printed: $(cat ./testdata/dd.err)" +rm ./testdata/testfile1 +# with --progress 1 the progress line, if it was shown at all, is cleared at the end +clear_line=$'\r\033[K' +for count in 128 4; do + $VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=$count status=noxfer 2>./testdata/dd.err + if [[ $(wc -l < ./testdata/dd.err) != 0 || ( -s ./testdata/dd.err && "$(cat ./testdata/dd.err)" != *copied*"$clear_line" ) ]]; then + format_error "dd status=noxfer count=$count printed: $(cat -v ./testdata/dd.err)" + fi + head -c ${count}M ./testdata/testfile | cmp - ./testdata/testfile1 + rm ./testdata/testfile1 +done + +# oflag=append appends to the output file +cp ./testdata/over ./testdata/testfile1 +$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=8 oflag=append +(cat ./testdata/over; head -c 8M ./testdata/testfile) | cmp - ./testdata/testfile1 || format_error "dd oflag=append did not append to the output file" +rm ./testdata/testfile1 +# conv=trunc contradicts oflag=append: fail without touching the output file +cp ./testdata/over ./testdata/testfile1 +rc=0 +$VITASTOR_CLI dd iimg=testimg of=./testdata/testfile1 iodepth=4 bs=1M count=8 oflag=append conv=trunc || rc=$? +[[ $rc = 22 ]] || format_error "dd conv=trunc oflag=append returned $rc instead of EINVAL" +cmp ./testdata/over ./testdata/testfile1 || format_error "dd conv=trunc oflag=append changed the output file" +rm ./testdata/testfile1 + +# pipe in without count - copy to the end of input +$VITASTOR_CLI dd iimg=testimg iodepth=4 | $VITASTOR_CLI dd of=./testdata/testfile1 iodepth=4 +diff ./testdata/testfile ./testdata/testfile1 +rm ./testdata/testfile1 +$VITASTOR_CLI dd iimg=testimg iodepth=4 | $VITASTOR_CLI dd of=./testdata/testfile1 iodepth=4 bs=1M seek=4 +(head -c 4M /dev/zero; cat ./testdata/testfile) | cmp - ./testdata/testfile1 || format_error "dd from a pipe with seek= did not copy the whole input" +rm ./testdata/testfile1 + +# unaligned image I/O fails with EINVAL before the output image is created or resized +for args in "if=./testdata/testfile oimg=testimg6 seek=512B" "if=./testdata/testfile oimg=testimg2 seek=512B" "iimg=testimg oimg=testimg7 skip=512B"; do + rc=0 + $VITASTOR_CLI dd iodepth=4 $args || rc=$? + [[ $rc = 22 ]] || format_error "dd $args returned $rc instead of EINVAL" +done +if $VITASTOR_CLI ls --json | jq -e '.[] | select(.name == "testimg6" or .name == "testimg7")'; then + format_error "dd with unaligned image I/O created the output image" +fi +size=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "testimg2") | .size') +[[ "$size" = $((128*1024*1024)) ]] || format_error "dd with unaligned seek resized testimg2 to $size" + +# a character device is read like a pipe, up to count= or size= +$VITASTOR_CLI dd if=/dev/zero oimg=testimg8 size=32M +size=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "testimg8") | .size') +[[ "$size" = $((32*1024*1024)) ]] || format_error "dd if=/dev/zero size=32M created testimg8 with size $size" +$VITASTOR_CLI dd if=/dev/zero of=./testdata/testfile1 bs=1M count=4 conv=nosparse +head -c 4M /dev/zero | cmp - ./testdata/testfile1 || format_error "dd if=/dev/zero count=4 did not write 4 MiB of zeros" +rm ./testdata/testfile1 + format_green OK