diff --git a/src/disk_tool/disk_tool.h b/src/disk_tool/disk_tool.h index 1f910d5f..447b3924 100644 --- a/src/disk_tool/disk_tool.h +++ b/src/disk_tool/disk_tool.h @@ -155,7 +155,8 @@ struct disk_tool_t int clear_osd_superblock(const std::string & dev); int trim_data(std::string device); - json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false); + // damaged: set if the device can't be read or has an invalid superblock, not just none + json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false, bool *damaged = NULL); uint32_t write_osd_superblock(std::string device, json11::Json params); void parse_meta_reserve(); diff --git a/src/disk_tool/disk_tool_prepare.cpp b/src/disk_tool/disk_tool_prepare.cpp index 269b47ac..a0a59c7d 100644 --- a/src/disk_tool/disk_tool_prepare.cpp +++ b/src/disk_tool/disk_tool_prepare.cpp @@ -24,6 +24,39 @@ void disk_tool_t::parse_meta_reserve() } } +static void set_hdd_defaults(std::map & options, bool is_hybrid) +{ + if (options["block_size"] == "") + options["block_size"] = "1M"; + if (is_hybrid && options["atomic_write_size"] == "") + options["atomic_write_size"] = "0"; + if (is_hybrid && options["throttle_small_writes"] == "") + options["throttle_small_writes"] = "1"; + if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" && + options["csum_block_size"] == "") + options["csum_block_size"] = "32k"; +} + +// Check the options which don't depend on the partition, before creating any partitions +static int check_prepare_options(std::map options, bool is_hdd, bool is_hybrid, uint64_t disk_size) +{ + if (is_hdd) + set_hdd_defaults(options, is_hybrid); + try + { + blockstore_disk_t dsk; + dsk.parse_config(options); + dsk.data_device_size = dsk.meta_device_size = dsk.journal_device_size = disk_size; + dsk.calc_lengths(true); + } + catch (std::exception & e) + { + fprintf(stderr, "%s\n", e.what()); + return 1; + } + return 0; +} + int disk_tool_t::prepare_one(std::map options, int is_hdd, json11::Json::object & result) { static const char *allow_additional_params[] = { @@ -147,15 +180,7 @@ int disk_tool_t::prepare_one(std::map options, int is_ bool is_hybrid = is_hdd && options["journal_device"] != "" && options["journal_device"] != options["data_device"]; if (is_hdd) { - if (options["block_size"] == "") - options["block_size"] = "1M"; - if (is_hybrid && options["atomic_write_size"] == "") - options["atomic_write_size"] = "0"; - if (is_hybrid && options["throttle_small_writes"] == "") - options["throttle_small_writes"] = "1"; - if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" && - options["csum_block_size"] == "") - options["csum_block_size"] = "32k"; + set_hdd_defaults(options, is_hybrid); } else if (!json_is_true(options["disable_data_fsync"])) { @@ -333,8 +358,9 @@ int disk_tool_t::check_existing_partition(std::string & dev) fprintf(stderr, "%s contains data, not creating OSD without --force. wipefs shows:\n%s", dev.c_str(), out.c_str()); return 1; } - json11::Json sb = read_osd_superblock(dev, false); - if (!sb.is_null()) + bool damaged = false; + json11::Json sb = read_osd_superblock(dev, false, false, &damaged); + if (!sb.is_null() || damaged) { fprintf(stderr, "%s already contains Vitastor OSD superblock, not creating OSD without --force\n", dev.c_str()); return 1; @@ -561,13 +587,20 @@ std::vector disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d { // Disk already has partitions. If these are empty Vitastor OSD partitions, we can use them uint64_t osds_exist = 0, osds_size = 0; + uint64_t sector_size = dev.pt["sectorsize"].uint64_value() ? dev.pt["sectorsize"].uint64_value() : 512; for (const auto & part: dev.pt["partitions"].array_items()) { if (strtolower(part["type"].string_value()) == VITASTOR_PART_TYPE) { // Check if an existing Vitastor partition is empty - json11::Json sb = read_osd_superblock(part["node"].string_value(), false); - if (sb.is_null()) + bool damaged = false; + json11::Json sb = read_osd_superblock(part["node"].string_value(), false, false, &damaged); + if (damaged) + { + // It may still be repaired, so it's other data + fprintf(stderr, "%s has an unreadable or invalid OSD superblock, skipping\n", part["node"].string_value().c_str()); + } + else if (sb.is_null()) { // Use this partition use_parts.push_back(part["uuid"].string_value()); @@ -585,7 +618,7 @@ std::vector disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d ); if (is_data || sb["params"]["data_device"].string_value().substr(0, 22) != "/dev/disk/by-partuuid/") { - osds_size += part["size"].uint64_value()*dev.pt["sectorsize"].uint64_value(); + osds_size += part["size"].uint64_value()*sector_size; osds_exist++; } } @@ -813,6 +846,11 @@ int disk_tool_t::prepare(std::vector devices) options.erase("disable_meta_fsync"); options.erase("disable_journal_fsync"); } + for (auto & dev: devinfo) + { + if (check_prepare_options(options, dev.is_hdd, hybrid && dev.is_hdd, dev.size) != 0) + return 1; + } json11::Json::array all_results, errors; auto journal_size = options["journal_size"]; if (options.find("dry_run") != options.end()) diff --git a/src/disk_tool/disk_tool_udev.cpp b/src/disk_tool/disk_tool_udev.cpp index 504efbb9..5eaa3e7c 100644 --- a/src/disk_tool/disk_tool_udev.cpp +++ b/src/disk_tool/disk_tool_udev.cpp @@ -148,17 +148,20 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para return sb_size; } -json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors) +json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors, bool *damaged) { vitastor_disk_superblock_t *sb = NULL; uint8_t *buf = NULL; json11::Json osd_params; std::string json_err; std::string real_device, device_type, real_data, real_meta, real_journal; + bool absent = false; int r, fd = open(device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR); if (fd < 0) { fprintf(stderr, "Failed to open device %s: %s\n", device.c_str(), strerror(errno)); + if (damaged) + *damaged = true; return osd_params; } buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096); @@ -173,6 +176,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex { if (expect_exist) fprintf(stderr, "Invalid OSD superblock on %s: magic number mismatch\n", device.c_str()); + absent = true; goto ex; } if (sb->size > VITASTOR_DISK_MAX_SB_SIZE || @@ -265,6 +269,8 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex { "real_journal_device", real_journal }, }; ex: + if (damaged) + *damaged = osd_params.is_null() && !absent; free(buf); close(fd); return osd_params; diff --git a/tests/test_resize_auto.sh b/tests/test_resize_auto.sh index 82546106..b1876c3c 100755 --- a/tests/test_resize_auto.sh +++ b/tests/test_resize_auto.sh @@ -8,6 +8,39 @@ ANTIETCD=1 dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1 count=1 seek=$((100*1024*1024*1024-1)) LOOP1=$(sudo losetup --show -f ./testdata/bin/test_osd1.bin) trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1"' || true' EXIT + +# prepare should refuse invalid options before creating partitions +if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 \ + --block_size 4M --data_csum_type crc32c --csum_block_size 4k $LOOP1; then + format_error "prepare accepted options which don't fit into a metadata block" +fi +if sudo sfdisk --dump $LOOP1; then + format_error "prepare created partitions before refusing the options" +fi + +# prepare should leave alone an OSD partition with a superblock of another device or a damaged one, +# the OSD may still be repaired. The partitions are small, so the rest of the disk is still used +mount | grep '/dev type devtmpfs' || sudo mount udev /dev/ -t devtmpfs +printf ',1G,e7009fac-a5a1-4d72-af72-53de13059903\n,1G,e7009fac-a5a1-4d72-af72-53de13059903\n' | \ + sudo sfdisk --no-reread --no-tell-kernel --label gpt $LOOP1 +sudo partprobe $LOOP1 +for i in $(seq 1 100); do [[ -e ${LOOP1}p2 ]] && break; sleep 0.1; done +echo '{"osd_num":987654,"data_device":"/dev/disk/by-partuuid/00000000-0000-0000-0000-000000000000"}' | \ + sudo build/src/disk_tool/vitastor-disk write-sb ${LOOP1}p1 +echo '{"osd_num":987654,"data_device":"'${LOOP1}p2'"}' | sudo build/src/disk_tool/vitastor-disk write-sb ${LOOP1}p2 +printf X | sudo dd of=${LOOP1}p2 bs=1 seek=40 conv=notrunc +SB_BEFORE=$(for p in 1 2; do sudo dd if=${LOOP1}p$p bs=4096 count=1 status=none; done | md5sum) +if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 --data_device ${LOOP1}p2; then + format_error "prepare overwrote a damaged OSD superblock without --force" +fi +sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 $LOOP1 +[[ "$(for p in 1 2; do sudo dd if=${LOOP1}p$p bs=4096 count=1 status=none; done | md5sum)" == "$SB_BEFORE" ]] || \ + format_error "prepare reused an OSD partition with a superblock of another device or a damaged one" +sudo build/src/disk_tool/vitastor-disk read-sb ${LOOP1}p3 > /dev/null || format_error "prepare didn't create an OSD on the free space" +for p in 1 2 3; do sudo dd if=/dev/zero of=${LOOP1}p$p bs=4096 count=1 conv=notrunc; done +sudo wipefs -a $LOOP1 +sudo partprobe $LOOP1 + dd if=/dev/zero of=./testdata/bin/test_meta.bin bs=1 count=1 seek=$((1024*1024*1024-1)) LOOP2=$(sudo losetup --show -f ./testdata/bin/test_meta.bin) trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2"' || true' EXIT @@ -155,10 +188,10 @@ diff ./testdata/1.json ./testdata/2.json dd if=/dev/zero of=./testdata/bin/test_bad.bin bs=1 count=1 seek=$((1024*1024*1024-1)) LOOP3=$(sudo losetup --show -f ./testdata/bin/test_bad.bin) trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2 $LOOP3"' || true' EXIT -if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 \ - --block_size 4M --data_csum_type crc32c --csum_block_size 4k $LOOP3; then +if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 0 $LOOP3; then format_error "prepare exited with 0 after failing to initialize an OSD" fi +sudo sfdisk --dump $LOOP3 > /dev/null || format_error "prepare failed before partitioning, not when initializing the OSD" # prepare should exit with 0 when it skips every disk, and list the skipped disks in JSON by the path given sudo -E build/src/disk_tool/vitastor-disk-test prepare --json --no_init 1 --osd_num 987655 ./testdata/bin/test_osd1_link1 > ./testdata/prepare.json || \