vitastor-disk: Check prepare options before partitioning, don't reuse damaged OSD partitions #139

Open
koolkhel wants to merge 6 commits from koolkhel/vitastor:prepare-check-before-partitioning into master
4 changed files with 96 additions and 18 deletions

View File

@@ -155,7 +155,8 @@ struct disk_tool_t
int clear_osd_superblock(const std::string & dev);
int trim_data(std::string device);
json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
// damaged: set if the device can't be read or has an invalid superblock, not just none
json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false, bool *damaged = NULL);
uint32_t write_osd_superblock(std::string device, json11::Json params);
void parse_meta_reserve();

View File

@@ -24,6 +24,39 @@ void disk_tool_t::parse_meta_reserve()
}
}
static void set_hdd_defaults(std::map<std::string, std::string> & options, bool is_hybrid)
{
if (options["block_size"] == "")
options["block_size"] = "1M";
if (is_hybrid && options["atomic_write_size"] == "")
options["atomic_write_size"] = "0";
if (is_hybrid && options["throttle_small_writes"] == "")
options["throttle_small_writes"] = "1";
if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" &&
options["csum_block_size"] == "")
options["csum_block_size"] = "32k";
}
// Check the options which don't depend on the partition, before creating any partitions
static int check_prepare_options(std::map<std::string, std::string> options, bool is_hdd, bool is_hybrid, uint64_t disk_size)
{
if (is_hdd)
set_hdd_defaults(options, is_hybrid);
try
{
blockstore_disk_t dsk;
dsk.parse_config(options);
dsk.data_device_size = dsk.meta_device_size = dsk.journal_device_size = disk_size;
dsk.calc_lengths(true);
}
catch (std::exception & e)
{
fprintf(stderr, "%s\n", e.what());
return 1;
}
return 0;
}
int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result)
{
static const char *allow_additional_params[] = {
@@ -147,15 +180,7 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
bool is_hybrid = is_hdd && options["journal_device"] != "" && options["journal_device"] != options["data_device"];
if (is_hdd)
{
if (options["block_size"] == "")
options["block_size"] = "1M";
if (is_hybrid && options["atomic_write_size"] == "")
options["atomic_write_size"] = "0";
if (is_hybrid && options["throttle_small_writes"] == "")
options["throttle_small_writes"] = "1";
if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" &&
options["csum_block_size"] == "")
options["csum_block_size"] = "32k";
set_hdd_defaults(options, is_hybrid);
}
else if (!json_is_true(options["disable_data_fsync"]))
{
@@ -333,8 +358,9 @@ int disk_tool_t::check_existing_partition(std::string & dev)
fprintf(stderr, "%s contains data, not creating OSD without --force. wipefs shows:\n%s", dev.c_str(), out.c_str());
return 1;
}
json11::Json sb = read_osd_superblock(dev, false);
if (!sb.is_null())
bool damaged = false;
json11::Json sb = read_osd_superblock(dev, false, false, &damaged);
if (!sb.is_null() || damaged)
{
fprintf(stderr, "%s already contains Vitastor OSD superblock, not creating OSD without --force\n", dev.c_str());
return 1;
@@ -561,13 +587,20 @@ std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d
{
// Disk already has partitions. If these are empty Vitastor OSD partitions, we can use them
uint64_t osds_exist = 0, osds_size = 0;
uint64_t sector_size = dev.pt["sectorsize"].uint64_value() ? dev.pt["sectorsize"].uint64_value() : 512;
for (const auto & part: dev.pt["partitions"].array_items())
{
if (strtolower(part["type"].string_value()) == VITASTOR_PART_TYPE)
{
// Check if an existing Vitastor partition is empty
json11::Json sb = read_osd_superblock(part["node"].string_value(), false);
if (sb.is_null())
bool damaged = false;
json11::Json sb = read_osd_superblock(part["node"].string_value(), false, false, &damaged);
if (damaged)
{
// It may still be repaired, so it's other data
fprintf(stderr, "%s has an unreadable or invalid OSD superblock, skipping\n", part["node"].string_value().c_str());
}
else if (sb.is_null())
{
// Use this partition
use_parts.push_back(part["uuid"].string_value());
@@ -585,7 +618,7 @@ std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d
);
if (is_data || sb["params"]["data_device"].string_value().substr(0, 22) != "/dev/disk/by-partuuid/")
{
osds_size += part["size"].uint64_value()*dev.pt["sectorsize"].uint64_value();
osds_size += part["size"].uint64_value()*sector_size;
osds_exist++;
}
}
@@ -813,6 +846,11 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
options.erase("disable_meta_fsync");
options.erase("disable_journal_fsync");
}
for (auto & dev: devinfo)
{
if (check_prepare_options(options, dev.is_hdd, hybrid && dev.is_hdd, dev.size) != 0)
return 1;
}
json11::Json::array all_results, errors;
auto journal_size = options["journal_size"];
if (options.find("dry_run") != options.end())

View File

@@ -148,17 +148,20 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
return sb_size;
}
json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors)
json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors, bool *damaged)
{
vitastor_disk_superblock_t *sb = NULL;
uint8_t *buf = NULL;
json11::Json osd_params;
std::string json_err;
std::string real_device, device_type, real_data, real_meta, real_journal;
bool absent = false;
int r, fd = open(device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
if (fd < 0)
{
fprintf(stderr, "Failed to open device %s: %s\n", device.c_str(), strerror(errno));
if (damaged)
*damaged = true;
return osd_params;
}
buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096);
@@ -173,6 +176,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
{
if (expect_exist)
fprintf(stderr, "Invalid OSD superblock on %s: magic number mismatch\n", device.c_str());
absent = true;
goto ex;
}
if (sb->size > VITASTOR_DISK_MAX_SB_SIZE ||
@@ -265,6 +269,8 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
{ "real_journal_device", real_journal },
};
ex:
if (damaged)
*damaged = osd_params.is_null() && !absent;
free(buf);
close(fd);
return osd_params;

View File

@@ -8,6 +8,39 @@ ANTIETCD=1
dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1 count=1 seek=$((100*1024*1024*1024-1))
LOOP1=$(sudo losetup --show -f ./testdata/bin/test_osd1.bin)
trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1"' || true' EXIT
# prepare should refuse invalid options before creating partitions
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 \
--block_size 4M --data_csum_type crc32c --csum_block_size 4k $LOOP1; then
format_error "prepare accepted options which don't fit into a metadata block"
fi
if sudo sfdisk --dump $LOOP1; then
format_error "prepare created partitions before refusing the options"
fi
# prepare should leave alone an OSD partition with a superblock of another device or a damaged one,
# the OSD may still be repaired. The partitions are small, so the rest of the disk is still used
mount | grep '/dev type devtmpfs' || sudo mount udev /dev/ -t devtmpfs
printf ',1G,e7009fac-a5a1-4d72-af72-53de13059903\n,1G,e7009fac-a5a1-4d72-af72-53de13059903\n' | \
sudo sfdisk --no-reread --no-tell-kernel --label gpt $LOOP1
sudo partprobe $LOOP1
for i in $(seq 1 100); do [[ -e ${LOOP1}p2 ]] && break; sleep 0.1; done
echo '{"osd_num":987654,"data_device":"/dev/disk/by-partuuid/00000000-0000-0000-0000-000000000000"}' | \
sudo build/src/disk_tool/vitastor-disk write-sb ${LOOP1}p1
echo '{"osd_num":987654,"data_device":"'${LOOP1}p2'"}' | sudo build/src/disk_tool/vitastor-disk write-sb ${LOOP1}p2
printf X | sudo dd of=${LOOP1}p2 bs=1 seek=40 conv=notrunc
SB_BEFORE=$(for p in 1 2; do sudo dd if=${LOOP1}p$p bs=4096 count=1 status=none; done | md5sum)
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 --data_device ${LOOP1}p2; then
format_error "prepare overwrote a damaged OSD superblock without --force"
fi
sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 $LOOP1
[[ "$(for p in 1 2; do sudo dd if=${LOOP1}p$p bs=4096 count=1 status=none; done | md5sum)" == "$SB_BEFORE" ]] || \
format_error "prepare reused an OSD partition with a superblock of another device or a damaged one"
sudo build/src/disk_tool/vitastor-disk read-sb ${LOOP1}p3 > /dev/null || format_error "prepare didn't create an OSD on the free space"
for p in 1 2 3; do sudo dd if=/dev/zero of=${LOOP1}p$p bs=4096 count=1 conv=notrunc; done
sudo wipefs -a $LOOP1
sudo partprobe $LOOP1
dd if=/dev/zero of=./testdata/bin/test_meta.bin bs=1 count=1 seek=$((1024*1024*1024-1))
LOOP2=$(sudo losetup --show -f ./testdata/bin/test_meta.bin)
trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2"' || true' EXIT
@@ -155,10 +188,10 @@ diff ./testdata/1.json ./testdata/2.json
dd if=/dev/zero of=./testdata/bin/test_bad.bin bs=1 count=1 seek=$((1024*1024*1024-1))
LOOP3=$(sudo losetup --show -f ./testdata/bin/test_bad.bin)
trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2 $LOOP3"' || true' EXIT
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 \
--block_size 4M --data_csum_type crc32c --csum_block_size 4k $LOOP3; then
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 0 $LOOP3; then
format_error "prepare exited with 0 after failing to initialize an OSD"
fi
sudo sfdisk --dump $LOOP3 > /dev/null || format_error "prepare failed before partitioning, not when initializing the OSD"
# prepare should exit with 0 when it skips every disk, and list the skipped disks in JSON by the path given
sudo -E build/src/disk_tool/vitastor-disk-test prepare --json --no_init 1 --osd_num 987655 ./testdata/bin/test_osd1_link1 > ./testdata/prepare.json || \