vitastor-disk: Check prepare options before partitioning, don't reuse damaged OSD partitions #139
@@ -155,7 +155,8 @@ struct disk_tool_t
|
||||
int clear_osd_superblock(const std::string & dev);
|
||||
int trim_data(std::string device);
|
||||
|
||||
json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
|
||||
// damaged: set if the device can't be read or has an invalid superblock, not just none
|
||||
json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false, bool *damaged = NULL);
|
||||
uint32_t write_osd_superblock(std::string device, json11::Json params);
|
||||
|
||||
void parse_meta_reserve();
|
||||
|
||||
@@ -24,6 +24,39 @@ void disk_tool_t::parse_meta_reserve()
|
||||
}
|
||||
}
|
||||
|
||||
static void set_hdd_defaults(std::map<std::string, std::string> & options, bool is_hybrid)
|
||||
{
|
||||
if (options["block_size"] == "")
|
||||
options["block_size"] = "1M";
|
||||
if (is_hybrid && options["atomic_write_size"] == "")
|
||||
options["atomic_write_size"] = "0";
|
||||
if (is_hybrid && options["throttle_small_writes"] == "")
|
||||
options["throttle_small_writes"] = "1";
|
||||
if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" &&
|
||||
options["csum_block_size"] == "")
|
||||
options["csum_block_size"] = "32k";
|
||||
}
|
||||
|
||||
// Check the options which don't depend on the partition, before creating any partitions
|
||||
static int check_prepare_options(std::map<std::string, std::string> options, bool is_hdd, bool is_hybrid, uint64_t disk_size)
|
||||
{
|
||||
if (is_hdd)
|
||||
set_hdd_defaults(options, is_hybrid);
|
||||
try
|
||||
{
|
||||
blockstore_disk_t dsk;
|
||||
dsk.parse_config(options);
|
||||
dsk.data_device_size = dsk.meta_device_size = dsk.journal_device_size = disk_size;
|
||||
dsk.calc_lengths(true);
|
||||
}
|
||||
catch (std::exception & e)
|
||||
{
|
||||
fprintf(stderr, "%s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result)
|
||||
{
|
||||
static const char *allow_additional_params[] = {
|
||||
@@ -147,15 +180,7 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
|
||||
bool is_hybrid = is_hdd && options["journal_device"] != "" && options["journal_device"] != options["data_device"];
|
||||
if (is_hdd)
|
||||
{
|
||||
if (options["block_size"] == "")
|
||||
options["block_size"] = "1M";
|
||||
if (is_hybrid && options["atomic_write_size"] == "")
|
||||
options["atomic_write_size"] = "0";
|
||||
if (is_hybrid && options["throttle_small_writes"] == "")
|
||||
options["throttle_small_writes"] = "1";
|
||||
if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" &&
|
||||
options["csum_block_size"] == "")
|
||||
options["csum_block_size"] = "32k";
|
||||
set_hdd_defaults(options, is_hybrid);
|
||||
}
|
||||
else if (!json_is_true(options["disable_data_fsync"]))
|
||||
{
|
||||
@@ -333,8 +358,9 @@ int disk_tool_t::check_existing_partition(std::string & dev)
|
||||
fprintf(stderr, "%s contains data, not creating OSD without --force. wipefs shows:\n%s", dev.c_str(), out.c_str());
|
||||
return 1;
|
||||
}
|
||||
json11::Json sb = read_osd_superblock(dev, false);
|
||||
if (!sb.is_null())
|
||||
bool damaged = false;
|
||||
json11::Json sb = read_osd_superblock(dev, false, false, &damaged);
|
||||
if (!sb.is_null() || damaged)
|
||||
{
|
||||
fprintf(stderr, "%s already contains Vitastor OSD superblock, not creating OSD without --force\n", dev.c_str());
|
||||
return 1;
|
||||
@@ -561,13 +587,20 @@ std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d
|
||||
{
|
||||
// Disk already has partitions. If these are empty Vitastor OSD partitions, we can use them
|
||||
uint64_t osds_exist = 0, osds_size = 0;
|
||||
uint64_t sector_size = dev.pt["sectorsize"].uint64_value() ? dev.pt["sectorsize"].uint64_value() : 512;
|
||||
for (const auto & part: dev.pt["partitions"].array_items())
|
||||
{
|
||||
if (strtolower(part["type"].string_value()) == VITASTOR_PART_TYPE)
|
||||
{
|
||||
// Check if an existing Vitastor partition is empty
|
||||
json11::Json sb = read_osd_superblock(part["node"].string_value(), false);
|
||||
if (sb.is_null())
|
||||
bool damaged = false;
|
||||
json11::Json sb = read_osd_superblock(part["node"].string_value(), false, false, &damaged);
|
||||
if (damaged)
|
||||
{
|
||||
// It may still be repaired, so it's other data
|
||||
fprintf(stderr, "%s has an unreadable or invalid OSD superblock, skipping\n", part["node"].string_value().c_str());
|
||||
}
|
||||
else if (sb.is_null())
|
||||
{
|
||||
// Use this partition
|
||||
use_parts.push_back(part["uuid"].string_value());
|
||||
@@ -585,7 +618,7 @@ std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d
|
||||
);
|
||||
if (is_data || sb["params"]["data_device"].string_value().substr(0, 22) != "/dev/disk/by-partuuid/")
|
||||
{
|
||||
osds_size += part["size"].uint64_value()*dev.pt["sectorsize"].uint64_value();
|
||||
osds_size += part["size"].uint64_value()*sector_size;
|
||||
osds_exist++;
|
||||
}
|
||||
}
|
||||
@@ -813,6 +846,11 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
|
||||
options.erase("disable_meta_fsync");
|
||||
options.erase("disable_journal_fsync");
|
||||
}
|
||||
for (auto & dev: devinfo)
|
||||
{
|
||||
if (check_prepare_options(options, dev.is_hdd, hybrid && dev.is_hdd, dev.size) != 0)
|
||||
return 1;
|
||||
}
|
||||
json11::Json::array all_results, errors;
|
||||
auto journal_size = options["journal_size"];
|
||||
if (options.find("dry_run") != options.end())
|
||||
|
||||
@@ -148,17 +148,20 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
|
||||
return sb_size;
|
||||
}
|
||||
|
||||
json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors)
|
||||
json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors, bool *damaged)
|
||||
{
|
||||
vitastor_disk_superblock_t *sb = NULL;
|
||||
uint8_t *buf = NULL;
|
||||
json11::Json osd_params;
|
||||
std::string json_err;
|
||||
std::string real_device, device_type, real_data, real_meta, real_journal;
|
||||
bool absent = false;
|
||||
int r, fd = open(device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
|
||||
if (fd < 0)
|
||||
{
|
||||
fprintf(stderr, "Failed to open device %s: %s\n", device.c_str(), strerror(errno));
|
||||
if (damaged)
|
||||
*damaged = true;
|
||||
return osd_params;
|
||||
}
|
||||
buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096);
|
||||
@@ -173,6 +176,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
|
||||
{
|
||||
if (expect_exist)
|
||||
fprintf(stderr, "Invalid OSD superblock on %s: magic number mismatch\n", device.c_str());
|
||||
absent = true;
|
||||
goto ex;
|
||||
}
|
||||
if (sb->size > VITASTOR_DISK_MAX_SB_SIZE ||
|
||||
@@ -265,6 +269,8 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
|
||||
{ "real_journal_device", real_journal },
|
||||
};
|
||||
ex:
|
||||
if (damaged)
|
||||
*damaged = osd_params.is_null() && !absent;
|
||||
free(buf);
|
||||
close(fd);
|
||||
return osd_params;
|
||||
|
||||
@@ -8,6 +8,39 @@ ANTIETCD=1
|
||||
dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1 count=1 seek=$((100*1024*1024*1024-1))
|
||||
LOOP1=$(sudo losetup --show -f ./testdata/bin/test_osd1.bin)
|
||||
trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1"' || true' EXIT
|
||||
|
||||
# prepare should refuse invalid options before creating partitions
|
||||
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 \
|
||||
--block_size 4M --data_csum_type crc32c --csum_block_size 4k $LOOP1; then
|
||||
format_error "prepare accepted options which don't fit into a metadata block"
|
||||
fi
|
||||
if sudo sfdisk --dump $LOOP1; then
|
||||
format_error "prepare created partitions before refusing the options"
|
||||
fi
|
||||
|
||||
# prepare should leave alone an OSD partition with a superblock of another device or a damaged one,
|
||||
# the OSD may still be repaired. The partitions are small, so the rest of the disk is still used
|
||||
mount | grep '/dev type devtmpfs' || sudo mount udev /dev/ -t devtmpfs
|
||||
printf ',1G,e7009fac-a5a1-4d72-af72-53de13059903\n,1G,e7009fac-a5a1-4d72-af72-53de13059903\n' | \
|
||||
sudo sfdisk --no-reread --no-tell-kernel --label gpt $LOOP1
|
||||
sudo partprobe $LOOP1
|
||||
for i in $(seq 1 100); do [[ -e ${LOOP1}p2 ]] && break; sleep 0.1; done
|
||||
echo '{"osd_num":987654,"data_device":"/dev/disk/by-partuuid/00000000-0000-0000-0000-000000000000"}' | \
|
||||
sudo build/src/disk_tool/vitastor-disk write-sb ${LOOP1}p1
|
||||
echo '{"osd_num":987654,"data_device":"'${LOOP1}p2'"}' | sudo build/src/disk_tool/vitastor-disk write-sb ${LOOP1}p2
|
||||
printf X | sudo dd of=${LOOP1}p2 bs=1 seek=40 conv=notrunc
|
||||
SB_BEFORE=$(for p in 1 2; do sudo dd if=${LOOP1}p$p bs=4096 count=1 status=none; done | md5sum)
|
||||
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 --data_device ${LOOP1}p2; then
|
||||
format_error "prepare overwrote a damaged OSD superblock without --force"
|
||||
fi
|
||||
sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 $LOOP1
|
||||
[[ "$(for p in 1 2; do sudo dd if=${LOOP1}p$p bs=4096 count=1 status=none; done | md5sum)" == "$SB_BEFORE" ]] || \
|
||||
format_error "prepare reused an OSD partition with a superblock of another device or a damaged one"
|
||||
sudo build/src/disk_tool/vitastor-disk read-sb ${LOOP1}p3 > /dev/null || format_error "prepare didn't create an OSD on the free space"
|
||||
for p in 1 2 3; do sudo dd if=/dev/zero of=${LOOP1}p$p bs=4096 count=1 conv=notrunc; done
|
||||
sudo wipefs -a $LOOP1
|
||||
sudo partprobe $LOOP1
|
||||
|
||||
dd if=/dev/zero of=./testdata/bin/test_meta.bin bs=1 count=1 seek=$((1024*1024*1024-1))
|
||||
LOOP2=$(sudo losetup --show -f ./testdata/bin/test_meta.bin)
|
||||
trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2"' || true' EXIT
|
||||
@@ -155,10 +188,10 @@ diff ./testdata/1.json ./testdata/2.json
|
||||
dd if=/dev/zero of=./testdata/bin/test_bad.bin bs=1 count=1 seek=$((1024*1024*1024-1))
|
||||
LOOP3=$(sudo losetup --show -f ./testdata/bin/test_bad.bin)
|
||||
trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2 $LOOP3"' || true' EXIT
|
||||
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 987654 \
|
||||
--block_size 4M --data_csum_type crc32c --csum_block_size 4k $LOOP3; then
|
||||
if sudo -E build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --osd_num 0 $LOOP3; then
|
||||
format_error "prepare exited with 0 after failing to initialize an OSD"
|
||||
fi
|
||||
sudo sfdisk --dump $LOOP3 > /dev/null || format_error "prepare failed before partitioning, not when initializing the OSD"
|
||||
|
||||
# prepare should exit with 0 when it skips every disk, and list the skipped disks in JSON by the path given
|
||||
sudo -E build/src/disk_tool/vitastor-disk-test prepare --json --no_init 1 --osd_num 987655 ./testdata/bin/test_osd1_link1 > ./testdata/prepare.json || \
|
||||
|
||||
Reference in New Issue
Block a user