From 29b40aba93ae8adc41729672d262c9d547e413e3 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 20 Aug 2022 15:07:28 +0300 Subject: [PATCH] Add write-meta command (for debug) --- src/disk_tool.cpp | 25 ++++++++++++- src/disk_tool.h | 3 ++ src/disk_tool_journal.cpp | 74 ++++++++++++--------------------------- src/disk_tool_meta.cpp | 43 +++++++++++++++++++++++ src/disk_tool_utils.cpp | 34 ++++++++++++++++++ 5 files changed, 127 insertions(+), 52 deletions(-) diff --git a/src/disk_tool.cpp b/src/disk_tool.cpp index f74356b2..c6cbc16e 100644 --- a/src/disk_tool.cpp +++ b/src/disk_tool.cpp @@ -107,6 +107,10 @@ static const char *help_text = "vitastor-disk dump-meta \n" " Dump metadata in JSON format.\n" "\n" + "vitastor-disk write-meta \n" + " Write metadata from JSON taken from standard input in the same format as produced by\n" + " dump-meta --json.\n" + "\n" "vitastor-disk simple-offsets \n" " Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n" " --object_size 128k Set blockstore block size\n" @@ -185,7 +189,7 @@ int main(int argc, char *argv[]) { if (cmd.size() < 5) { - print_help(help_text, "vitastor-disk", cmd[0], false); + print_help(help_text, aliased ? "vitastor-dump-journal" : "vitastor-disk", cmd[0], false); return 1; } self.dsk.journal_device = cmd[1]; @@ -228,6 +232,25 @@ int main(int argc, char *argv[]) self.dsk.meta_len = strtoull(cmd[4], NULL, 10); return self.dump_meta(); } + else if (!strcmp(cmd[0], "write-meta")) + { + if (cmd.size() < 4) + { + print_help(help_text, "vitastor-disk", cmd[0], false); + return 1; + } + self.new_meta_device = cmd[1]; + self.new_meta_offset = strtoull(cmd[2], NULL, 10); + self.new_meta_len = strtoull(cmd[3], NULL, 10); + std::string json_err; + json11::Json meta = json11::Json::parse(read_all_fd(0), json_err); + if (json_err != "") + { + fprintf(stderr, "Invalid JSON: %s\n", json_err.c_str()); + return 1; + } + return self.write_json_meta(meta); + } else if (!strcmp(cmd[0], "resize")) { return self.resize_data(); diff --git a/src/disk_tool.h b/src/disk_tool.h index bfe6ba75..3fa02e37 100644 --- a/src/disk_tool.h +++ b/src/disk_tool.h @@ -92,6 +92,7 @@ struct disk_tool_t void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap); int write_json_journal(json11::Json entries); + int write_json_meta(json11::Json meta); int resize_data(); int resize_parse_params(); @@ -125,6 +126,8 @@ struct disk_tool_t void disk_tool_simple_offsets(json11::Json cfg, bool json_output); +uint64_t sscanf_json(const char *fmt, const json11::Json & str); +void fromhexstr(const std::string & from, int bytes, uint8_t *to); std::string realpath_str(std::string path, bool nofail = true); std::string read_all_fd(int fd); std::string read_file(std::string file, bool allow_enoent = false); diff --git a/src/disk_tool_journal.cpp b/src/disk_tool_journal.cpp index 2446cb41..afee34a5 100644 --- a/src/disk_tool_journal.cpp +++ b/src/disk_tool_journal.cpp @@ -317,35 +317,6 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json) } } -static uint64_t sscanf_num(const char *fmt, const std::string & str) -{ - uint64_t value = 0; - sscanf(str.c_str(), fmt, &value); - return value; -} - -static int fromhex(char c) -{ - if (c >= '0' && c <= '9') - return (c-'0'); - else if (c >= 'a' && c <= 'f') - return (c-'a'+10); - else if (c >= 'A' && c <= 'F') - return (c-'A'+10); - return -1; -} - -static void fromhexstr(const std::string & from, int bytes, uint8_t *to) -{ - for (int i = 0; i < from.size() && i < bytes; i++) - { - int x = fromhex(from[2*i]), y = fromhex(from[2*i+1]); - if (x < 0 || y < 0) - break; - to[i] = x*16 + y; - } -} - int disk_tool_t::write_json_journal(json11::Json entries) { new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len); @@ -363,6 +334,18 @@ int disk_tool_t::write_json_journal(json11::Json entries) { "delete", JE_DELETE }, { "rollback", JE_ROLLBACK }, }; + // Write start entry into the first block + *((journal_entry_start*)new_journal_buf) = (journal_entry_start){ + .magic = JOURNAL_MAGIC, + .type = JE_START, + .size = sizeof(journal_entry_start), + .journal_start = dsk.journal_block_size, + .version = JOURNAL_VERSION, + }; + ((journal_entry*)new_journal_buf)->crc32 = je_crc32((journal_entry*)new_journal_buf); + new_journal_ptr += dsk.journal_block_size; + new_journal_data = new_journal_ptr+dsk.journal_block_size; + new_journal_in_pos = 0; for (const auto & rec: entries.array_items()) { auto t_it = type_by_name.find(rec["type"].string_value()); @@ -372,6 +355,8 @@ int disk_tool_t::write_json_journal(json11::Json entries) continue; } uint16_t type = t_it->second; + if (type == JE_START) + continue; uint32_t entry_size = (type == JE_START ? sizeof(journal_entry_start) : (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT @@ -398,20 +383,7 @@ int disk_tool_t::write_json_journal(json11::Json entries) } } journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos); - if (type == JE_START) - { - *((journal_entry_start*)ne) = (journal_entry_start){ - .magic = JOURNAL_MAGIC, - .type = type, - .size = entry_size, - .journal_start = dsk.journal_block_size, - .version = JOURNAL_VERSION, - }; - new_journal_ptr += dsk.journal_block_size; - new_journal_data = new_journal_ptr+dsk.journal_block_size; - new_journal_in_pos = 0; - } - else if (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT) + if (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT) { if (new_journal_data - new_journal_buf + ne->small_write.len > new_journal_len) { @@ -425,14 +397,14 @@ int disk_tool_t::write_json_journal(json11::Json entries) .size = entry_size, .crc32_prev = new_crc32_prev, .oid = { - .inode = sscanf_num("0x%lx", rec["inode"].string_value()), - .stripe = sscanf_num("0x%lx", rec["stripe"].string_value()), + .inode = sscanf_json(NULL, rec["inode"]), + .stripe = sscanf_json(NULL, rec["stripe"]), }, .version = rec["ver"].uint64_value(), .offset = (uint32_t)rec["offset"].uint64_value(), .len = (uint32_t)rec["len"].uint64_value(), .data_offset = (uint64_t)(new_journal_data-new_journal_buf), - .crc32_data = (uint32_t)sscanf_num("%x", rec["data_crc32"].string_value()), + .crc32_data = (uint32_t)sscanf_json("%x", rec["data_crc32"]), }; fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write)); fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data); @@ -448,11 +420,11 @@ int disk_tool_t::write_json_journal(json11::Json entries) .size = entry_size, .crc32_prev = new_crc32_prev, .oid = { - .inode = sscanf_num("0x%lx", rec["inode"].string_value()), - .stripe = sscanf_num("0x%lx", rec["stripe"].string_value()), + .inode = sscanf_json(NULL, rec["inode"]), + .stripe = sscanf_json(NULL, rec["stripe"]), }, .version = rec["ver"].uint64_value(), - .location = sscanf_num("0x%lx", rec["loc"].string_value()), + .location = sscanf_json(NULL, rec["loc"]), }; fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write)); } @@ -464,8 +436,8 @@ int disk_tool_t::write_json_journal(json11::Json entries) .size = entry_size, .crc32_prev = new_crc32_prev, .oid = { - .inode = sscanf_num("0x%lx", rec["inode"].string_value()), - .stripe = sscanf_num("0x%lx", rec["stripe"].string_value()), + .inode = sscanf_json(NULL, rec["inode"]), + .stripe = sscanf_json(NULL, rec["stripe"]), }, .version = rec["ver"].uint64_value(), }; diff --git a/src/disk_tool_meta.cpp b/src/disk_tool_meta.cpp index 1edf5cc0..2e9f04f1 100644 --- a/src/disk_tool_meta.cpp +++ b/src/disk_tool_meta.cpp @@ -156,3 +156,46 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u } first = false; } + +int disk_tool_t::write_json_meta(json11::Json meta) +{ + new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len); + memset(new_meta_buf, 0, new_meta_len); + blockstore_meta_header_v1_t *new_hdr = (blockstore_meta_header_v1_t *)new_meta_buf; + new_hdr->zero = 0; + new_hdr->magic = BLOCKSTORE_META_MAGIC_V1; + new_hdr->version = BLOCKSTORE_META_VERSION_V1; + new_hdr->meta_block_size = meta["meta_block_size"].uint64_value() + ? meta["meta_block_size"].uint64_value() : 4096; + new_hdr->data_block_size = meta["data_block_size"].uint64_value() + ? meta["data_block_size"].uint64_value() : 131072; + new_hdr->bitmap_granularity = meta["bitmap_granularity"].uint64_value() + ? meta["bitmap_granularity"].uint64_value() : 4096; + new_clean_entry_bitmap_size = new_hdr->data_block_size / new_hdr->bitmap_granularity / 8; + new_clean_entry_size = sizeof(clean_disk_entry) + 2*new_clean_entry_bitmap_size; + new_entries_per_block = new_hdr->meta_block_size / new_clean_entry_size; + for (const auto & e: meta["entries"].array_items()) + { + uint64_t data_block = e["block"].uint64_value(); + uint64_t mb = 1 + data_block/new_entries_per_block; + if (mb >= new_meta_len/new_hdr->meta_block_size) + { + free(new_meta_buf); + new_meta_buf = NULL; + fprintf(stderr, "Metadata (data block %lu) doesn't fit into the new area\n", data_block); + return 1; + } + clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + + new_hdr->meta_block_size*mb + + new_clean_entry_size*(data_block % new_entries_per_block)); + new_entry->oid.inode = (sscanf_json(NULL, e["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, e["inode"]); + new_entry->oid.stripe = sscanf_json(NULL, e["stripe"]); + new_entry->version = sscanf_json(NULL, e["version"]); + fromhexstr(e["bitmap"].string_value(), new_clean_entry_bitmap_size, ((uint8_t*)new_entry) + sizeof(clean_disk_entry)); + fromhexstr(e["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, ((uint8_t*)new_entry) + sizeof(clean_disk_entry) + new_clean_entry_bitmap_size); + } + int r = resize_write_new_meta(); + free(new_meta_buf); + new_meta_buf = NULL; + return r; +} diff --git a/src/disk_tool_utils.cpp b/src/disk_tool_utils.cpp index 9eef0088..6bdababe 100644 --- a/src/disk_tool_utils.cpp +++ b/src/disk_tool_utils.cpp @@ -7,6 +7,40 @@ #include "rw_blocking.h" #include "str_util.h" +uint64_t sscanf_json(const char *fmt, const json11::Json & str) +{ + uint64_t value = 0; + if (fmt) + sscanf(str.string_value().c_str(), "%lx", &value); + else if (str.string_value().size() > 2 && (str.string_value()[0] == '0' && str.string_value()[1] == 'x')) + sscanf(str.string_value().c_str(), "0x%lx", &value); + else + value = str.uint64_value(); + return value; +} + +static int fromhex(char c) +{ + if (c >= '0' && c <= '9') + return (c-'0'); + else if (c >= 'a' && c <= 'f') + return (c-'a'+10); + else if (c >= 'A' && c <= 'F') + return (c-'A'+10); + return -1; +} + +void fromhexstr(const std::string & from, int bytes, uint8_t *to) +{ + for (int i = 0; i < from.size() && i < bytes; i++) + { + int x = fromhex(from[2*i]), y = fromhex(from[2*i+1]); + if (x < 0 || y < 0) + break; + to[i] = x*16 + y; + } +} + std::string realpath_str(std::string path, bool nofail) { char *p = realpath((char*)path.c_str(), NULL);