From a52f2b0e8ff58dffe6d9cb89442b84617fc3a9ca Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 20 Aug 2022 14:05:53 +0300 Subject: [PATCH] Add write-journal command (for debug) --- src/disk_tool.cpp | 32 +++++- src/disk_tool.h | 15 ++- src/disk_tool_journal.cpp | 204 +++++++++++++++++++++++++++++++++++++- 3 files changed, 236 insertions(+), 15 deletions(-) diff --git a/src/disk_tool.cpp b/src/disk_tool.cpp index 886d3f73..f74356b2 100644 --- a/src/disk_tool.cpp +++ b/src/disk_tool.cpp @@ -93,11 +93,16 @@ static const char *help_text = " For now, this only checks that device cache is in write-through mode if fsync is disabled.\n" " Intended for use from startup scripts (i.e. from systemd units).\n" "\n" - "vitastor-disk dump-journal [--all] [--json] \n" + "vitastor-disk dump-journal [--all] [--json [--format entries|blocks|data]] \n" " Dump journal in human-readable or JSON (if --json is specified) format.\n" " Without --all, only actual part of the journal is dumped.\n" " With --all, the whole journal area is scanned for journal entries,\n" " some of which may be outdated.\n" + " In JSON mode, the journal by default is dumped with --format entries.\n" + "\n" + "vitastor-disk write-journal \n" + " Write journal from JSON taken from standard input in the same format as produced by\n" + " dump-journal --json --format data.\n" "\n" "vitastor-disk dump-meta \n" " Dump metadata in JSON format.\n" @@ -180,7 +185,7 @@ int main(int argc, char *argv[]) { if (cmd.size() < 5) { - fprintf(stderr, "USAGE: %s%s [--all] [--json] \n", argv[0], aliased ? "" : " dump-journal"); + print_help(help_text, "vitastor-disk", cmd[0], false); return 1; } self.dsk.journal_device = cmd[1]; @@ -189,11 +194,32 @@ int main(int argc, char *argv[]) self.dsk.journal_len = strtoull(cmd[4], NULL, 10); return self.dump_journal(); } + else if (!strcmp(cmd[0], "write-journal")) + { + if (cmd.size() < 6) + { + print_help(help_text, "vitastor-disk", cmd[0], false); + return 1; + } + self.new_journal_device = cmd[1]; + self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10); + self.dsk.clean_entry_bitmap_size = strtoul(cmd[3], NULL, 10); + self.new_journal_offset = strtoull(cmd[4], NULL, 10); + self.new_journal_len = strtoull(cmd[5], NULL, 10); + std::string json_err; + json11::Json entries = json11::Json::parse(read_all_fd(0), json_err); + if (json_err != "") + { + fprintf(stderr, "Invalid JSON: %s\n", json_err.c_str()); + return 1; + } + return self.write_json_journal(entries); + } else if (!strcmp(cmd[0], "dump-meta")) { if (cmd.size() < 5) { - fprintf(stderr, "USAGE: %s dump-meta \n", argv[0]); + print_help(help_text, "vitastor-disk", cmd[0], false); return 1; } self.dsk.meta_device = cmd[1]; diff --git a/src/disk_tool.h b/src/disk_tool.h index 6209322d..bfe6ba75 100644 --- a/src/disk_tool.h +++ b/src/disk_tool.h @@ -41,6 +41,7 @@ struct disk_tool_t std::map options; bool all, json, now; + bool dump_with_blocks, dump_with_data; blockstore_disk_t dsk; // resize data and/or move metadata and journal @@ -77,24 +78,22 @@ struct disk_tool_t uint32_t crc32_last; uint32_t new_crc32_prev; - /**** Commands ****/ - - int dump_journal(); - int dump_meta(); - int resize_data(); - - /**** Methods ****/ - ~disk_tool_t(); + int dump_journal(); void dump_journal_entry(int num, journal_entry *je, bool json); int process_journal(std::function block_fn); int process_journal_block(void *buf, std::function iter_fn); int process_meta(std::function hdr_fn, std::function record_fn); + + int dump_meta(); void dump_meta_header(blockstore_meta_header_v1_t *hdr); void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap); + int write_json_journal(json11::Json entries); + + int resize_data(); int resize_parse_params(); void resize_init(blockstore_meta_header_v1_t *hdr); int resize_remap_blocks(); diff --git a/src/disk_tool_journal.cpp b/src/disk_tool_journal.cpp index 24e2f31d..2446cb41 100644 --- a/src/disk_tool_journal.cpp +++ b/src/disk_tool_journal.cpp @@ -5,6 +5,8 @@ int disk_tool_t::dump_journal() { + dump_with_blocks = options["format"] == "blocks"; + dump_with_data = options["format"] == "data" || options["format"] == "blocks,data"; if (dsk.journal_block_size < DIRECT_IO_ALIGNMENT || (dsk.journal_block_size % DIRECT_IO_ALIGNMENT) || dsk.journal_block_size > 128*1024) { @@ -83,13 +85,17 @@ int disk_tool_t::dump_journal() { if (json && first2) { - printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first ? "" : ",\n", pos); + if (dump_with_blocks) + printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first ? "" : ",\n", pos); first = false; } dump_journal_entry(num, je, json); }); if (json) - printf(first2 ? "" : "\n]}"); + { + if (dump_with_blocks && !first2) + printf("\n]}"); + } else if (r <= 0) printf("end of the journal\n"); return r; @@ -240,6 +246,24 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json) printf(json ? ",\"bad_loc\":true,\"calc_loc\":\"0x%lx\"" : " (mismatched, calculated = %lu)", journal_pos); } + if (je->small_write.size > sizeof(journal_entry_small_write)) + { + printf(json ? ",\"bitmap\":\"" : " (bitmap: "); + for (int i = sizeof(journal_entry_small_write); i < je->small_write.size; i++) + { + printf("%02x", ((uint8_t*)je)[i]); + } + printf(json ? "\"" : ")"); + } + if (dump_with_data) + { + printf(json ? ",\"data\":\"" : " (data: "); + for (int i = 0; i < je->small_write.len; i++) + { + printf("%02x", ((uint8_t*)small_write_data)[i]); + } + printf(json ? "\"" : ")"); + } printf( json ? ",\"data_crc32\":\"%08x\",\"data_valid\":%s}" : " data_crc32=%08x%s\n", je->small_write.crc32_data, @@ -251,11 +275,21 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json) else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT) { printf( - json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"loc\":\"0x%lx\"}" - : "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx\n", + json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"loc\":\"0x%lx\"" + : "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx", je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "", je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location ); + if (je->big_write.size > sizeof(journal_entry_big_write)) + { + printf(json ? ",\"bitmap\":\"" : " (bitmap: "); + for (int i = sizeof(journal_entry_big_write); i < je->small_write.size; i++) + { + printf("%02x", ((uint8_t*)je)[i]); + } + printf(json ? "\"" : ")"); + } + printf(json ? "}" : "\n"); } else if (je->type == JE_STABLE) { @@ -282,3 +316,165 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json) ); } } + +static uint64_t sscanf_num(const char *fmt, const std::string & str) +{ + uint64_t value = 0; + sscanf(str.c_str(), fmt, &value); + return value; +} + +static int fromhex(char c) +{ + if (c >= '0' && c <= '9') + return (c-'0'); + else if (c >= 'a' && c <= 'f') + return (c-'a'+10); + else if (c >= 'A' && c <= 'F') + return (c-'A'+10); + return -1; +} + +static void fromhexstr(const std::string & from, int bytes, uint8_t *to) +{ + for (int i = 0; i < from.size() && i < bytes; i++) + { + int x = fromhex(from[2*i]), y = fromhex(from[2*i+1]); + if (x < 0 || y < 0) + break; + to[i] = x*16 + y; + } +} + +int disk_tool_t::write_json_journal(json11::Json entries) +{ + new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len); + new_journal_ptr = new_journal_buf; + new_journal_data = new_journal_ptr + dsk.journal_block_size; + new_journal_in_pos = 0; + memset(new_journal_buf, 0, new_journal_len); + std::map type_by_name = { + { "start", JE_START }, + { "small_write", JE_SMALL_WRITE }, + { "small_write_instant", JE_SMALL_WRITE_INSTANT }, + { "big_write", JE_BIG_WRITE }, + { "big_write_instant", JE_BIG_WRITE_INSTANT }, + { "stable", JE_STABLE }, + { "delete", JE_DELETE }, + { "rollback", JE_ROLLBACK }, + }; + for (const auto & rec: entries.array_items()) + { + auto t_it = type_by_name.find(rec["type"].string_value()); + if (t_it == type_by_name.end()) + { + fprintf(stderr, "Unknown journal entry type \"%s\", skipping\n", rec["type"].string_value().c_str()); + continue; + } + uint16_t type = t_it->second; + uint32_t entry_size = (type == JE_START + ? sizeof(journal_entry_start) + : (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT + ? sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size + : (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT + ? sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size + : sizeof(journal_entry_del)))); + if (dsk.journal_block_size < new_journal_in_pos + entry_size) + { + new_journal_ptr = new_journal_data; + if (new_journal_ptr-new_journal_buf >= new_journal_len) + { + fprintf(stderr, "Error: entries don't fit to the new journal\n"); + free(new_journal_buf); + return 1; + } + new_journal_data = new_journal_ptr+dsk.journal_block_size; + new_journal_in_pos = 0; + if (dsk.journal_block_size < entry_size) + { + fprintf(stderr, "Error: journal entry too large (%u bytes)\n", entry_size); + free(new_journal_buf); + return 1; + } + } + journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos); + if (type == JE_START) + { + *((journal_entry_start*)ne) = (journal_entry_start){ + .magic = JOURNAL_MAGIC, + .type = type, + .size = entry_size, + .journal_start = dsk.journal_block_size, + .version = JOURNAL_VERSION, + }; + new_journal_ptr += dsk.journal_block_size; + new_journal_data = new_journal_ptr+dsk.journal_block_size; + new_journal_in_pos = 0; + } + else if (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT) + { + if (new_journal_data - new_journal_buf + ne->small_write.len > new_journal_len) + { + fprintf(stderr, "Error: entries don't fit to the new journal\n"); + free(new_journal_buf); + return 1; + } + *((journal_entry_small_write*)ne) = (journal_entry_small_write){ + .magic = JOURNAL_MAGIC, + .type = type, + .size = entry_size, + .crc32_prev = new_crc32_prev, + .oid = { + .inode = sscanf_num("0x%lx", rec["inode"].string_value()), + .stripe = sscanf_num("0x%lx", rec["stripe"].string_value()), + }, + .version = rec["ver"].uint64_value(), + .offset = (uint32_t)rec["offset"].uint64_value(), + .len = (uint32_t)rec["len"].uint64_value(), + .data_offset = (uint64_t)(new_journal_data-new_journal_buf), + .crc32_data = (uint32_t)sscanf_num("%x", rec["data_crc32"].string_value()), + }; + fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write)); + fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data); + if (rec["data"].is_string()) + ne->small_write.crc32_data = crc32c(0, new_journal_data, ne->small_write.len); + new_journal_data += ne->small_write.len; + } + else if (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT) + { + *((journal_entry_big_write*)ne) = (journal_entry_big_write){ + .magic = JOURNAL_MAGIC, + .type = type, + .size = entry_size, + .crc32_prev = new_crc32_prev, + .oid = { + .inode = sscanf_num("0x%lx", rec["inode"].string_value()), + .stripe = sscanf_num("0x%lx", rec["stripe"].string_value()), + }, + .version = rec["ver"].uint64_value(), + .location = sscanf_num("0x%lx", rec["loc"].string_value()), + }; + fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write)); + } + else if (type == JE_STABLE || type == JE_ROLLBACK || type == JE_DELETE) + { + *((journal_entry_del*)ne) = (journal_entry_del){ + .magic = JOURNAL_MAGIC, + .type = type, + .size = entry_size, + .crc32_prev = new_crc32_prev, + .oid = { + .inode = sscanf_num("0x%lx", rec["inode"].string_value()), + .stripe = sscanf_num("0x%lx", rec["stripe"].string_value()), + }, + .version = rec["ver"].uint64_value(), + }; + } + ne->crc32 = je_crc32(ne); + new_crc32_prev = ne->crc32; + new_journal_in_pos += ne->size; + } + int r = resize_write_new_journal(); + free(new_journal_buf); + return r; +}