forked from vitalif/vitastor
Add write-journal command (for debug)
parent
1407db9c08
commit
a52f2b0e8f
|
@ -93,11 +93,16 @@ static const char *help_text =
|
||||||
" For now, this only checks that device cache is in write-through mode if fsync is disabled.\n"
|
" For now, this only checks that device cache is in write-through mode if fsync is disabled.\n"
|
||||||
" Intended for use from startup scripts (i.e. from systemd units).\n"
|
" Intended for use from startup scripts (i.e. from systemd units).\n"
|
||||||
"\n"
|
"\n"
|
||||||
"vitastor-disk dump-journal [--all] [--json] <journal_file> <journal_block_size> <offset> <size>\n"
|
"vitastor-disk dump-journal [--all] [--json [--format entries|blocks|data]] <journal_file> <journal_block_size> <offset> <size>\n"
|
||||||
" Dump journal in human-readable or JSON (if --json is specified) format.\n"
|
" Dump journal in human-readable or JSON (if --json is specified) format.\n"
|
||||||
" Without --all, only actual part of the journal is dumped.\n"
|
" Without --all, only actual part of the journal is dumped.\n"
|
||||||
" With --all, the whole journal area is scanned for journal entries,\n"
|
" With --all, the whole journal area is scanned for journal entries,\n"
|
||||||
" some of which may be outdated.\n"
|
" some of which may be outdated.\n"
|
||||||
|
" In JSON mode, the journal by default is dumped with --format entries.\n"
|
||||||
|
"\n"
|
||||||
|
"vitastor-disk write-journal <journal_file> <journal_block_size> <bitmap_size> <offset> <size>\n"
|
||||||
|
" Write journal from JSON taken from standard input in the same format as produced by\n"
|
||||||
|
" dump-journal --json --format data.\n"
|
||||||
"\n"
|
"\n"
|
||||||
"vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
|
"vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
|
||||||
" Dump metadata in JSON format.\n"
|
" Dump metadata in JSON format.\n"
|
||||||
|
@ -180,7 +185,7 @@ int main(int argc, char *argv[])
|
||||||
{
|
{
|
||||||
if (cmd.size() < 5)
|
if (cmd.size() < 5)
|
||||||
{
|
{
|
||||||
fprintf(stderr, "USAGE: %s%s [--all] [--json] <journal_file> <journal_block_size> <offset> <size>\n", argv[0], aliased ? "" : " dump-journal");
|
print_help(help_text, "vitastor-disk", cmd[0], false);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
self.dsk.journal_device = cmd[1];
|
self.dsk.journal_device = cmd[1];
|
||||||
|
@ -189,11 +194,32 @@ int main(int argc, char *argv[])
|
||||||
self.dsk.journal_len = strtoull(cmd[4], NULL, 10);
|
self.dsk.journal_len = strtoull(cmd[4], NULL, 10);
|
||||||
return self.dump_journal();
|
return self.dump_journal();
|
||||||
}
|
}
|
||||||
|
else if (!strcmp(cmd[0], "write-journal"))
|
||||||
|
{
|
||||||
|
if (cmd.size() < 6)
|
||||||
|
{
|
||||||
|
print_help(help_text, "vitastor-disk", cmd[0], false);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
self.new_journal_device = cmd[1];
|
||||||
|
self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10);
|
||||||
|
self.dsk.clean_entry_bitmap_size = strtoul(cmd[3], NULL, 10);
|
||||||
|
self.new_journal_offset = strtoull(cmd[4], NULL, 10);
|
||||||
|
self.new_journal_len = strtoull(cmd[5], NULL, 10);
|
||||||
|
std::string json_err;
|
||||||
|
json11::Json entries = json11::Json::parse(read_all_fd(0), json_err);
|
||||||
|
if (json_err != "")
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Invalid JSON: %s\n", json_err.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return self.write_json_journal(entries);
|
||||||
|
}
|
||||||
else if (!strcmp(cmd[0], "dump-meta"))
|
else if (!strcmp(cmd[0], "dump-meta"))
|
||||||
{
|
{
|
||||||
if (cmd.size() < 5)
|
if (cmd.size() < 5)
|
||||||
{
|
{
|
||||||
fprintf(stderr, "USAGE: %s dump-meta <meta_file> <meta_block_size> <offset> <size>\n", argv[0]);
|
print_help(help_text, "vitastor-disk", cmd[0], false);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
self.dsk.meta_device = cmd[1];
|
self.dsk.meta_device = cmd[1];
|
||||||
|
|
|
@ -41,6 +41,7 @@ struct disk_tool_t
|
||||||
|
|
||||||
std::map<std::string, std::string> options;
|
std::map<std::string, std::string> options;
|
||||||
bool all, json, now;
|
bool all, json, now;
|
||||||
|
bool dump_with_blocks, dump_with_data;
|
||||||
blockstore_disk_t dsk;
|
blockstore_disk_t dsk;
|
||||||
|
|
||||||
// resize data and/or move metadata and journal
|
// resize data and/or move metadata and journal
|
||||||
|
@ -77,24 +78,22 @@ struct disk_tool_t
|
||||||
uint32_t crc32_last;
|
uint32_t crc32_last;
|
||||||
uint32_t new_crc32_prev;
|
uint32_t new_crc32_prev;
|
||||||
|
|
||||||
/**** Commands ****/
|
|
||||||
|
|
||||||
int dump_journal();
|
|
||||||
int dump_meta();
|
|
||||||
int resize_data();
|
|
||||||
|
|
||||||
/**** Methods ****/
|
|
||||||
|
|
||||||
~disk_tool_t();
|
~disk_tool_t();
|
||||||
|
|
||||||
|
int dump_journal();
|
||||||
void dump_journal_entry(int num, journal_entry *je, bool json);
|
void dump_journal_entry(int num, journal_entry *je, bool json);
|
||||||
int process_journal(std::function<int(void*)> block_fn);
|
int process_journal(std::function<int(void*)> block_fn);
|
||||||
int process_journal_block(void *buf, std::function<void(int, journal_entry*)> iter_fn);
|
int process_journal_block(void *buf, std::function<void(int, journal_entry*)> iter_fn);
|
||||||
int process_meta(std::function<void(blockstore_meta_header_v1_t *)> hdr_fn,
|
int process_meta(std::function<void(blockstore_meta_header_v1_t *)> hdr_fn,
|
||||||
std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn);
|
std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn);
|
||||||
|
|
||||||
|
int dump_meta();
|
||||||
void dump_meta_header(blockstore_meta_header_v1_t *hdr);
|
void dump_meta_header(blockstore_meta_header_v1_t *hdr);
|
||||||
void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
|
void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
|
||||||
|
|
||||||
|
int write_json_journal(json11::Json entries);
|
||||||
|
|
||||||
|
int resize_data();
|
||||||
int resize_parse_params();
|
int resize_parse_params();
|
||||||
void resize_init(blockstore_meta_header_v1_t *hdr);
|
void resize_init(blockstore_meta_header_v1_t *hdr);
|
||||||
int resize_remap_blocks();
|
int resize_remap_blocks();
|
||||||
|
|
|
@ -5,6 +5,8 @@
|
||||||
|
|
||||||
int disk_tool_t::dump_journal()
|
int disk_tool_t::dump_journal()
|
||||||
{
|
{
|
||||||
|
dump_with_blocks = options["format"] == "blocks";
|
||||||
|
dump_with_data = options["format"] == "data" || options["format"] == "blocks,data";
|
||||||
if (dsk.journal_block_size < DIRECT_IO_ALIGNMENT || (dsk.journal_block_size % DIRECT_IO_ALIGNMENT) ||
|
if (dsk.journal_block_size < DIRECT_IO_ALIGNMENT || (dsk.journal_block_size % DIRECT_IO_ALIGNMENT) ||
|
||||||
dsk.journal_block_size > 128*1024)
|
dsk.journal_block_size > 128*1024)
|
||||||
{
|
{
|
||||||
|
@ -83,13 +85,17 @@ int disk_tool_t::dump_journal()
|
||||||
{
|
{
|
||||||
if (json && first2)
|
if (json && first2)
|
||||||
{
|
{
|
||||||
|
if (dump_with_blocks)
|
||||||
printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first ? "" : ",\n", pos);
|
printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first ? "" : ",\n", pos);
|
||||||
first = false;
|
first = false;
|
||||||
}
|
}
|
||||||
dump_journal_entry(num, je, json);
|
dump_journal_entry(num, je, json);
|
||||||
});
|
});
|
||||||
if (json)
|
if (json)
|
||||||
printf(first2 ? "" : "\n]}");
|
{
|
||||||
|
if (dump_with_blocks && !first2)
|
||||||
|
printf("\n]}");
|
||||||
|
}
|
||||||
else if (r <= 0)
|
else if (r <= 0)
|
||||||
printf("end of the journal\n");
|
printf("end of the journal\n");
|
||||||
return r;
|
return r;
|
||||||
|
@ -240,6 +246,24 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
|
||||||
printf(json ? ",\"bad_loc\":true,\"calc_loc\":\"0x%lx\""
|
printf(json ? ",\"bad_loc\":true,\"calc_loc\":\"0x%lx\""
|
||||||
: " (mismatched, calculated = %lu)", journal_pos);
|
: " (mismatched, calculated = %lu)", journal_pos);
|
||||||
}
|
}
|
||||||
|
if (je->small_write.size > sizeof(journal_entry_small_write))
|
||||||
|
{
|
||||||
|
printf(json ? ",\"bitmap\":\"" : " (bitmap: ");
|
||||||
|
for (int i = sizeof(journal_entry_small_write); i < je->small_write.size; i++)
|
||||||
|
{
|
||||||
|
printf("%02x", ((uint8_t*)je)[i]);
|
||||||
|
}
|
||||||
|
printf(json ? "\"" : ")");
|
||||||
|
}
|
||||||
|
if (dump_with_data)
|
||||||
|
{
|
||||||
|
printf(json ? ",\"data\":\"" : " (data: ");
|
||||||
|
for (int i = 0; i < je->small_write.len; i++)
|
||||||
|
{
|
||||||
|
printf("%02x", ((uint8_t*)small_write_data)[i]);
|
||||||
|
}
|
||||||
|
printf(json ? "\"" : ")");
|
||||||
|
}
|
||||||
printf(
|
printf(
|
||||||
json ? ",\"data_crc32\":\"%08x\",\"data_valid\":%s}" : " data_crc32=%08x%s\n",
|
json ? ",\"data_crc32\":\"%08x\",\"data_valid\":%s}" : " data_crc32=%08x%s\n",
|
||||||
je->small_write.crc32_data,
|
je->small_write.crc32_data,
|
||||||
|
@ -251,11 +275,21 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
|
||||||
else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
|
else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
|
||||||
{
|
{
|
||||||
printf(
|
printf(
|
||||||
json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"loc\":\"0x%lx\"}"
|
json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"loc\":\"0x%lx\""
|
||||||
: "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx\n",
|
: "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx",
|
||||||
je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
|
je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
|
||||||
je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location
|
je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location
|
||||||
);
|
);
|
||||||
|
if (je->big_write.size > sizeof(journal_entry_big_write))
|
||||||
|
{
|
||||||
|
printf(json ? ",\"bitmap\":\"" : " (bitmap: ");
|
||||||
|
for (int i = sizeof(journal_entry_big_write); i < je->small_write.size; i++)
|
||||||
|
{
|
||||||
|
printf("%02x", ((uint8_t*)je)[i]);
|
||||||
|
}
|
||||||
|
printf(json ? "\"" : ")");
|
||||||
|
}
|
||||||
|
printf(json ? "}" : "\n");
|
||||||
}
|
}
|
||||||
else if (je->type == JE_STABLE)
|
else if (je->type == JE_STABLE)
|
||||||
{
|
{
|
||||||
|
@ -282,3 +316,165 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static uint64_t sscanf_num(const char *fmt, const std::string & str)
|
||||||
|
{
|
||||||
|
uint64_t value = 0;
|
||||||
|
sscanf(str.c_str(), fmt, &value);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int fromhex(char c)
|
||||||
|
{
|
||||||
|
if (c >= '0' && c <= '9')
|
||||||
|
return (c-'0');
|
||||||
|
else if (c >= 'a' && c <= 'f')
|
||||||
|
return (c-'a'+10);
|
||||||
|
else if (c >= 'A' && c <= 'F')
|
||||||
|
return (c-'A'+10);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void fromhexstr(const std::string & from, int bytes, uint8_t *to)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < from.size() && i < bytes; i++)
|
||||||
|
{
|
||||||
|
int x = fromhex(from[2*i]), y = fromhex(from[2*i+1]);
|
||||||
|
if (x < 0 || y < 0)
|
||||||
|
break;
|
||||||
|
to[i] = x*16 + y;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
|
{
|
||||||
|
new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
|
||||||
|
new_journal_ptr = new_journal_buf;
|
||||||
|
new_journal_data = new_journal_ptr + dsk.journal_block_size;
|
||||||
|
new_journal_in_pos = 0;
|
||||||
|
memset(new_journal_buf, 0, new_journal_len);
|
||||||
|
std::map<std::string,uint16_t> type_by_name = {
|
||||||
|
{ "start", JE_START },
|
||||||
|
{ "small_write", JE_SMALL_WRITE },
|
||||||
|
{ "small_write_instant", JE_SMALL_WRITE_INSTANT },
|
||||||
|
{ "big_write", JE_BIG_WRITE },
|
||||||
|
{ "big_write_instant", JE_BIG_WRITE_INSTANT },
|
||||||
|
{ "stable", JE_STABLE },
|
||||||
|
{ "delete", JE_DELETE },
|
||||||
|
{ "rollback", JE_ROLLBACK },
|
||||||
|
};
|
||||||
|
for (const auto & rec: entries.array_items())
|
||||||
|
{
|
||||||
|
auto t_it = type_by_name.find(rec["type"].string_value());
|
||||||
|
if (t_it == type_by_name.end())
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Unknown journal entry type \"%s\", skipping\n", rec["type"].string_value().c_str());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
uint16_t type = t_it->second;
|
||||||
|
uint32_t entry_size = (type == JE_START
|
||||||
|
? sizeof(journal_entry_start)
|
||||||
|
: (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT
|
||||||
|
? sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size
|
||||||
|
: (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT
|
||||||
|
? sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
|
||||||
|
: sizeof(journal_entry_del))));
|
||||||
|
if (dsk.journal_block_size < new_journal_in_pos + entry_size)
|
||||||
|
{
|
||||||
|
new_journal_ptr = new_journal_data;
|
||||||
|
if (new_journal_ptr-new_journal_buf >= new_journal_len)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error: entries don't fit to the new journal\n");
|
||||||
|
free(new_journal_buf);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
new_journal_data = new_journal_ptr+dsk.journal_block_size;
|
||||||
|
new_journal_in_pos = 0;
|
||||||
|
if (dsk.journal_block_size < entry_size)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error: journal entry too large (%u bytes)\n", entry_size);
|
||||||
|
free(new_journal_buf);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
|
||||||
|
if (type == JE_START)
|
||||||
|
{
|
||||||
|
*((journal_entry_start*)ne) = (journal_entry_start){
|
||||||
|
.magic = JOURNAL_MAGIC,
|
||||||
|
.type = type,
|
||||||
|
.size = entry_size,
|
||||||
|
.journal_start = dsk.journal_block_size,
|
||||||
|
.version = JOURNAL_VERSION,
|
||||||
|
};
|
||||||
|
new_journal_ptr += dsk.journal_block_size;
|
||||||
|
new_journal_data = new_journal_ptr+dsk.journal_block_size;
|
||||||
|
new_journal_in_pos = 0;
|
||||||
|
}
|
||||||
|
else if (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT)
|
||||||
|
{
|
||||||
|
if (new_journal_data - new_journal_buf + ne->small_write.len > new_journal_len)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error: entries don't fit to the new journal\n");
|
||||||
|
free(new_journal_buf);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
*((journal_entry_small_write*)ne) = (journal_entry_small_write){
|
||||||
|
.magic = JOURNAL_MAGIC,
|
||||||
|
.type = type,
|
||||||
|
.size = entry_size,
|
||||||
|
.crc32_prev = new_crc32_prev,
|
||||||
|
.oid = {
|
||||||
|
.inode = sscanf_num("0x%lx", rec["inode"].string_value()),
|
||||||
|
.stripe = sscanf_num("0x%lx", rec["stripe"].string_value()),
|
||||||
|
},
|
||||||
|
.version = rec["ver"].uint64_value(),
|
||||||
|
.offset = (uint32_t)rec["offset"].uint64_value(),
|
||||||
|
.len = (uint32_t)rec["len"].uint64_value(),
|
||||||
|
.data_offset = (uint64_t)(new_journal_data-new_journal_buf),
|
||||||
|
.crc32_data = (uint32_t)sscanf_num("%x", rec["data_crc32"].string_value()),
|
||||||
|
};
|
||||||
|
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
|
||||||
|
fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
|
||||||
|
if (rec["data"].is_string())
|
||||||
|
ne->small_write.crc32_data = crc32c(0, new_journal_data, ne->small_write.len);
|
||||||
|
new_journal_data += ne->small_write.len;
|
||||||
|
}
|
||||||
|
else if (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT)
|
||||||
|
{
|
||||||
|
*((journal_entry_big_write*)ne) = (journal_entry_big_write){
|
||||||
|
.magic = JOURNAL_MAGIC,
|
||||||
|
.type = type,
|
||||||
|
.size = entry_size,
|
||||||
|
.crc32_prev = new_crc32_prev,
|
||||||
|
.oid = {
|
||||||
|
.inode = sscanf_num("0x%lx", rec["inode"].string_value()),
|
||||||
|
.stripe = sscanf_num("0x%lx", rec["stripe"].string_value()),
|
||||||
|
},
|
||||||
|
.version = rec["ver"].uint64_value(),
|
||||||
|
.location = sscanf_num("0x%lx", rec["loc"].string_value()),
|
||||||
|
};
|
||||||
|
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
|
||||||
|
}
|
||||||
|
else if (type == JE_STABLE || type == JE_ROLLBACK || type == JE_DELETE)
|
||||||
|
{
|
||||||
|
*((journal_entry_del*)ne) = (journal_entry_del){
|
||||||
|
.magic = JOURNAL_MAGIC,
|
||||||
|
.type = type,
|
||||||
|
.size = entry_size,
|
||||||
|
.crc32_prev = new_crc32_prev,
|
||||||
|
.oid = {
|
||||||
|
.inode = sscanf_num("0x%lx", rec["inode"].string_value()),
|
||||||
|
.stripe = sscanf_num("0x%lx", rec["stripe"].string_value()),
|
||||||
|
},
|
||||||
|
.version = rec["ver"].uint64_value(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
ne->crc32 = je_crc32(ne);
|
||||||
|
new_crc32_prev = ne->crc32;
|
||||||
|
new_journal_in_pos += ne->size;
|
||||||
|
}
|
||||||
|
int r = resize_write_new_journal();
|
||||||
|
free(new_journal_buf);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue