forked from vitalif/vitastor
Add write-meta command (for debug)
parent
a52f2b0e8f
commit
29b40aba93
|
@ -107,6 +107,10 @@ static const char *help_text =
|
||||||
"vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
|
"vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
|
||||||
" Dump metadata in JSON format.\n"
|
" Dump metadata in JSON format.\n"
|
||||||
"\n"
|
"\n"
|
||||||
|
"vitastor-disk write-meta <meta_file> <offset> <size>\n"
|
||||||
|
" Write metadata from JSON taken from standard input in the same format as produced by\n"
|
||||||
|
" dump-meta --json.\n"
|
||||||
|
"\n"
|
||||||
"vitastor-disk simple-offsets <device>\n"
|
"vitastor-disk simple-offsets <device>\n"
|
||||||
" Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n"
|
" Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n"
|
||||||
" --object_size 128k Set blockstore block size\n"
|
" --object_size 128k Set blockstore block size\n"
|
||||||
|
@ -185,7 +189,7 @@ int main(int argc, char *argv[])
|
||||||
{
|
{
|
||||||
if (cmd.size() < 5)
|
if (cmd.size() < 5)
|
||||||
{
|
{
|
||||||
print_help(help_text, "vitastor-disk", cmd[0], false);
|
print_help(help_text, aliased ? "vitastor-dump-journal" : "vitastor-disk", cmd[0], false);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
self.dsk.journal_device = cmd[1];
|
self.dsk.journal_device = cmd[1];
|
||||||
|
@ -228,6 +232,25 @@ int main(int argc, char *argv[])
|
||||||
self.dsk.meta_len = strtoull(cmd[4], NULL, 10);
|
self.dsk.meta_len = strtoull(cmd[4], NULL, 10);
|
||||||
return self.dump_meta();
|
return self.dump_meta();
|
||||||
}
|
}
|
||||||
|
else if (!strcmp(cmd[0], "write-meta"))
|
||||||
|
{
|
||||||
|
if (cmd.size() < 4)
|
||||||
|
{
|
||||||
|
print_help(help_text, "vitastor-disk", cmd[0], false);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
self.new_meta_device = cmd[1];
|
||||||
|
self.new_meta_offset = strtoull(cmd[2], NULL, 10);
|
||||||
|
self.new_meta_len = strtoull(cmd[3], NULL, 10);
|
||||||
|
std::string json_err;
|
||||||
|
json11::Json meta = json11::Json::parse(read_all_fd(0), json_err);
|
||||||
|
if (json_err != "")
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Invalid JSON: %s\n", json_err.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return self.write_json_meta(meta);
|
||||||
|
}
|
||||||
else if (!strcmp(cmd[0], "resize"))
|
else if (!strcmp(cmd[0], "resize"))
|
||||||
{
|
{
|
||||||
return self.resize_data();
|
return self.resize_data();
|
||||||
|
|
|
@ -92,6 +92,7 @@ struct disk_tool_t
|
||||||
void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
|
void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
|
||||||
|
|
||||||
int write_json_journal(json11::Json entries);
|
int write_json_journal(json11::Json entries);
|
||||||
|
int write_json_meta(json11::Json meta);
|
||||||
|
|
||||||
int resize_data();
|
int resize_data();
|
||||||
int resize_parse_params();
|
int resize_parse_params();
|
||||||
|
@ -125,6 +126,8 @@ struct disk_tool_t
|
||||||
|
|
||||||
void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
|
void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
|
||||||
|
|
||||||
|
uint64_t sscanf_json(const char *fmt, const json11::Json & str);
|
||||||
|
void fromhexstr(const std::string & from, int bytes, uint8_t *to);
|
||||||
std::string realpath_str(std::string path, bool nofail = true);
|
std::string realpath_str(std::string path, bool nofail = true);
|
||||||
std::string read_all_fd(int fd);
|
std::string read_all_fd(int fd);
|
||||||
std::string read_file(std::string file, bool allow_enoent = false);
|
std::string read_file(std::string file, bool allow_enoent = false);
|
||||||
|
|
|
@ -317,35 +317,6 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint64_t sscanf_num(const char *fmt, const std::string & str)
|
|
||||||
{
|
|
||||||
uint64_t value = 0;
|
|
||||||
sscanf(str.c_str(), fmt, &value);
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int fromhex(char c)
|
|
||||||
{
|
|
||||||
if (c >= '0' && c <= '9')
|
|
||||||
return (c-'0');
|
|
||||||
else if (c >= 'a' && c <= 'f')
|
|
||||||
return (c-'a'+10);
|
|
||||||
else if (c >= 'A' && c <= 'F')
|
|
||||||
return (c-'A'+10);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void fromhexstr(const std::string & from, int bytes, uint8_t *to)
|
|
||||||
{
|
|
||||||
for (int i = 0; i < from.size() && i < bytes; i++)
|
|
||||||
{
|
|
||||||
int x = fromhex(from[2*i]), y = fromhex(from[2*i+1]);
|
|
||||||
if (x < 0 || y < 0)
|
|
||||||
break;
|
|
||||||
to[i] = x*16 + y;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int disk_tool_t::write_json_journal(json11::Json entries)
|
int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
{
|
{
|
||||||
new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
|
new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
|
||||||
|
@ -363,6 +334,18 @@ int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
{ "delete", JE_DELETE },
|
{ "delete", JE_DELETE },
|
||||||
{ "rollback", JE_ROLLBACK },
|
{ "rollback", JE_ROLLBACK },
|
||||||
};
|
};
|
||||||
|
// Write start entry into the first block
|
||||||
|
*((journal_entry_start*)new_journal_buf) = (journal_entry_start){
|
||||||
|
.magic = JOURNAL_MAGIC,
|
||||||
|
.type = JE_START,
|
||||||
|
.size = sizeof(journal_entry_start),
|
||||||
|
.journal_start = dsk.journal_block_size,
|
||||||
|
.version = JOURNAL_VERSION,
|
||||||
|
};
|
||||||
|
((journal_entry*)new_journal_buf)->crc32 = je_crc32((journal_entry*)new_journal_buf);
|
||||||
|
new_journal_ptr += dsk.journal_block_size;
|
||||||
|
new_journal_data = new_journal_ptr+dsk.journal_block_size;
|
||||||
|
new_journal_in_pos = 0;
|
||||||
for (const auto & rec: entries.array_items())
|
for (const auto & rec: entries.array_items())
|
||||||
{
|
{
|
||||||
auto t_it = type_by_name.find(rec["type"].string_value());
|
auto t_it = type_by_name.find(rec["type"].string_value());
|
||||||
|
@ -372,6 +355,8 @@ int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
uint16_t type = t_it->second;
|
uint16_t type = t_it->second;
|
||||||
|
if (type == JE_START)
|
||||||
|
continue;
|
||||||
uint32_t entry_size = (type == JE_START
|
uint32_t entry_size = (type == JE_START
|
||||||
? sizeof(journal_entry_start)
|
? sizeof(journal_entry_start)
|
||||||
: (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT
|
: (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT
|
||||||
|
@ -398,20 +383,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
|
journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
|
||||||
if (type == JE_START)
|
if (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT)
|
||||||
{
|
|
||||||
*((journal_entry_start*)ne) = (journal_entry_start){
|
|
||||||
.magic = JOURNAL_MAGIC,
|
|
||||||
.type = type,
|
|
||||||
.size = entry_size,
|
|
||||||
.journal_start = dsk.journal_block_size,
|
|
||||||
.version = JOURNAL_VERSION,
|
|
||||||
};
|
|
||||||
new_journal_ptr += dsk.journal_block_size;
|
|
||||||
new_journal_data = new_journal_ptr+dsk.journal_block_size;
|
|
||||||
new_journal_in_pos = 0;
|
|
||||||
}
|
|
||||||
else if (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT)
|
|
||||||
{
|
{
|
||||||
if (new_journal_data - new_journal_buf + ne->small_write.len > new_journal_len)
|
if (new_journal_data - new_journal_buf + ne->small_write.len > new_journal_len)
|
||||||
{
|
{
|
||||||
|
@ -425,14 +397,14 @@ int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
.size = entry_size,
|
.size = entry_size,
|
||||||
.crc32_prev = new_crc32_prev,
|
.crc32_prev = new_crc32_prev,
|
||||||
.oid = {
|
.oid = {
|
||||||
.inode = sscanf_num("0x%lx", rec["inode"].string_value()),
|
.inode = sscanf_json(NULL, rec["inode"]),
|
||||||
.stripe = sscanf_num("0x%lx", rec["stripe"].string_value()),
|
.stripe = sscanf_json(NULL, rec["stripe"]),
|
||||||
},
|
},
|
||||||
.version = rec["ver"].uint64_value(),
|
.version = rec["ver"].uint64_value(),
|
||||||
.offset = (uint32_t)rec["offset"].uint64_value(),
|
.offset = (uint32_t)rec["offset"].uint64_value(),
|
||||||
.len = (uint32_t)rec["len"].uint64_value(),
|
.len = (uint32_t)rec["len"].uint64_value(),
|
||||||
.data_offset = (uint64_t)(new_journal_data-new_journal_buf),
|
.data_offset = (uint64_t)(new_journal_data-new_journal_buf),
|
||||||
.crc32_data = (uint32_t)sscanf_num("%x", rec["data_crc32"].string_value()),
|
.crc32_data = (uint32_t)sscanf_json("%x", rec["data_crc32"]),
|
||||||
};
|
};
|
||||||
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
|
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
|
||||||
fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
|
fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
|
||||||
|
@ -448,11 +420,11 @@ int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
.size = entry_size,
|
.size = entry_size,
|
||||||
.crc32_prev = new_crc32_prev,
|
.crc32_prev = new_crc32_prev,
|
||||||
.oid = {
|
.oid = {
|
||||||
.inode = sscanf_num("0x%lx", rec["inode"].string_value()),
|
.inode = sscanf_json(NULL, rec["inode"]),
|
||||||
.stripe = sscanf_num("0x%lx", rec["stripe"].string_value()),
|
.stripe = sscanf_json(NULL, rec["stripe"]),
|
||||||
},
|
},
|
||||||
.version = rec["ver"].uint64_value(),
|
.version = rec["ver"].uint64_value(),
|
||||||
.location = sscanf_num("0x%lx", rec["loc"].string_value()),
|
.location = sscanf_json(NULL, rec["loc"]),
|
||||||
};
|
};
|
||||||
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
|
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
|
||||||
}
|
}
|
||||||
|
@ -464,8 +436,8 @@ int disk_tool_t::write_json_journal(json11::Json entries)
|
||||||
.size = entry_size,
|
.size = entry_size,
|
||||||
.crc32_prev = new_crc32_prev,
|
.crc32_prev = new_crc32_prev,
|
||||||
.oid = {
|
.oid = {
|
||||||
.inode = sscanf_num("0x%lx", rec["inode"].string_value()),
|
.inode = sscanf_json(NULL, rec["inode"]),
|
||||||
.stripe = sscanf_num("0x%lx", rec["stripe"].string_value()),
|
.stripe = sscanf_json(NULL, rec["stripe"]),
|
||||||
},
|
},
|
||||||
.version = rec["ver"].uint64_value(),
|
.version = rec["ver"].uint64_value(),
|
||||||
};
|
};
|
||||||
|
|
|
@ -156,3 +156,46 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
|
||||||
}
|
}
|
||||||
first = false;
|
first = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int disk_tool_t::write_json_meta(json11::Json meta)
|
||||||
|
{
|
||||||
|
new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
|
||||||
|
memset(new_meta_buf, 0, new_meta_len);
|
||||||
|
blockstore_meta_header_v1_t *new_hdr = (blockstore_meta_header_v1_t *)new_meta_buf;
|
||||||
|
new_hdr->zero = 0;
|
||||||
|
new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
|
||||||
|
new_hdr->version = BLOCKSTORE_META_VERSION_V1;
|
||||||
|
new_hdr->meta_block_size = meta["meta_block_size"].uint64_value()
|
||||||
|
? meta["meta_block_size"].uint64_value() : 4096;
|
||||||
|
new_hdr->data_block_size = meta["data_block_size"].uint64_value()
|
||||||
|
? meta["data_block_size"].uint64_value() : 131072;
|
||||||
|
new_hdr->bitmap_granularity = meta["bitmap_granularity"].uint64_value()
|
||||||
|
? meta["bitmap_granularity"].uint64_value() : 4096;
|
||||||
|
new_clean_entry_bitmap_size = new_hdr->data_block_size / new_hdr->bitmap_granularity / 8;
|
||||||
|
new_clean_entry_size = sizeof(clean_disk_entry) + 2*new_clean_entry_bitmap_size;
|
||||||
|
new_entries_per_block = new_hdr->meta_block_size / new_clean_entry_size;
|
||||||
|
for (const auto & e: meta["entries"].array_items())
|
||||||
|
{
|
||||||
|
uint64_t data_block = e["block"].uint64_value();
|
||||||
|
uint64_t mb = 1 + data_block/new_entries_per_block;
|
||||||
|
if (mb >= new_meta_len/new_hdr->meta_block_size)
|
||||||
|
{
|
||||||
|
free(new_meta_buf);
|
||||||
|
new_meta_buf = NULL;
|
||||||
|
fprintf(stderr, "Metadata (data block %lu) doesn't fit into the new area\n", data_block);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf +
|
||||||
|
new_hdr->meta_block_size*mb +
|
||||||
|
new_clean_entry_size*(data_block % new_entries_per_block));
|
||||||
|
new_entry->oid.inode = (sscanf_json(NULL, e["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, e["inode"]);
|
||||||
|
new_entry->oid.stripe = sscanf_json(NULL, e["stripe"]);
|
||||||
|
new_entry->version = sscanf_json(NULL, e["version"]);
|
||||||
|
fromhexstr(e["bitmap"].string_value(), new_clean_entry_bitmap_size, ((uint8_t*)new_entry) + sizeof(clean_disk_entry));
|
||||||
|
fromhexstr(e["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, ((uint8_t*)new_entry) + sizeof(clean_disk_entry) + new_clean_entry_bitmap_size);
|
||||||
|
}
|
||||||
|
int r = resize_write_new_meta();
|
||||||
|
free(new_meta_buf);
|
||||||
|
new_meta_buf = NULL;
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
|
@ -7,6 +7,40 @@
|
||||||
#include "rw_blocking.h"
|
#include "rw_blocking.h"
|
||||||
#include "str_util.h"
|
#include "str_util.h"
|
||||||
|
|
||||||
|
uint64_t sscanf_json(const char *fmt, const json11::Json & str)
|
||||||
|
{
|
||||||
|
uint64_t value = 0;
|
||||||
|
if (fmt)
|
||||||
|
sscanf(str.string_value().c_str(), "%lx", &value);
|
||||||
|
else if (str.string_value().size() > 2 && (str.string_value()[0] == '0' && str.string_value()[1] == 'x'))
|
||||||
|
sscanf(str.string_value().c_str(), "0x%lx", &value);
|
||||||
|
else
|
||||||
|
value = str.uint64_value();
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int fromhex(char c)
|
||||||
|
{
|
||||||
|
if (c >= '0' && c <= '9')
|
||||||
|
return (c-'0');
|
||||||
|
else if (c >= 'a' && c <= 'f')
|
||||||
|
return (c-'a'+10);
|
||||||
|
else if (c >= 'A' && c <= 'F')
|
||||||
|
return (c-'A'+10);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
void fromhexstr(const std::string & from, int bytes, uint8_t *to)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < from.size() && i < bytes; i++)
|
||||||
|
{
|
||||||
|
int x = fromhex(from[2*i]), y = fromhex(from[2*i+1]);
|
||||||
|
if (x < 0 || y < 0)
|
||||||
|
break;
|
||||||
|
to[i] = x*16 + y;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
std::string realpath_str(std::string path, bool nofail)
|
std::string realpath_str(std::string path, bool nofail)
|
||||||
{
|
{
|
||||||
char *p = realpath((char*)path.c_str(), NULL);
|
char *p = realpath((char*)path.c_str(), NULL);
|
||||||
|
|
Loading…
Reference in New Issue