Compare commits

...

38 Commits

Author SHA1 Message Date
Vitaliy Filippov 4efb8bf38a Return block NFS implementation back as an option too
Test / test_minsize_1 (push) Has been cancelled Details
Test / test_move_reappear (push) Has been cancelled Details
Test / test_rm (push) Has been cancelled Details
Test / test_snapshot_chain (push) Has been cancelled Details
Test / test_snapshot_chain_ec (push) Has been cancelled Details
Test / test_snapshot_down (push) Has been cancelled Details
Test / test_snapshot_down_ec (push) Has been cancelled Details
Test / test_splitbrain (push) Has been cancelled Details
Test / test_rebalance_verify (push) Has been cancelled Details
Test / test_rebalance_verify_imm (push) Has been cancelled Details
Test / test_rebalance_verify_ec (push) Has been cancelled Details
Test / test_rebalance_verify_ec_imm (push) Has been cancelled Details
Test / test_switch_primary (push) Has been cancelled Details
Test / test_write (push) Has been cancelled Details
Test / test_write_xor (push) Has been cancelled Details
Test / test_write_no_same (push) Has been cancelled Details
Test / test_heal_pg_size_2 (push) Has been cancelled Details
Test / test_heal_ec (push) Has been cancelled Details
Test / test_heal_csum_32k_dmj (push) Has been cancelled Details
Test / test_heal_csum_32k_dj (push) Has been cancelled Details
Test / test_heal_csum_32k (push) Has been cancelled Details
Test / test_heal_csum_4k_dmj (push) Has been cancelled Details
Test / test_heal_csum_4k_dj (push) Has been cancelled Details
Test / test_heal_csum_4k (push) Has been cancelled Details
Test / test_scrub (push) Has been cancelled Details
Test / test_scrub_zero_osd_2 (push) Has been cancelled Details
Test / test_scrub_xor (push) Has been cancelled Details
Test / test_scrub_pg_size_3 (push) Has been cancelled Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Has been cancelled Details
Test / test_scrub_ec (push) Has been cancelled Details
2024-02-04 14:09:47 +03:00
Vitaliy Filippov 0a63c645e5 Move KV FS header into a separate file 2024-02-04 14:09:47 +03:00
Vitaliy Filippov 7a85486ec3 Implement packing small files into shared inodes 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 3b59422280 Split new NFS proxy implementation into multiple files 2024-02-04 14:09:03 +03:00
Vitaliy Filippov af7f6f7ce7 WIP VitastorFS with metadata storage in VitastorKV 2024-02-04 14:09:03 +03:00
Vitaliy Filippov e18e33944b Fix eviction when random_pos selects the end
Test / test_minsize_1 (push) Has been cancelled Details
Test / test_move_reappear (push) Has been cancelled Details
Test / test_rm (push) Has been cancelled Details
Test / test_snapshot_chain (push) Has been cancelled Details
Test / test_snapshot_chain_ec (push) Has been cancelled Details
Test / test_snapshot_down (push) Has been cancelled Details
Test / test_snapshot_down_ec (push) Has been cancelled Details
Test / test_splitbrain (push) Has been cancelled Details
Test / test_rebalance_verify (push) Has been cancelled Details
Test / test_rebalance_verify_imm (push) Has been cancelled Details
Test / test_rebalance_verify_ec (push) Has been cancelled Details
Test / test_rebalance_verify_ec_imm (push) Has been cancelled Details
Test / test_switch_primary (push) Has been cancelled Details
Test / test_write (push) Has been cancelled Details
Test / test_write_xor (push) Has been cancelled Details
Test / test_write_no_same (push) Has been cancelled Details
Test / test_heal_pg_size_2 (push) Has been cancelled Details
Test / test_heal_ec (push) Has been cancelled Details
Test / test_heal_csum_32k_dmj (push) Has been cancelled Details
Test / test_heal_csum_32k_dj (push) Has been cancelled Details
Test / test_heal_csum_32k (push) Has been cancelled Details
Test / test_heal_csum_4k_dmj (push) Has been cancelled Details
Test / test_heal_csum_4k_dj (push) Has been cancelled Details
Test / test_heal_csum_4k (push) Has been cancelled Details
Test / test_scrub (push) Has been cancelled Details
Test / test_scrub_zero_osd_2 (push) Has been cancelled Details
Test / test_scrub_xor (push) Has been cancelled Details
Test / test_scrub_pg_size_3 (push) Has been cancelled Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Has been cancelled Details
Test / test_scrub_ec (push) Has been cancelled Details
2024-02-04 14:09:03 +03:00
Vitaliy Filippov dce1a557cf Implement min/max list_count to make listings during performance test reasonable 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 1f5c613926 Fix and improve parallel allocation
- Do not try to allocate more DB blocks in an inode block until it's "confirmed" and "locked" by the first write
- Do not recheck for new zero DB blocks on first write into an inode block - a CAS failure means someone else is already writing into it
- Throw new allocation blocks away regardless of whether the known_version is 0 on a CAS failure
2024-02-04 14:09:03 +03:00
Vitaliy Filippov 612972d4e0 Implement key_prefix for K/V stress test 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 6ca1bb61b7 More fixes
- do not overwrite a block with older version if known version is newer
  (read may start before update and end after update)
- invalidated block versions can't be remembered and trusted
- right boundary for split blocks is right_half when diving down, not key_lt
- restart update also when block is "invalidated", not just on version mismatch
- copy callback in listings to avoid closure destruction bugs too
2024-02-04 14:09:03 +03:00
Vitaliy Filippov 19d312805e Add logging and one more assert 2024-02-04 14:09:03 +03:00
Vitaliy Filippov bbb7665e84 Make get_block() wait for updating when unrelated block is found along the path 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 1d0837fcbf Fix a race condition where changed blocks were parsed over existing cached blocks and getting a mix of data 2024-02-04 14:09:03 +03:00
Vitaliy Filippov e4f9d39823 Simplify code by removing an unneeded "optimisation" 2024-02-04 14:09:03 +03:00
Vitaliy Filippov a47ebc4b8e Add kv_log_level, print warnings on level 1, trace ops on level 10 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 9c0e7e8710 Fix duplicate keys in listings on parallel updates -- do not rewind key "iterator position" 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 797a27063d Implement key suffix to avoid collisions of multiple test workers 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 59d9a5706a Do not complain on empty first block 2024-02-04 14:09:03 +03:00
Vitaliy Filippov a872d087ad Add JSON output for stress-tester 2024-02-04 14:09:03 +03:00
Vitaliy Filippov c3f0550c9c Print total stats 2024-02-04 14:09:03 +03:00
Vitaliy Filippov c70dc313ef Do not send more than op_count operations (fix segfault on finish) 2024-02-04 14:09:03 +03:00
Vitaliy Filippov c5950468d5 Add some more resiliency to serialize() 2024-02-04 14:09:03 +03:00
Vitaliy Filippov d562547991 Invalidate blocks being updated too 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 04925057ad Change new block allocation method: make each writer choose multiple empty PG blocks and place blocks in them 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 91d665a2f6 Remove blocks from cache on unsuccessful updates 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 0167ebf5c9 Allow to track multiple updates per block (it should never happen though) 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 2b6394aeac Do not call stop_updating after failed write_new_block and after clear_block (both delete the item) 2024-02-04 14:09:03 +03:00
Vitaliy Filippov d33d0875b7 Track versions of parent blocks and recheck if changed during update 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 4afc5bf299 Fix resume_split condition (key_lt can also be "") 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 3a8fee5a80 Experiment: transform offsets for better sharding 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 170f91ccf2 More post-stress-test fixes
- Prevent _split types of new blocks
- Stop updating new blocks only after the whole update, otherwise pointers
  may become invalid
- Use recheck_none for updates initially
- Use UINT64_MAX as initial block version when postponing ops, otherwise the
  check fails when the block is initially empty. This for example leads to
  writing both leaf items & block pointers (which is incorrect) into the root
  block when starting stress-test with --parallelism 32
- Fix -EINTR comparison
2024-02-04 14:09:03 +03:00
Vitaliy Filippov b9f959b6c8 Print operation statistics 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 1a75e308e2 K/V fixes after stress-test :-)
- track block versions correctly - per inode block (128kb) instead of tree block (4kb)
- prevent multiple parallel CAS writes of the same inode block
- add logging for EILSEQ which means invalid data in the tree
- fix get_block updated flag which was true for blocks already in cache and was leading to infinite loops on "unrelated block" errors
- apply changes to blocks in cache only after successful writes (using "virtual changes")
- do not replace cached block with an older version from disk
- recheck "unrelated blocks" (read/update collisions) until data stops changing
- track tree path correctly - do not treat split block as parent of its right half
- correctly move blocks when finding new empty place on disk
- restart updates from the beginning when one of blocks is changed by a parallel update
- fix delete using SET opcode and setting key to the empty value instead
- prevent changing the same key more than 1 time in parallel
- fix listing verification
- resume continue_updates in update_find (required because it uses continue_update itself)
- add allow_old_cached parameter to get()
2024-02-04 14:09:03 +03:00
Vitaliy Filippov 448bd292d5 Implement K/V DB stress tester 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 7be4771edb Evict blocks based on memory limit & block usage 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 5c613bdd53 Track blocks per level 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 2ef2b1f1a0 Track block level 2024-02-04 14:09:03 +03:00
Vitaliy Filippov 3529683dd1 Experimental B-Tree Vitastor embedded K/V database implementation! 2024-02-04 14:09:03 +03:00
26 changed files with 6790 additions and 425 deletions

View File

@ -181,10 +181,48 @@ target_link_libraries(vitastor-nbd
vitastor_client vitastor_client
) )
# libvitastor_kv.so
add_library(vitastor_kv SHARED
kv_db.cpp
kv_db.h
)
target_link_libraries(vitastor_kv
vitastor_client
)
set_target_properties(vitastor_kv PROPERTIES VERSION ${VERSION} SOVERSION 0)
# vitastor-kv
add_executable(vitastor-kv
kv_cli.cpp
)
target_link_libraries(vitastor-kv
vitastor_kv
)
add_executable(vitastor-kv-stress
kv_stress.cpp
)
target_link_libraries(vitastor-kv-stress
vitastor_kv
)
# vitastor-nfs # vitastor-nfs
add_executable(vitastor-nfs add_executable(vitastor-nfs
nfs_proxy.cpp nfs_proxy.cpp
nfs_conn.cpp nfs_block.cpp
nfs_kv.cpp
nfs_kv_create.cpp
nfs_kv_getattr.cpp
nfs_kv_link.cpp
nfs_kv_lookup.cpp
nfs_kv_read.cpp
nfs_kv_readdir.cpp
nfs_kv_remove.cpp
nfs_kv_rename.cpp
nfs_kv_setattr.cpp
nfs_kv_write.cpp
nfs_fsstat.cpp
nfs_mount.cpp
nfs_portmap.cpp nfs_portmap.cpp
sha256.c sha256.c
nfs/xdr_impl.cpp nfs/xdr_impl.cpp
@ -194,6 +232,7 @@ add_executable(vitastor-nfs
) )
target_link_libraries(vitastor-nfs target_link_libraries(vitastor-nfs
vitastor_client vitastor_client
vitastor_kv
) )
# vitastor-cli # vitastor-cli

View File

@ -6,7 +6,7 @@
#include "cluster_client_impl.h" #include "cluster_client_impl.h"
#include "http_client.h" // json_is_true #include "http_client.h" // json_is_true
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config) cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
{ {
wb = new writeback_cache_t(); wb = new writeback_cache_t();
@ -534,7 +534,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
return; return;
} }
if (op->opcode == OSD_OP_WRITE && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) && if (op->opcode == OSD_OP_WRITE && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
!op->version /* FIXME no CAS writeback */) !op->version /* no CAS writeback */)
{ {
if (wb->writebacks_active >= client_max_writeback_iodepth) if (wb->writebacks_active >= client_max_writeback_iodepth)
{ {
@ -555,7 +555,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
} }
if (op->opcode == OSD_OP_WRITE && !(op->flags & OP_IMMEDIATE_COMMIT)) if (op->opcode == OSD_OP_WRITE && !(op->flags & OP_IMMEDIATE_COMMIT))
{ {
if (!(op->flags & OP_FLUSH_BUFFER)) if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
{ {
wb->copy_write(op, CACHE_WRITTEN); wb->copy_write(op, CACHE_WRITTEN);
} }
@ -1161,7 +1161,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
); );
} }
} }
else else if (log_level > 0)
{ {
fprintf( fprintf(
stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d)\n", stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d)\n",

View File

@ -121,7 +121,7 @@ public:
json11::Json::object cli_config, file_config, etcd_global_config; json11::Json::object cli_config, file_config, etcd_global_config;
json11::Json::object config; json11::Json::object config;
cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config); cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config);
~cluster_client_t(); ~cluster_client_t();
void execute(cluster_op_t *op); void execute(cluster_op_t *op);
void execute_raw(osd_num_t osd_num, osd_op_t *op); void execute_raw(osd_num_t osd_num, osd_op_t *op);

401
src/kv_cli.cpp Normal file
View File

@ -0,0 +1,401 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// Vitastor shared key/value database test CLI
#define _XOPEN_SOURCE
#include <limits.h>
#include <netinet/tcp.h>
#include <sys/epoll.h>
#include <unistd.h>
#include <fcntl.h>
//#include <signal.h>
#include "epoll_manager.h"
#include "str_util.h"
#include "kv_db.h"
const char *exe_name = NULL;
class kv_cli_t
{
public:
kv_dbw_t *db = NULL;
ring_loop_t *ringloop = NULL;
epoll_manager_t *epmgr = NULL;
cluster_client_t *cli = NULL;
bool interactive = false;
int in_progress = 0;
char *cur_cmd = NULL;
int cur_cmd_size = 0, cur_cmd_alloc = 0;
bool finished = false, eof = false;
json11::Json::object cfg;
~kv_cli_t();
static json11::Json::object parse_args(int narg, const char *args[]);
void run(const json11::Json::object & cfg);
void read_cmd();
void next_cmd();
void handle_cmd(const std::string & cmd, std::function<void()> cb);
};
kv_cli_t::~kv_cli_t()
{
if (cur_cmd)
{
free(cur_cmd);
cur_cmd = NULL;
}
cur_cmd_alloc = 0;
if (db)
delete db;
if (cli)
{
cli->flush();
delete cli;
}
if (epmgr)
delete epmgr;
if (ringloop)
delete ringloop;
}
json11::Json::object kv_cli_t::parse_args(int narg, const char *args[])
{
json11::Json::object cfg;
for (int i = 1; i < narg; i++)
{
if (!strcmp(args[i], "-h") || !strcmp(args[i], "--help"))
{
printf(
"Vitastor Key/Value CLI\n"
"(c) Vitaliy Filippov, 2023+ (VNPL-1.1)\n"
"\n"
"USAGE: %s [--etcd_address ADDR] [OTHER OPTIONS]\n",
exe_name
);
exit(0);
}
else if (args[i][0] == '-' && args[i][1] == '-')
{
const char *opt = args[i]+2;
cfg[opt] = !strcmp(opt, "json") || i == narg-1 ? "1" : args[++i];
}
}
return cfg;
}
void kv_cli_t::run(const json11::Json::object & cfg)
{
// Create client
ringloop = new ring_loop_t(512);
epmgr = new epoll_manager_t(ringloop);
cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
db = new kv_dbw_t(cli);
// Load image metadata
while (!cli->is_ready())
{
ringloop->loop();
if (cli->is_ready())
break;
ringloop->wait();
}
// Run
fcntl(0, F_SETFL, fcntl(0, F_GETFL, 0) | O_NONBLOCK);
try
{
epmgr->tfd->set_fd_handler(0, false, [this](int fd, int events)
{
if (events & EPOLLIN)
{
read_cmd();
}
if (events & EPOLLRDHUP)
{
epmgr->tfd->set_fd_handler(0, false, NULL);
finished = true;
}
});
interactive = true;
printf("> ");
}
catch (std::exception & e)
{
// Can't add to epoll, STDIN is probably a file
read_cmd();
}
while (!finished)
{
ringloop->loop();
if (!finished)
ringloop->wait();
}
// Destroy the client
delete db;
db = NULL;
cli->flush();
delete cli;
delete epmgr;
delete ringloop;
cli = NULL;
epmgr = NULL;
ringloop = NULL;
}
void kv_cli_t::read_cmd()
{
if (!cur_cmd_alloc)
{
cur_cmd_alloc = 65536;
cur_cmd = (char*)malloc_or_die(cur_cmd_alloc);
}
while (cur_cmd_size < cur_cmd_alloc)
{
int r = read(0, cur_cmd+cur_cmd_size, cur_cmd_alloc-cur_cmd_size);
if (r < 0 && errno != EAGAIN)
fprintf(stderr, "Error reading from stdin: %s\n", strerror(errno));
if (r > 0)
cur_cmd_size += r;
if (r == 0)
eof = true;
if (r <= 0)
break;
}
next_cmd();
}
void kv_cli_t::next_cmd()
{
if (in_progress > 0)
{
return;
}
int pos = 0;
for (; pos < cur_cmd_size; pos++)
{
if (cur_cmd[pos] == '\n' || cur_cmd[pos] == '\r')
{
auto cmd = trim(std::string(cur_cmd, pos));
pos++;
memmove(cur_cmd, cur_cmd+pos, cur_cmd_size-pos);
cur_cmd_size -= pos;
in_progress++;
handle_cmd(cmd, [this]()
{
in_progress--;
if (interactive)
printf("> ");
next_cmd();
if (!in_progress)
read_cmd();
});
break;
}
}
if (eof && !in_progress)
{
finished = true;
}
}
void kv_cli_t::handle_cmd(const std::string & cmd, std::function<void()> cb)
{
if (cmd == "")
{
cb();
return;
}
auto pos = cmd.find_first_of(" \t");
if (pos != std::string::npos)
{
while (pos < cmd.size()-1 && (cmd[pos+1] == ' ' || cmd[pos+1] == '\t'))
pos++;
}
auto opname = strtolower(pos == std::string::npos ? cmd : cmd.substr(0, pos));
if (opname == "open")
{
uint64_t pool_id = 0;
inode_t inode_id = 0;
uint32_t kv_block_size = 0;
int scanned = sscanf(cmd.c_str() + pos+1, "%lu %lu %u", &pool_id, &inode_id, &kv_block_size);
if (scanned == 2)
{
kv_block_size = 4096;
}
if (scanned < 2 || !pool_id || !inode_id || !kv_block_size || (kv_block_size & (kv_block_size-1)) != 0)
{
fprintf(stderr, "Usage: open <pool_id> <inode_id> [block_size]. Block size must be a power of 2. Default is 4096.\n");
cb();
return;
}
cfg["kv_block_size"] = (uint64_t)kv_block_size;
db->open(INODE_WITH_POOL(pool_id, inode_id), cfg, [=](int res)
{
if (res < 0)
fprintf(stderr, "Error opening index: %s (code %d)\n", strerror(-res), res);
else
printf("Index opened. Current size: %lu bytes\n", db->get_size());
cb();
});
}
else if (opname == "config")
{
auto pos2 = cmd.find_first_of(" \t", pos+1);
if (pos2 == std::string::npos)
{
fprintf(stderr, "Usage: config <property> <value>\n");
cb();
return;
}
auto key = trim(cmd.substr(pos+1, pos2-pos-1));
auto value = parse_size(trim(cmd.substr(pos2+1)));
if (key != "kv_memory_limit" &&
key != "kv_allocate_blocks" &&
key != "kv_evict_max_misses" &&
key != "kv_evict_attempts_per_level" &&
key != "kv_evict_unused_age" &&
key != "kv_log_level")
{
fprintf(
stderr, "Allowed properties: kv_memory_limit, kv_allocate_blocks,"
" kv_evict_max_misses, kv_evict_attempts_per_level, kv_evict_unused_age, kv_log_level\n"
);
}
else
{
cfg[key] = value;
db->set_config(cfg);
}
cb();
}
else if (opname == "get" || opname == "set" || opname == "del")
{
if (opname == "get" || opname == "del")
{
if (pos == std::string::npos)
{
fprintf(stderr, "Usage: %s <key>\n", opname.c_str());
cb();
return;
}
auto key = trim(cmd.substr(pos+1));
if (opname == "get")
{
db->get(key, [this, cb](int res, const std::string & value)
{
if (res < 0)
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
else
{
write(1, value.c_str(), value.size());
write(1, "\n", 1);
}
cb();
});
}
else
{
db->del(key, [this, cb](int res)
{
if (res < 0)
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
else
printf("OK\n");
cb();
});
}
}
else
{
auto pos2 = cmd.find_first_of(" \t", pos+1);
if (pos2 == std::string::npos)
{
fprintf(stderr, "Usage: set <key> <value>\n");
cb();
return;
}
auto key = trim(cmd.substr(pos+1, pos2-pos-1));
auto value = trim(cmd.substr(pos2+1));
db->set(key, value, [this, cb](int res)
{
if (res < 0)
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
else
printf("OK\n");
cb();
});
}
}
else if (opname == "list")
{
std::string start, end;
if (pos != std::string::npos)
{
auto pos2 = cmd.find_first_of(" \t", pos+1);
if (pos2 != std::string::npos)
{
start = trim(cmd.substr(pos+1, pos2-pos-1));
end = trim(cmd.substr(pos2+1));
}
else
{
start = trim(cmd.substr(pos+1));
}
}
void *handle = db->list_start(start);
db->list_next(handle, [=](int res, const std::string & key, const std::string & value)
{
if (res < 0)
{
if (res != -ENOENT)
{
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
}
db->list_close(handle);
cb();
}
else
{
printf("%s = %s\n", key.c_str(), value.c_str());
db->list_next(handle, NULL);
}
});
}
else if (opname == "close")
{
db->close([=]()
{
printf("Index closed\n");
cb();
});
}
else if (opname == "quit" || opname == "q")
{
::close(0);
finished = true;
}
else
{
fprintf(
stderr, "Unknown operation: %s. Supported operations:\n"
"open <pool_id> <inode_id> [block_size]\n"
"config <property> <value>\n"
"get <key>\nset <key> <value>\ndel <key>\nlist [<start> [end]]\n"
"close\nquit\n", opname.c_str()
);
cb();
}
}
int main(int narg, const char *args[])
{
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
exe_name = args[0];
kv_cli_t *p = new kv_cli_t();
p->run(kv_cli_t::parse_args(narg, args));
delete p;
return 0;
}

2037
src/kv_db.cpp Normal file

File diff suppressed because it is too large Load Diff

36
src/kv_db.h Normal file
View File

@ -0,0 +1,36 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// Vitastor shared key/value database
// Parallel optimistic B-Tree O:-)
#pragma once
#include "cluster_client.h"
struct kv_db_t;
struct kv_dbw_t
{
kv_dbw_t(cluster_client_t *cli);
~kv_dbw_t();
void open(inode_t inode_id, json11::Json cfg, std::function<void(int)> cb);
void set_config(json11::Json cfg);
void close(std::function<void()> cb);
uint64_t get_size();
void get(const std::string & key, std::function<void(int res, const std::string & value)> cb,
bool allow_old_cached = false);
void set(const std::string & key, const std::string & value, std::function<void(int res)> cb,
std::function<bool(int res, const std::string & value)> cas_compare = NULL);
void del(const std::string & key, std::function<void(int res)> cb,
std::function<bool(int res, const std::string & value)> cas_compare = NULL);
void* list_start(const std::string & start);
void list_next(void *handle, std::function<void(int res, const std::string & key, const std::string & value)> cb);
void list_close(void *handle);
kv_db_t *db;
};

697
src/kv_stress.cpp Normal file
View File

@ -0,0 +1,697 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// Vitastor shared key/value database stress tester / benchmark
#define _XOPEN_SOURCE
#include <limits.h>
#include <netinet/tcp.h>
#include <sys/epoll.h>
#include <unistd.h>
#include <fcntl.h>
//#include <signal.h>
#include "epoll_manager.h"
#include "str_util.h"
#include "kv_db.h"
const char *exe_name = NULL;
struct kv_test_listing_t
{
uint64_t count = 0, done = 0;
void *handle = NULL;
std::string next_after;
std::set<std::string> inflights;
timespec tv_begin;
bool error = false;
};
struct kv_test_lat_t
{
const char *name = NULL;
uint64_t usec = 0, count = 0;
};
struct kv_test_stat_t
{
kv_test_lat_t get, add, update, del, list;
uint64_t list_keys = 0;
};
class kv_test_t
{
public:
// Config
json11::Json::object kv_cfg;
std::string key_prefix, key_suffix;
uint64_t inode_id = 0;
uint64_t op_count = 1000000;
uint64_t runtime_sec = 0;
uint64_t parallelism = 4;
uint64_t reopen_prob = 1;
uint64_t get_prob = 30000;
uint64_t add_prob = 20000;
uint64_t update_prob = 20000;
uint64_t del_prob = 5000;
uint64_t list_prob = 300;
uint64_t min_key_len = 10;
uint64_t max_key_len = 70;
uint64_t min_value_len = 50;
uint64_t max_value_len = 300;
uint64_t min_list_count = 10;
uint64_t max_list_count = 1000;
uint64_t print_stats_interval = 1;
bool json_output = false;
uint64_t log_level = 1;
bool trace = false;
bool stop_on_error = false;
// FIXME: Multiple clients
kv_test_stat_t stat, prev_stat;
timespec prev_stat_time, start_stat_time;
// State
kv_dbw_t *db = NULL;
ring_loop_t *ringloop = NULL;
epoll_manager_t *epmgr = NULL;
cluster_client_t *cli = NULL;
ring_consumer_t consumer;
bool finished = false;
uint64_t total_prob = 0;
uint64_t ops_sent = 0, ops_done = 0;
int stat_timer_id = -1;
int in_progress = 0;
bool reopening = false;
std::set<kv_test_listing_t*> listings;
std::set<std::string> changing_keys;
std::map<std::string, std::string> values;
~kv_test_t();
static json11::Json::object parse_args(int narg, const char *args[]);
void parse_config(json11::Json cfg);
void run(json11::Json cfg);
void loop();
void print_stats(kv_test_stat_t & prev_stat, timespec & prev_stat_time);
void print_total_stats();
void start_change(const std::string & key);
void stop_change(const std::string & key);
void add_stat(kv_test_lat_t & stat, timespec tv_begin);
};
kv_test_t::~kv_test_t()
{
if (db)
delete db;
if (cli)
{
cli->flush();
delete cli;
}
if (epmgr)
delete epmgr;
if (ringloop)
delete ringloop;
}
json11::Json::object kv_test_t::parse_args(int narg, const char *args[])
{
json11::Json::object cfg;
for (int i = 1; i < narg; i++)
{
if (!strcmp(args[i], "-h") || !strcmp(args[i], "--help"))
{
printf(
"Vitastor Key/Value DB stress tester / benchmark\n"
"(c) Vitaliy Filippov, 2023+ (VNPL-1.1)\n"
"\n"
"USAGE: %s --pool_id POOL_ID --inode_id INODE_ID [OPTIONS]\n"
" --op_count 1000000\n"
" Total operations to run during test. 0 means unlimited\n"
" --key_prefix \"\"\n"
" Prefix for all keys read or written (to avoid collisions)\n"
" --key_suffix \"\"\n"
" Suffix for all keys read or written (to avoid collisions, but scan all DB)\n"
" --runtime 0\n"
" Run for this number of seconds. 0 means unlimited\n"
" --parallelism 4\n"
" Run this number of operations in parallel\n"
" --get_prob 30000\n"
" Fraction of key retrieve operations\n"
" --add_prob 20000\n"
" Fraction of key addition operations\n"
" --update_prob 20000\n"
" Fraction of key update operations\n"
" --del_prob 30000\n"
" Fraction of key delete operations\n"
" --list_prob 300\n"
" Fraction of listing operations\n"
" --min_key_len 10\n"
" Minimum key size in bytes\n"
" --max_key_len 70\n"
" Maximum key size in bytes\n"
" --min_value_len 50\n"
" Minimum value size in bytes\n"
" --max_value_len 300\n"
" Maximum value size in bytes\n"
" --min_list_count 10\n"
" Minimum number of keys read in listing (0 = all keys)\n"
" --max_list_count 1000\n"
" Maximum number of keys read in listing\n"
" --print_stats 1\n"
" Print operation statistics every this number of seconds\n"
" --json\n"
" JSON output\n"
" --stop_on_error 0\n"
" Stop on first execution error, mismatch, lost key or extra key during listing\n"
" --kv_memory_limit 128M\n"
" Maximum memory to use for vitastor-kv index cache\n"
" --kv_allocate_blocks 4\n"
" Number of PG blocks used for new tree block allocation in parallel\n"
" --kv_evict_max_misses 10\n"
" Eviction algorithm parameter: retry eviction from another random spot\n"
" if this number of keys is used currently or was used recently\n"
" --kv_evict_attempts_per_level 3\n"
" Retry eviction at most this number of times per tree level, starting\n"
" with bottom-most levels\n"
" --kv_evict_unused_age 1000\n"
" Evict only keys unused during this number of last operations\n"
" --kv_log_level 1\n"
" Log level. 0 = errors, 1 = warnings, 10 = trace operations\n",
exe_name
);
exit(0);
}
else if (args[i][0] == '-' && args[i][1] == '-')
{
const char *opt = args[i]+2;
cfg[opt] = !strcmp(opt, "json") || i == narg-1 ? "1" : args[++i];
}
}
return cfg;
}
void kv_test_t::parse_config(json11::Json cfg)
{
inode_id = INODE_WITH_POOL(cfg["pool_id"].uint64_value(), cfg["inode_id"].uint64_value());
if (cfg["op_count"].uint64_value() > 0)
op_count = cfg["op_count"].uint64_value();
key_prefix = cfg["key_prefix"].string_value();
key_suffix = cfg["key_suffix"].string_value();
if (cfg["runtime"].uint64_value() > 0)
runtime_sec = cfg["runtime"].uint64_value();
if (cfg["parallelism"].uint64_value() > 0)
parallelism = cfg["parallelism"].uint64_value();
if (!cfg["reopen_prob"].is_null())
reopen_prob = cfg["reopen_prob"].uint64_value();
if (!cfg["get_prob"].is_null())
get_prob = cfg["get_prob"].uint64_value();
if (!cfg["add_prob"].is_null())
add_prob = cfg["add_prob"].uint64_value();
if (!cfg["update_prob"].is_null())
update_prob = cfg["update_prob"].uint64_value();
if (!cfg["del_prob"].is_null())
del_prob = cfg["del_prob"].uint64_value();
if (!cfg["list_prob"].is_null())
list_prob = cfg["list_prob"].uint64_value();
if (!cfg["min_key_len"].is_null())
min_key_len = cfg["min_key_len"].uint64_value();
if (cfg["max_key_len"].uint64_value() > 0)
max_key_len = cfg["max_key_len"].uint64_value();
if (!cfg["min_value_len"].is_null())
min_value_len = cfg["min_value_len"].uint64_value();
if (cfg["max_value_len"].uint64_value() > 0)
max_value_len = cfg["max_value_len"].uint64_value();
if (!cfg["min_list_count"].is_null())
min_list_count = cfg["min_list_count"].uint64_value();
if (!cfg["max_list_count"].is_null())
max_list_count = cfg["max_list_count"].uint64_value();
if (!cfg["print_stats"].is_null())
print_stats_interval = cfg["print_stats"].uint64_value();
if (!cfg["json"].is_null())
json_output = true;
if (!cfg["stop_on_error"].is_null())
stop_on_error = cfg["stop_on_error"].bool_value();
if (!cfg["kv_memory_limit"].is_null())
kv_cfg["kv_memory_limit"] = cfg["kv_memory_limit"];
if (!cfg["kv_allocate_blocks"].is_null())
kv_cfg["kv_allocate_blocks"] = cfg["kv_allocate_blocks"];
if (!cfg["kv_evict_max_misses"].is_null())
kv_cfg["kv_evict_max_misses"] = cfg["kv_evict_max_misses"];
if (!cfg["kv_evict_attempts_per_level"].is_null())
kv_cfg["kv_evict_attempts_per_level"] = cfg["kv_evict_attempts_per_level"];
if (!cfg["kv_evict_unused_age"].is_null())
kv_cfg["kv_evict_unused_age"] = cfg["kv_evict_unused_age"];
if (!cfg["kv_log_level"].is_null())
{
log_level = cfg["kv_log_level"].uint64_value();
trace = log_level >= 10;
kv_cfg["kv_log_level"] = cfg["kv_log_level"];
}
total_prob = reopen_prob+get_prob+add_prob+update_prob+del_prob+list_prob;
stat.get.name = "get";
stat.add.name = "add";
stat.update.name = "update";
stat.del.name = "del";
stat.list.name = "list";
}
void kv_test_t::run(json11::Json cfg)
{
srand48(time(NULL));
parse_config(cfg);
// Create client
ringloop = new ring_loop_t(512);
epmgr = new epoll_manager_t(ringloop);
cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
db = new kv_dbw_t(cli);
// Load image metadata
while (!cli->is_ready())
{
ringloop->loop();
if (cli->is_ready())
break;
ringloop->wait();
}
// Run
reopening = true;
db->open(inode_id, kv_cfg, [this](int res)
{
reopening = false;
if (res < 0)
{
fprintf(stderr, "ERROR: Open index: %d (%s)\n", res, strerror(-res));
exit(1);
}
if (trace)
printf("Index opened\n");
ringloop->wakeup();
});
consumer.loop = [this]() { loop(); };
ringloop->register_consumer(&consumer);
if (print_stats_interval)
stat_timer_id = epmgr->tfd->set_timer(print_stats_interval*1000, true, [this](int) { print_stats(prev_stat, prev_stat_time); });
clock_gettime(CLOCK_REALTIME, &start_stat_time);
prev_stat_time = start_stat_time;
while (!finished)
{
ringloop->loop();
if (!finished)
ringloop->wait();
}
if (stat_timer_id >= 0)
epmgr->tfd->clear_timer(stat_timer_id);
ringloop->unregister_consumer(&consumer);
// Print total stats
print_total_stats();
// Destroy the client
delete db;
db = NULL;
cli->flush();
delete cli;
delete epmgr;
delete ringloop;
cli = NULL;
epmgr = NULL;
ringloop = NULL;
}
static const char *base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789@+/";
std::string random_str(int len)
{
std::string str;
str.resize(len);
for (int i = 0; i < len; i++)
{
str[i] = base64_chars[lrand48() % 64];
}
return str;
}
void kv_test_t::loop()
{
if (reopening)
{
return;
}
if (ops_done >= op_count)
{
finished = true;
}
while (!finished && ops_sent < op_count && in_progress < parallelism)
{
uint64_t dice = (lrand48() % total_prob);
if (dice < reopen_prob)
{
reopening = true;
db->close([this]()
{
if (trace)
printf("Index closed\n");
db->open(inode_id, kv_cfg, [this](int res)
{
reopening = false;
if (res < 0)
{
fprintf(stderr, "ERROR: Reopen index: %d (%s)\n", res, strerror(-res));
finished = true;
return;
}
if (trace)
printf("Index reopened\n");
ringloop->wakeup();
});
});
return;
}
else if (dice < reopen_prob+get_prob)
{
// get existing
auto key = random_str(max_key_len);
auto k_it = values.lower_bound(key);
if (k_it == values.end())
continue;
key = k_it->first;
if (changing_keys.find(key) != changing_keys.end())
continue;
in_progress++;
ops_sent++;
if (trace)
printf("get %s\n", key.c_str());
timespec tv_begin;
clock_gettime(CLOCK_REALTIME, &tv_begin);
db->get(key, [this, key, tv_begin](int res, const std::string & value)
{
add_stat(stat.get, tv_begin);
ops_done++;
in_progress--;
auto it = values.find(key);
if (res != (it == values.end() ? -ENOENT : 0))
{
fprintf(stderr, "ERROR: get %s: %d (%s)\n", key.c_str(), res, strerror(-res));
if (stop_on_error)
exit(1);
}
else if (it != values.end() && value != it->second)
{
fprintf(stderr, "ERROR: get %s: mismatch: %s vs %s\n", key.c_str(), value.c_str(), it->second.c_str());
if (stop_on_error)
exit(1);
}
ringloop->wakeup();
});
}
else if (dice < reopen_prob+get_prob+add_prob+update_prob)
{
bool is_add = false;
std::string key;
if (dice < reopen_prob+get_prob+add_prob)
{
// add
is_add = true;
uint64_t key_len = min_key_len + (max_key_len > min_key_len ? lrand48() % (max_key_len-min_key_len) : 0);
key = key_prefix + random_str(key_len) + key_suffix;
}
else
{
// update
key = random_str(max_key_len);
auto k_it = values.lower_bound(key);
if (k_it == values.end())
continue;
key = k_it->first;
}
if (changing_keys.find(key) != changing_keys.end())
continue;
uint64_t value_len = min_value_len + (max_value_len > min_value_len ? lrand48() % (max_value_len-min_value_len) : 0);
auto value = random_str(value_len);
start_change(key);
ops_sent++;
in_progress++;
if (trace)
printf("set %s = %s\n", key.c_str(), value.c_str());
timespec tv_begin;
clock_gettime(CLOCK_REALTIME, &tv_begin);
db->set(key, value, [this, key, value, tv_begin, is_add](int res)
{
add_stat(is_add ? stat.add : stat.update, tv_begin);
stop_change(key);
ops_done++;
in_progress--;
if (res != 0)
{
fprintf(stderr, "ERROR: set %s = %s: %d (%s)\n", key.c_str(), value.c_str(), res, strerror(-res));
if (stop_on_error)
exit(1);
}
else
{
values[key] = value;
}
ringloop->wakeup();
}, NULL);
}
else if (dice < reopen_prob+get_prob+add_prob+update_prob+del_prob)
{
// delete
auto key = random_str(max_key_len);
auto k_it = values.lower_bound(key);
if (k_it == values.end())
continue;
key = k_it->first;
if (changing_keys.find(key) != changing_keys.end())
continue;
start_change(key);
ops_sent++;
in_progress++;
if (trace)
printf("del %s\n", key.c_str());
timespec tv_begin;
clock_gettime(CLOCK_REALTIME, &tv_begin);
db->del(key, [this, key, tv_begin](int res)
{
add_stat(stat.del, tv_begin);
stop_change(key);
ops_done++;
in_progress--;
if (res != 0)
{
fprintf(stderr, "ERROR: del %s: %d (%s)\n", key.c_str(), res, strerror(-res));
if (stop_on_error)
exit(1);
}
else
{
values.erase(key);
}
ringloop->wakeup();
}, NULL);
}
else if (dice < reopen_prob+get_prob+add_prob+update_prob+del_prob+list_prob)
{
// list
ops_sent++;
in_progress++;
auto key = random_str(max_key_len);
auto lst = new kv_test_listing_t;
auto k_it = values.lower_bound(key);
lst->count = min_list_count + (max_list_count > min_list_count ? lrand48() % (max_list_count-min_list_count) : 0);
lst->handle = db->list_start(k_it == values.begin() ? key_prefix : key);
lst->next_after = k_it == values.begin() ? key_prefix : key;
lst->inflights = changing_keys;
listings.insert(lst);
if (trace)
printf("list from %s\n", key.c_str());
clock_gettime(CLOCK_REALTIME, &lst->tv_begin);
db->list_next(lst->handle, [this, lst](int res, const std::string & key, const std::string & value)
{
if (log_level >= 11)
printf("list: %s = %s\n", key.c_str(), value.c_str());
if (res >= 0 && key_prefix.size() && (key.size() < key_prefix.size() ||
key.substr(0, key_prefix.size()) != key_prefix))
{
// stop at this key
res = -ENOENT;
}
if (res < 0 || (lst->count > 0 && lst->done >= lst->count))
{
add_stat(stat.list, lst->tv_begin);
if (res == 0)
{
// ok (done >= count)
}
else if (res != -ENOENT)
{
fprintf(stderr, "ERROR: list: %d (%s)\n", res, strerror(-res));
lst->error = true;
}
else
{
auto k_it = lst->next_after == "" ? values.begin() : values.upper_bound(lst->next_after);
while (k_it != values.end())
{
while (k_it != values.end() && lst->inflights.find(k_it->first) != lst->inflights.end())
k_it++;
if (k_it != values.end())
{
fprintf(stderr, "ERROR: list: missing key %s\n", (k_it++)->first.c_str());
lst->error = true;
}
}
}
if (lst->error && stop_on_error)
exit(1);
ops_done++;
in_progress--;
db->list_close(lst->handle);
delete lst;
listings.erase(lst);
ringloop->wakeup();
}
else
{
stat.list_keys++;
// Do not check modified keys in listing
// Listing may return their old or new state
if ((!key_suffix.size() || key.size() >= key_suffix.size() &&
key.substr(key.size()-key_suffix.size()) == key_suffix) &&
lst->inflights.find(key) == lst->inflights.end())
{
lst->done++;
auto k_it = lst->next_after == "" ? values.begin() : values.upper_bound(lst->next_after);
while (true)
{
while (k_it != values.end() && lst->inflights.find(k_it->first) != lst->inflights.end())
{
k_it++;
}
if (k_it == values.end() || k_it->first > key)
{
fprintf(stderr, "ERROR: list: extra key %s\n", key.c_str());
lst->error = true;
break;
}
else if (k_it->first < key)
{
fprintf(stderr, "ERROR: list: missing key %s\n", k_it->first.c_str());
lst->error = true;
lst->next_after = k_it->first;
k_it++;
}
else
{
if (k_it->second != value)
{
fprintf(stderr, "ERROR: list: mismatch: %s = %s but should be %s\n",
key.c_str(), value.c_str(), k_it->second.c_str());
lst->error = true;
}
lst->next_after = k_it->first;
break;
}
}
}
db->list_next(lst->handle, NULL);
}
});
}
}
}
void kv_test_t::add_stat(kv_test_lat_t & stat, timespec tv_begin)
{
timespec tv_end;
clock_gettime(CLOCK_REALTIME, &tv_end);
int64_t usec = (tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
(tv_end.tv_nsec - tv_begin.tv_nsec)/1000;
if (usec > 0)
{
stat.usec += usec;
stat.count++;
}
}
void kv_test_t::print_stats(kv_test_stat_t & prev_stat, timespec & prev_stat_time)
{
timespec cur_stat_time;
clock_gettime(CLOCK_REALTIME, &cur_stat_time);
int64_t usec = (cur_stat_time.tv_sec - prev_stat_time.tv_sec)*1000000 +
(cur_stat_time.tv_nsec - prev_stat_time.tv_nsec)/1000;
if (usec > 0)
{
kv_test_lat_t *lats[] = { &stat.get, &stat.add, &stat.update, &stat.del, &stat.list };
kv_test_lat_t *prev[] = { &prev_stat.get, &prev_stat.add, &prev_stat.update, &prev_stat.del, &prev_stat.list };
if (!json_output)
{
char buf[128] = { 0 };
for (int i = 0; i < sizeof(lats)/sizeof(lats[0]); i++)
{
snprintf(buf, sizeof(buf)-1, "%.1f %s/s (%lu us)", (lats[i]->count-prev[i]->count)*1000000.0/usec,
lats[i]->name, (lats[i]->usec-prev[i]->usec)/(lats[i]->count-prev[i]->count > 0 ? lats[i]->count-prev[i]->count : 1));
int k;
for (k = strlen(buf); k < strlen(lats[i]->name)+21; k++)
buf[k] = ' ';
buf[k] = 0;
printf("%s", buf);
}
printf("\n");
}
else
{
int64_t runtime = (cur_stat_time.tv_sec - start_stat_time.tv_sec)*1000000 +
(cur_stat_time.tv_nsec - start_stat_time.tv_nsec)/1000;
printf("{\"runtime\":%.1f", (double)runtime/1000000.0);
for (int i = 0; i < sizeof(lats)/sizeof(lats[0]); i++)
{
if (lats[i]->count > prev[i]->count)
{
printf(
",\"%s\":{\"avg\":{\"iops\":%.1f,\"usec\":%lu},\"total\":{\"count\":%lu,\"usec\":%lu}}",
lats[i]->name, (lats[i]->count-prev[i]->count)*1000000.0/usec,
(lats[i]->usec-prev[i]->usec)/(lats[i]->count-prev[i]->count),
lats[i]->count, lats[i]->usec
);
}
}
printf("}\n");
}
}
prev_stat = stat;
prev_stat_time = cur_stat_time;
}
void kv_test_t::print_total_stats()
{
if (!json_output)
printf("Total:\n");
kv_test_stat_t start_stats;
timespec start_stat_time = this->start_stat_time;
print_stats(start_stats, start_stat_time);
}
void kv_test_t::start_change(const std::string & key)
{
changing_keys.insert(key);
for (auto lst: listings)
{
lst->inflights.insert(key);
}
}
void kv_test_t::stop_change(const std::string & key)
{
changing_keys.erase(key);
}
int main(int narg, const char *args[])
{
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
exe_name = args[0];
kv_test_t *p = new kv_test_t();
p->run(kv_test_t::parse_args(narg, args));
delete p;
return 0;
}

View File

@ -1,23 +1,18 @@
// Copyright (c) Vitaliy Filippov, 2019+ // Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details) // License: VNPL-1.1 (see README.md for details)
// //
// NFS connection handler for NFS proxy // NFS proxy over Vitastor block images
#include <sys/time.h> #include <sys/time.h>
#include "str_util.h" #include "str_util.h"
#include "nfs_proxy.h" #include "nfs_proxy.h"
#include "nfs_common.h"
#include "nfs_block.h"
#include "nfs/nfs.h" #include "nfs/nfs.h"
#include "cli.h" #include "cli.h"
#define TRUE 1
#define FALSE 0
#define MAX_REQUEST_SIZE 128*1024*1024
static unsigned len_pad4(unsigned len) static unsigned len_pad4(unsigned len)
{ {
return len + (len&3 ? 4-(len&3) : 0); return len + (len&3 ? 4-(len&3) : 0);
@ -28,10 +23,10 @@ static std::string get_inode_name(nfs_client_t *self, diropargs3 & what)
// Get name // Get name
std::string dirhash = what.dir; std::string dirhash = what.dir;
std::string dir; std::string dir;
if (dirhash != "roothandle") if (dirhash != NFS_ROOT_HANDLE)
{ {
auto dir_it = self->parent->dir_by_hash.find(dirhash); auto dir_it = self->parent->blockfs->dir_by_hash.find(dirhash);
if (dir_it != self->parent->dir_by_hash.end()) if (dir_it != self->parent->blockfs->dir_by_hash.end())
dir = dir_it->second; dir = dir_it->second;
else else
return ""; return "";
@ -42,24 +37,9 @@ static std::string get_inode_name(nfs_client_t *self, diropargs3 & what)
: self->parent->name_prefix+name); : self->parent->name_prefix+name);
} }
static nfsstat3 vitastor_nfs_map_err(int err)
{
return (err == EINVAL ? NFS3ERR_INVAL
: (err == ENOENT ? NFS3ERR_NOENT
: (err == ENOSPC ? NFS3ERR_NOSPC
: (err == EEXIST ? NFS3ERR_EXIST
: (err == EIO ? NFS3ERR_IO : (err ? NFS3ERR_IO : NFS3_OK))))));
}
static int nfs3_null_proc(void *opaque, rpc_op_t *rop)
{
rpc_queue_reply(rop);
return 0;
}
static fattr3 get_dir_attributes(nfs_client_t *self, std::string dir) static fattr3 get_dir_attributes(nfs_client_t *self, std::string dir)
{ {
auto & dinf = self->parent->dir_info.at(dir); auto & dinf = self->parent->blockfs->dir_info.at(dir);
return (fattr3){ return (fattr3){
.type = NF3DIR, .type = NF3DIR,
.mode = 0755, .mode = 0755,
@ -108,7 +88,7 @@ static fattr3 get_file_attributes(nfs_client_t *self, inode_t inode_num)
}; };
} }
static int nfs3_getattr_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
GETATTR3args *args = (GETATTR3args*)rop->request; GETATTR3args *args = (GETATTR3args*)rop->request;
@ -116,12 +96,12 @@ static int nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
bool is_dir = false; bool is_dir = false;
std::string dirhash = args->object; std::string dirhash = args->object;
std::string dir; std::string dir;
if (args->object == "roothandle") if (args->object == NFS_ROOT_HANDLE)
is_dir = true; is_dir = true;
else else
{ {
auto dir_it = self->parent->dir_by_hash.find(dirhash); auto dir_it = self->parent->blockfs->dir_by_hash.find(dirhash);
if (dir_it != self->parent->dir_by_hash.end()) if (dir_it != self->parent->blockfs->dir_by_hash.end())
{ {
is_dir = true; is_dir = true;
dir = dir_it->second; dir = dir_it->second;
@ -140,8 +120,8 @@ static int nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
else else
{ {
uint64_t inode_num = 0; uint64_t inode_num = 0;
auto inode_num_it = self->parent->inode_by_hash.find(dirhash); auto inode_num_it = self->parent->blockfs->inode_by_hash.find(dirhash);
if (inode_num_it != self->parent->inode_by_hash.end()) if (inode_num_it != self->parent->blockfs->inode_by_hash.end())
inode_num = inode_num_it->second; inode_num = inode_num_it->second;
auto inode_it = self->parent->cli->st_cli.inode_config.find(inode_num); auto inode_it = self->parent->cli->st_cli.inode_config.find(inode_num);
if (inode_num && inode_it != self->parent->cli->st_cli.inode_config.end()) if (inode_num && inode_it != self->parent->cli->st_cli.inode_config.end())
@ -179,16 +159,16 @@ static int nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_setattr_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_setattr_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
SETATTR3args *args = (SETATTR3args*)rop->request; SETATTR3args *args = (SETATTR3args*)rop->request;
SETATTR3res *reply = (SETATTR3res*)rop->reply; SETATTR3res *reply = (SETATTR3res*)rop->reply;
std::string handle = args->object; std::string handle = args->object;
auto ino_it = self->parent->inode_by_hash.find(handle); auto ino_it = self->parent->blockfs->inode_by_hash.find(handle);
if (ino_it == self->parent->inode_by_hash.end()) if (ino_it == self->parent->blockfs->inode_by_hash.end())
{ {
if (handle == "roothandle" || self->parent->dir_by_hash.find(handle) != self->parent->dir_by_hash.end()) if (handle == NFS_ROOT_HANDLE || self->parent->blockfs->dir_by_hash.find(handle) != self->parent->blockfs->dir_by_hash.end())
{ {
if (args->new_attributes.size.set_it) if (args->new_attributes.size.set_it)
{ {
@ -228,7 +208,7 @@ static int nfs3_setattr_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_lookup_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
LOOKUP3args *args = (LOOKUP3args*)rop->request; LOOKUP3args *args = (LOOKUP3args*)rop->request;
@ -255,8 +235,8 @@ static int nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
} }
auto dir_it = self->parent->dir_info.find(full_name); auto dir_it = self->parent->blockfs->dir_info.find(full_name);
if (dir_it != self->parent->dir_info.end()) if (dir_it != self->parent->blockfs->dir_info.end())
{ {
*reply = (LOOKUP3res){ *reply = (LOOKUP3res){
.status = NFS3_OK, .status = NFS3_OK,
@ -277,7 +257,7 @@ static int nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_access_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_access_proc(void *opaque, rpc_op_t *rop)
{ {
//nfs_client_t *self = (nfs_client_t*)opaque; //nfs_client_t *self = (nfs_client_t*)opaque;
ACCESS3args *args = (ACCESS3args*)rop->request; ACCESS3args *args = (ACCESS3args*)rop->request;
@ -292,7 +272,7 @@ static int nfs3_access_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_readlink_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_readlink_proc(void *opaque, rpc_op_t *rop)
{ {
//nfs_client_t *self = (nfs_client_t*)opaque; //nfs_client_t *self = (nfs_client_t*)opaque;
//READLINK3args *args = (READLINK3args*)rop->request; //READLINK3args *args = (READLINK3args*)rop->request;
@ -303,14 +283,14 @@ static int nfs3_readlink_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_read_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
READ3args *args = (READ3args*)rop->request; READ3args *args = (READ3args*)rop->request;
READ3res *reply = (READ3res*)rop->reply; READ3res *reply = (READ3res*)rop->reply;
std::string handle = args->file; std::string handle = args->file;
auto ino_it = self->parent->inode_by_hash.find(handle); auto ino_it = self->parent->blockfs->inode_by_hash.find(handle);
if (ino_it == self->parent->inode_by_hash.end()) if (ino_it == self->parent->blockfs->inode_by_hash.end())
{ {
*reply = (READ3res){ .status = NFS3ERR_NOENT }; *reply = (READ3res){ .status = NFS3ERR_NOENT };
rpc_queue_reply(rop); rpc_queue_reply(rop);
@ -367,14 +347,14 @@ static int nfs3_read_proc(void *opaque, rpc_op_t *rop)
static void nfs_resize_write(nfs_client_t *self, rpc_op_t *rop, uint64_t inode, uint64_t new_size, uint64_t offset, uint64_t count, void *buf); static void nfs_resize_write(nfs_client_t *self, rpc_op_t *rop, uint64_t inode, uint64_t new_size, uint64_t offset, uint64_t count, void *buf);
static int nfs3_write_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_write_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
WRITE3args *args = (WRITE3args*)rop->request; WRITE3args *args = (WRITE3args*)rop->request;
WRITE3res *reply = (WRITE3res*)rop->reply; WRITE3res *reply = (WRITE3res*)rop->reply;
std::string handle = args->file; std::string handle = args->file;
auto ino_it = self->parent->inode_by_hash.find(handle); auto ino_it = self->parent->blockfs->inode_by_hash.find(handle);
if (ino_it == self->parent->inode_by_hash.end()) if (ino_it == self->parent->blockfs->inode_by_hash.end())
{ {
*reply = (WRITE3res){ .status = NFS3ERR_NOENT }; *reply = (WRITE3res){ .status = NFS3ERR_NOENT };
rpc_queue_reply(rop); rpc_queue_reply(rop);
@ -480,8 +460,8 @@ static void complete_extend_write(nfs_client_t *self, rpc_op_t *rop, inode_t ino
static void complete_extend_inode(nfs_client_t *self, uint64_t inode, uint64_t new_size, int err) static void complete_extend_inode(nfs_client_t *self, uint64_t inode, uint64_t new_size, int err)
{ {
auto ext_it = self->extend_writes.lower_bound((extend_size_t){ .inode = inode, .new_size = 0 }); auto ext_it = self->parent->blockfs->extend_writes.lower_bound((extend_size_t){ .inode = inode, .new_size = 0 });
while (ext_it != self->extend_writes.end() && while (ext_it != self->parent->blockfs->extend_writes.end() &&
ext_it->first.inode == inode && ext_it->first.inode == inode &&
ext_it->first.new_size <= new_size) ext_it->first.new_size <= new_size)
{ {
@ -490,7 +470,7 @@ static void complete_extend_inode(nfs_client_t *self, uint64_t inode, uint64_t n
{ {
complete_extend_write(self, ext_it->second.rop, inode, ext_it->second.write_res < 0 complete_extend_write(self, ext_it->second.rop, inode, ext_it->second.write_res < 0
? ext_it->second.write_res : ext_it->second.resize_res); ? ext_it->second.write_res : ext_it->second.resize_res);
self->extend_writes.erase(ext_it++); self->parent->blockfs->extend_writes.erase(ext_it++);
} }
else else
ext_it++; ext_it++;
@ -500,7 +480,7 @@ static void complete_extend_inode(nfs_client_t *self, uint64_t inode, uint64_t n
static void extend_inode(nfs_client_t *self, uint64_t inode, uint64_t new_size) static void extend_inode(nfs_client_t *self, uint64_t inode, uint64_t new_size)
{ {
// Send an extend request // Send an extend request
auto & ext = self->extends[inode]; auto & ext = self->parent->blockfs->extends[inode];
ext.cur_extend = new_size; ext.cur_extend = new_size;
auto inode_it = self->parent->cli->st_cli.inode_config.find(inode); auto inode_it = self->parent->cli->st_cli.inode_config.find(inode);
if (inode_it != self->parent->cli->st_cli.inode_config.end() && if (inode_it != self->parent->cli->st_cli.inode_config.end() &&
@ -514,7 +494,7 @@ static void extend_inode(nfs_client_t *self, uint64_t inode, uint64_t new_size)
{ "force_size", true }, { "force_size", true },
}), [=](const cli_result_t & r) }), [=](const cli_result_t & r)
{ {
auto & ext = self->extends[inode]; auto & ext = self->parent->blockfs->extends[inode];
if (r.err) if (r.err)
{ {
fprintf(stderr, "Error extending inode %lu to %lu bytes: %s\n", inode, new_size, r.text.c_str()); fprintf(stderr, "Error extending inode %lu to %lu bytes: %s\n", inode, new_size, r.text.c_str());
@ -548,7 +528,7 @@ static void nfs_do_write(nfs_client_t *self, std::multimap<extend_size_t, extend
{ {
auto inode = op->inode; auto inode = op->inode;
int write_res = op->retval < 0 ? op->retval : (op->retval != op->len ? -ERANGE : 0); int write_res = op->retval < 0 ? op->retval : (op->retval != op->len ? -ERANGE : 0);
if (ewr_it == self->extend_writes.end()) if (ewr_it == self->parent->blockfs->extend_writes.end())
{ {
complete_extend_write(self, rop, inode, write_res); complete_extend_write(self, rop, inode, write_res);
} }
@ -558,7 +538,7 @@ static void nfs_do_write(nfs_client_t *self, std::multimap<extend_size_t, extend
if (ewr_it->second.resize_res <= 0) if (ewr_it->second.resize_res <= 0)
{ {
complete_extend_write(self, rop, inode, write_res < 0 ? write_res : ewr_it->second.resize_res); complete_extend_write(self, rop, inode, write_res < 0 ? write_res : ewr_it->second.resize_res);
self->extend_writes.erase(ewr_it); self->parent->blockfs->extend_writes.erase(ewr_it);
} }
} }
}; };
@ -572,7 +552,7 @@ static void nfs_resize_write(nfs_client_t *self, rpc_op_t *rop, uint64_t inode,
if (inode_it != self->parent->cli->st_cli.inode_config.end() && if (inode_it != self->parent->cli->st_cli.inode_config.end() &&
inode_it->second.size < new_size) inode_it->second.size < new_size)
{ {
auto ewr_it = self->extend_writes.emplace((extend_size_t){ auto ewr_it = self->parent->blockfs->extend_writes.emplace((extend_size_t){
.inode = inode, .inode = inode,
.new_size = new_size, .new_size = new_size,
}, (extend_write_t){ }, (extend_write_t){
@ -580,7 +560,7 @@ static void nfs_resize_write(nfs_client_t *self, rpc_op_t *rop, uint64_t inode,
.resize_res = 1, .resize_res = 1,
.write_res = 1, .write_res = 1,
}); });
auto & ext = self->extends[inode]; auto & ext = self->parent->blockfs->extends[inode];
if (ext.cur_extend > 0) if (ext.cur_extend > 0)
{ {
// Already resizing, just wait // Already resizing, just wait
@ -595,11 +575,11 @@ static void nfs_resize_write(nfs_client_t *self, rpc_op_t *rop, uint64_t inode,
} }
else else
{ {
nfs_do_write(self, self->extend_writes.end(), rop, inode, offset, count, buf); nfs_do_write(self, self->parent->blockfs->extend_writes.end(), rop, inode, offset, count, buf);
} }
} }
static int nfs3_create_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_create_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
CREATE3args *args = (CREATE3args*)rop->request; CREATE3args *args = (CREATE3args*)rop->request;
@ -650,7 +630,7 @@ static int nfs3_create_proc(void *opaque, rpc_op_t *rop)
return 1; return 1;
} }
static int nfs3_mkdir_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_mkdir_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
MKDIR3args *args = (MKDIR3args*)rop->request; MKDIR3args *args = (MKDIR3args*)rop->request;
@ -669,19 +649,19 @@ static int nfs3_mkdir_proc(void *opaque, rpc_op_t *rop)
rpc_queue_reply(rop); rpc_queue_reply(rop);
return 0; return 0;
} }
auto dir_id_it = self->parent->dir_info.find(full_name); auto dir_id_it = self->parent->blockfs->dir_info.find(full_name);
if (dir_id_it != self->parent->dir_info.end()) if (dir_id_it != self->parent->blockfs->dir_info.end())
{ {
*reply = (MKDIR3res){ .status = NFS3ERR_EXIST }; *reply = (MKDIR3res){ .status = NFS3ERR_EXIST };
rpc_queue_reply(rop); rpc_queue_reply(rop);
return 0; return 0;
} }
// FIXME: Persist empty directories in some etcd keys, like /vitastor/dir/... // FIXME: Persist empty directories in some etcd keys, like /vitastor/dir/...
self->parent->dir_info[full_name] = (nfs_dir_t){ self->parent->blockfs->dir_info[full_name] = (nfs_dir_t){
.id = self->parent->next_dir_id++, .id = self->parent->blockfs->next_dir_id++,
.mod_rev = 0, .mod_rev = 0,
}; };
self->parent->dir_by_hash["S"+base64_encode(sha256(full_name))] = full_name; self->parent->blockfs->dir_by_hash["S"+base64_encode(sha256(full_name))] = full_name;
*reply = (MKDIR3res){ *reply = (MKDIR3res){
.status = NFS3_OK, .status = NFS3_OK,
.resok = (MKDIR3resok){ .resok = (MKDIR3resok){
@ -700,7 +680,7 @@ static int nfs3_mkdir_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_symlink_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_symlink_proc(void *opaque, rpc_op_t *rop)
{ {
// nfs_client_t *self = (nfs_client_t*)opaque; // nfs_client_t *self = (nfs_client_t*)opaque;
// SYMLINK3args *args = (SYMLINK3args*)rop->request; // SYMLINK3args *args = (SYMLINK3args*)rop->request;
@ -711,7 +691,7 @@ static int nfs3_symlink_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_mknod_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_mknod_proc(void *opaque, rpc_op_t *rop)
{ {
// nfs_client_t *self = (nfs_client_t*)opaque; // nfs_client_t *self = (nfs_client_t*)opaque;
// MKNOD3args *args = (MKNOD3args*)rop->request; // MKNOD3args *args = (MKNOD3args*)rop->request;
@ -722,7 +702,7 @@ static int nfs3_mknod_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
static int nfs3_remove_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_remove_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
REMOVE3res *reply = (REMOVE3res*)rop->reply; REMOVE3res *reply = (REMOVE3res*)rop->reply;
@ -752,7 +732,7 @@ static int nfs3_remove_proc(void *opaque, rpc_op_t *rop)
return 1; return 1;
} }
static int nfs3_rmdir_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_rmdir_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
RMDIR3args *args = (RMDIR3args*)rop->request; RMDIR3args *args = (RMDIR3args*)rop->request;
@ -764,8 +744,8 @@ static int nfs3_rmdir_proc(void *opaque, rpc_op_t *rop)
rpc_queue_reply(rop); rpc_queue_reply(rop);
return 0; return 0;
} }
auto dir_it = self->parent->dir_info.find(full_name); auto dir_it = self->parent->blockfs->dir_info.find(full_name);
if (dir_it == self->parent->dir_info.end()) if (dir_it == self->parent->blockfs->dir_info.end())
{ {
*reply = (RMDIR3res){ .status = NFS3ERR_NOENT }; *reply = (RMDIR3res){ .status = NFS3ERR_NOENT };
rpc_queue_reply(rop); rpc_queue_reply(rop);
@ -781,8 +761,8 @@ static int nfs3_rmdir_proc(void *opaque, rpc_op_t *rop)
return 0; return 0;
} }
} }
self->parent->dir_by_hash.erase("S"+base64_encode(sha256(full_name))); self->parent->blockfs->dir_by_hash.erase("S"+base64_encode(sha256(full_name)));
self->parent->dir_info.erase(dir_it); self->parent->blockfs->dir_info.erase(dir_it);
*reply = (RMDIR3res){ .status = NFS3_OK }; *reply = (RMDIR3res){ .status = NFS3_OK };
rpc_queue_reply(rop); rpc_queue_reply(rop);
return 0; return 0;
@ -811,12 +791,12 @@ static int continue_dir_rename(nfs_dir_rename_state *rename_st)
if (!rename_st->items.size()) if (!rename_st->items.size())
{ {
// old dir // old dir
auto old_info = self->parent->dir_info.at(rename_st->old_name); auto old_info = self->parent->blockfs->dir_info.at(rename_st->old_name);
self->parent->dir_info.erase(rename_st->old_name); self->parent->blockfs->dir_info.erase(rename_st->old_name);
self->parent->dir_by_hash.erase("S"+base64_encode(sha256(rename_st->old_name))); self->parent->blockfs->dir_by_hash.erase("S"+base64_encode(sha256(rename_st->old_name)));
// new dir // new dir
self->parent->dir_info[rename_st->new_name] = old_info; self->parent->blockfs->dir_info[rename_st->new_name] = old_info;
self->parent->dir_by_hash["S"+base64_encode(sha256(rename_st->new_name))] = rename_st->new_name; self->parent->blockfs->dir_by_hash["S"+base64_encode(sha256(rename_st->new_name))] = rename_st->new_name;
RENAME3res *reply = (RENAME3res*)rename_st->rop->reply; RENAME3res *reply = (RENAME3res*)rename_st->rop->reply;
*reply = (RENAME3res){ *reply = (RENAME3res){
.status = NFS3_OK, .status = NFS3_OK,
@ -853,7 +833,7 @@ static int continue_dir_rename(nfs_dir_rename_state *rename_st)
static void nfs_do_rename(nfs_client_t *self, rpc_op_t *rop, std::string old_name, std::string new_name); static void nfs_do_rename(nfs_client_t *self, rpc_op_t *rop, std::string old_name, std::string new_name);
static int nfs3_rename_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_rename_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
RENAME3args *args = (RENAME3args*)rop->request; RENAME3args *args = (RENAME3args*)rop->request;
@ -866,8 +846,8 @@ static int nfs3_rename_proc(void *opaque, rpc_op_t *rop)
rpc_queue_reply(rop); rpc_queue_reply(rop);
return 0; return 0;
} }
bool old_is_dir = self->parent->dir_info.find(old_name) != self->parent->dir_info.end(); bool old_is_dir = self->parent->blockfs->dir_info.find(old_name) != self->parent->blockfs->dir_info.end();
bool new_is_dir = self->parent->dir_info.find(new_name) != self->parent->dir_info.end(); bool new_is_dir = self->parent->blockfs->dir_info.find(new_name) != self->parent->blockfs->dir_info.end();
bool old_is_file = false, new_is_file = false; bool old_is_file = false, new_is_file = false;
for (auto & ic: self->parent->cli->st_cli.inode_config) for (auto & ic: self->parent->cli->st_cli.inode_config)
{ {
@ -948,7 +928,7 @@ static void nfs_do_rename(nfs_client_t *self, rpc_op_t *rop, std::string old_nam
}); });
} }
static int nfs3_link_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_link_proc(void *opaque, rpc_op_t *rop)
{ {
//nfs_client_t *self = (nfs_client_t*)opaque; //nfs_client_t *self = (nfs_client_t*)opaque;
//LINK3args *args = (LINK3args*)rop->request; //LINK3args *args = (LINK3args*)rop->request;
@ -962,7 +942,7 @@ static int nfs3_link_proc(void *opaque, rpc_op_t *rop)
static void fill_dir_entry(nfs_client_t *self, rpc_op_t *rop, static void fill_dir_entry(nfs_client_t *self, rpc_op_t *rop,
std::map<std::string, nfs_dir_t>::iterator dir_id_it, struct entryplus3 *entry, bool is_plus) std::map<std::string, nfs_dir_t>::iterator dir_id_it, struct entryplus3 *entry, bool is_plus)
{ {
if (dir_id_it == self->parent->dir_info.end()) if (dir_id_it == self->parent->blockfs->dir_info.end())
{ {
return; return;
} }
@ -980,7 +960,7 @@ static void fill_dir_entry(nfs_client_t *self, rpc_op_t *rop,
} }
} }
static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus) static void block_nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; nfs_client_t *self = (nfs_client_t*)opaque;
READDIRPLUS3args plus_args; READDIRPLUS3args plus_args;
@ -999,10 +979,10 @@ static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
} }
std::string dirhash = args->dir; std::string dirhash = args->dir;
std::string dir; std::string dir;
if (dirhash != "roothandle") if (dirhash != NFS_ROOT_HANDLE)
{ {
auto dir_it = self->parent->dir_by_hash.find(dirhash); auto dir_it = self->parent->blockfs->dir_by_hash.find(dirhash);
if (dir_it != self->parent->dir_by_hash.end()) if (dir_it != self->parent->blockfs->dir_by_hash.end())
dir = dir_it->second; dir = dir_it->second;
} }
std::string prefix = dir.size() ? dir+"/" : self->parent->name_prefix; std::string prefix = dir.size() ? dir+"/" : self->parent->name_prefix;
@ -1043,12 +1023,12 @@ static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
} }
else else
{ {
// skip directories, they will be added from dir_info // skip directories, they will be added from blockfs->dir_info
} }
} }
// Add directories from dir_info // Add directories from blockfs->dir_info
for (auto dir_id_it = self->parent->dir_info.lower_bound(prefix); for (auto dir_id_it = self->parent->blockfs->dir_info.lower_bound(prefix);
dir_id_it != self->parent->dir_info.end(); dir_id_it++) dir_id_it != self->parent->blockfs->dir_info.end(); dir_id_it++)
{ {
if (prefix != "" && dir_id_it->first.substr(0, prefix.size()) != prefix) if (prefix != "" && dir_id_it->first.substr(0, prefix.size()) != prefix)
break; break;
@ -1061,12 +1041,12 @@ static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
} }
// Add . and .. // Add . and ..
{ {
auto dir_id_it = self->parent->dir_info.find(dir); auto dir_id_it = self->parent->blockfs->dir_info.find(dir);
fill_dir_entry(self, rop, dir_id_it, &entries["."], is_plus); fill_dir_entry(self, rop, dir_id_it, &entries["."], is_plus);
auto sl = dir.rfind("/"); auto sl = dir.rfind("/");
if (sl != std::string::npos) if (sl != std::string::npos)
{ {
auto dir_id_it = self->parent->dir_info.find(dir.substr(0, sl)); auto dir_id_it = self->parent->blockfs->dir_info.find(dir.substr(0, sl));
fill_dir_entry(self, rop, dir_id_it, &entries[".."], is_plus); fill_dir_entry(self, rop, dir_id_it, &entries[".."], is_plus);
} }
} }
@ -1147,7 +1127,7 @@ static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
{ {
READDIRPLUS3res *reply = (READDIRPLUS3res*)rop->reply; READDIRPLUS3res *reply = (READDIRPLUS3res*)rop->reply;
*reply = { .status = NFS3_OK }; *reply = { .status = NFS3_OK };
*(uint64_t*)(reply->resok.cookieverf) = self->parent->dir_info.at(dir).mod_rev; *(uint64_t*)(reply->resok.cookieverf) = self->parent->blockfs->dir_info.at(dir).mod_rev;
reply->resok.reply.entries = entries.size() ? &entries.begin()->second : NULL; reply->resok.reply.entries = entries.size() ? &entries.begin()->second : NULL;
reply->resok.reply.eof = eof; reply->resok.reply.eof = eof;
} }
@ -1155,250 +1135,123 @@ static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
{ {
READDIR3res *reply = (READDIR3res*)rop->reply; READDIR3res *reply = (READDIR3res*)rop->reply;
*reply = { .status = NFS3_OK }; *reply = { .status = NFS3_OK };
*(uint64_t*)(reply->resok.cookieverf) = self->parent->dir_info.at(dir).mod_rev; *(uint64_t*)(reply->resok.cookieverf) = self->parent->blockfs->dir_info.at(dir).mod_rev;
reply->resok.reply.entries = entries.size() ? (entry3*)&entries.begin()->second : NULL; reply->resok.reply.entries = entries.size() ? (entry3*)&entries.begin()->second : NULL;
reply->resok.reply.eof = eof; reply->resok.reply.eof = eof;
} }
rpc_queue_reply(rop); rpc_queue_reply(rop);
} }
static int nfs3_readdir_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_readdir_proc(void *opaque, rpc_op_t *rop)
{ {
nfs3_readdir_common(opaque, rop, false); block_nfs3_readdir_common(opaque, rop, false);
return 0; return 0;
} }
static int nfs3_readdirplus_proc(void *opaque, rpc_op_t *rop) static int block_nfs3_readdirplus_proc(void *opaque, rpc_op_t *rop)
{ {
nfs3_readdir_common(opaque, rop, true); block_nfs3_readdir_common(opaque, rop, true);
return 0; return 0;
} }
// Get file system statistics void block_fs_state_t::init(nfs_proxy_t *proxy)
static int nfs3_fsstat_proc(void *opaque, rpc_op_t *rop)
{ {
nfs_client_t *self = (nfs_client_t*)opaque; // We need inode name hashes for NFS handles to remain stateless and <= 64 bytes long
//FSSTAT3args *args = (FSSTAT3args*)rop->request; dir_info[""] = (nfs_dir_t){
FSSTAT3res *reply = (FSSTAT3res*)rop->reply; .id = 1,
uint64_t tbytes = 0, fbytes = 0; .mod_rev = 0,
auto pst_it = self->parent->pool_stats.find(self->parent->default_pool_id);
if (pst_it != self->parent->pool_stats.end())
{
auto ttb = pst_it->second["total_raw_tb"].number_value();
auto ftb = (pst_it->second["total_raw_tb"].number_value() - pst_it->second["used_raw_tb"].number_value());
tbytes = ttb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
fbytes = ftb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
}
*reply = (FSSTAT3res){
.status = NFS3_OK,
.resok = (FSSTAT3resok){
.obj_attributes = {
.attributes_follow = 1,
.attributes = get_dir_attributes(self, ""),
},
.tbytes = tbytes, // total bytes
.fbytes = fbytes, // free bytes
.abytes = fbytes, // available bytes
.tfiles = (size3)(1 << 31), // maximum total files
.ffiles = (size3)(1 << 31), // free files
.afiles = (size3)(1 << 31), // available files
.invarsec = 0,
},
}; };
rpc_queue_reply(rop); clock_gettime(CLOCK_REALTIME, &dir_info[""].mtime);
return 0; assert(proxy->cli->st_cli.on_inode_change_hook == NULL);
} proxy->cli->st_cli.on_inode_change_hook = [this, proxy](inode_t changed_inode, bool removed)
static int nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
FSINFO3args *args = (FSINFO3args*)rop->request;
FSINFO3res *reply = (FSINFO3res*)rop->reply;
if (args->fsroot != "roothandle")
{ {
// Example error auto inode_cfg_it = proxy->cli->st_cli.inode_config.find(changed_inode);
*reply = (FSINFO3res){ .status = NFS3ERR_INVAL }; if (inode_cfg_it == proxy->cli->st_cli.inode_config.end())
} {
else return;
{ }
// Fill info auto & inode_cfg = inode_cfg_it->second;
*reply = (FSINFO3res){ std::string full_name = inode_cfg.name;
.status = NFS3_OK, if (proxy->name_prefix != "" && full_name.substr(0, proxy->name_prefix.size()) != proxy->name_prefix)
.resok = (FSINFO3resok){ {
.obj_attributes = { return;
.attributes_follow = 1, }
.attributes = get_dir_attributes(self, ""), // Calculate directory modification time and revision (used as "cookie verifier")
}, timespec now;
.rtmax = 128*1024*1024, clock_gettime(CLOCK_REALTIME, &now);
.rtpref = 128*1024*1024, dir_info[""].mod_rev = dir_info[""].mod_rev < inode_cfg.mod_revision ? inode_cfg.mod_revision : dir_info[""].mod_rev;
.rtmult = 4096, dir_info[""].mtime = now;
.wtmax = 128*1024*1024, int pos = full_name.find('/', proxy->name_prefix.size());
.wtpref = 128*1024*1024, while (pos >= 0)
.wtmult = 4096, {
.dtpref = 128, std::string dir = full_name.substr(0, pos);
.maxfilesize = 0x7fffffffffffffff, auto & dinf = dir_info[dir];
.time_delta = { if (!dinf.id)
.seconds = 1, dinf.id = next_dir_id++;
.nseconds = 0, dinf.mod_rev = dinf.mod_rev < inode_cfg.mod_revision ? inode_cfg.mod_revision : dinf.mod_rev;
}, dinf.mtime = now;
.properties = FSF3_SYMLINK | FSF3_HOMOGENEOUS, dir_by_hash["S"+base64_encode(sha256(dir))] = dir;
}, pos = full_name.find('/', pos+1);
}; }
} // Alter inode_by_hash
rpc_queue_reply(rop); if (removed)
return 0; {
} auto ino_it = hash_by_inode.find(changed_inode);
if (ino_it != hash_by_inode.end())
static int nfs3_pathconf_proc(void *opaque, rpc_op_t *rop) {
{ inode_by_hash.erase(ino_it->second);
//nfs_client_t *self = (nfs_client_t*)opaque; hash_by_inode.erase(ino_it);
PATHCONF3args *args = (PATHCONF3args*)rop->request; }
PATHCONF3res *reply = (PATHCONF3res*)rop->reply; }
if (args->object != "roothandle") else
{ {
// Example error std::string hash = "S"+base64_encode(sha256(full_name));
*reply = (PATHCONF3res){ .status = NFS3ERR_INVAL }; auto hbi_it = hash_by_inode.find(changed_inode);
} if (hbi_it != hash_by_inode.end() && hbi_it->second != hash)
else {
{ // inode had a different name, remove old hash=>inode pointer
// Fill info inode_by_hash.erase(hbi_it->second);
bool_t x = FALSE; }
*reply = (PATHCONF3res){ inode_by_hash[hash] = changed_inode;
.status = NFS3_OK, hash_by_inode[changed_inode] = hash;
.resok = (PATHCONF3resok){ }
.obj_attributes = {
// Without at least one reference to a non-constant value (local variable or something else),
// with gcc 8 we get "internal compiler error: side-effects element in no-side-effects CONSTRUCTOR" here
// FIXME: get rid of this after raising compiler requirement
.attributes_follow = x,
},
.linkmax = 0,
.name_max = 255,
.no_trunc = TRUE,
.chown_restricted = FALSE,
.case_insensitive = FALSE,
.case_preserving = TRUE,
},
};
}
rpc_queue_reply(rop);
return 0;
}
static int nfs3_commit_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
//COMMIT3args *args = (COMMIT3args*)rop->request;
cluster_op_t *op = new cluster_op_t;
// fsync. we don't know how to fsync a single inode, so just fsync everything
op->opcode = OSD_OP_SYNC;
op->callback = [self, rop](cluster_op_t *op)
{
COMMIT3res *reply = (COMMIT3res*)rop->reply;
*reply = (COMMIT3res){ .status = vitastor_nfs_map_err(op->retval) };
*(uint64_t*)reply->resok.verf = self->parent->server_id;
rpc_queue_reply(rop);
}; };
self->parent->cli->execute(op);
return 1;
} }
static int mount3_mnt_proc(void *opaque, rpc_op_t *rop) void nfs_block_procs(nfs_client_t *self)
{
//nfs_client_t *self = (nfs_client_t*)opaque;
//nfs_dirpath *args = (nfs_dirpath*)rop->request;
nfs_mountres3 *reply = (nfs_mountres3*)rop->reply;
u_int flavor = RPC_AUTH_NONE;
reply->fhs_status = MNT3_OK;
reply->mountinfo.fhandle = xdr_copy_string(rop->xdrs, "roothandle");
reply->mountinfo.auth_flavors.auth_flavors_len = 1;
reply->mountinfo.auth_flavors.auth_flavors_val = (u_int*)xdr_copy_string(rop->xdrs, (char*)&flavor, sizeof(u_int)).data;
rpc_queue_reply(rop);
return 0;
}
static int mount3_dump_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
nfs_mountlist *reply = (nfs_mountlist*)rop->reply;
*reply = (struct nfs_mountbody*)malloc_or_die(sizeof(struct nfs_mountbody));
xdr_add_malloc(rop->xdrs, *reply);
(*reply)->ml_hostname = xdr_copy_string(rop->xdrs, "127.0.0.1");
(*reply)->ml_directory = xdr_copy_string(rop->xdrs, self->parent->export_root);
(*reply)->ml_next = NULL;
rpc_queue_reply(rop);
return 0;
}
static int mount3_umnt_proc(void *opaque, rpc_op_t *rop)
{
//nfs_client_t *self = (nfs_client_t*)opaque;
//nfs_dirpath *arg = (nfs_dirpath*)rop->request;
// do nothing
rpc_queue_reply(rop);
return 0;
}
static int mount3_umntall_proc(void *opaque, rpc_op_t *rop)
{
// do nothing
rpc_queue_reply(rop);
return 0;
}
static int mount3_export_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
nfs_exports *reply = (nfs_exports*)rop->reply;
*reply = (struct nfs_exportnode*)calloc_or_die(1, sizeof(struct nfs_exportnode) + sizeof(struct nfs_groupnode));
xdr_add_malloc(rop->xdrs, *reply);
(*reply)->ex_dir = xdr_copy_string(rop->xdrs, self->parent->export_root);
(*reply)->ex_groups = (struct nfs_groupnode*)(reply+1);
(*reply)->ex_groups->gr_name = xdr_copy_string(rop->xdrs, "127.0.0.1");
(*reply)->ex_groups->gr_next = NULL;
(*reply)->ex_next = NULL;
rpc_queue_reply(rop);
return 0;
}
nfs_client_t::nfs_client_t()
{ {
struct rpc_service_proc_t pt[] = { struct rpc_service_proc_t pt[] = {
{NFS_PROGRAM, NFS_V3, NFS3_NULL, nfs3_null_proc, NULL, 0, NULL, 0, this}, {NFS_PROGRAM, NFS_V3, NFS3_NULL, nfs3_null_proc, NULL, 0, NULL, 0, self},
{NFS_PROGRAM, NFS_V3, NFS3_GETATTR, nfs3_getattr_proc, (xdrproc_t)xdr_GETATTR3args, sizeof(GETATTR3args), (xdrproc_t)xdr_GETATTR3res, sizeof(GETATTR3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_GETATTR, block_nfs3_getattr_proc, (xdrproc_t)xdr_GETATTR3args, sizeof(GETATTR3args), (xdrproc_t)xdr_GETATTR3res, sizeof(GETATTR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_SETATTR, nfs3_setattr_proc, (xdrproc_t)xdr_SETATTR3args, sizeof(SETATTR3args), (xdrproc_t)xdr_SETATTR3res, sizeof(SETATTR3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_SETATTR, block_nfs3_setattr_proc, (xdrproc_t)xdr_SETATTR3args, sizeof(SETATTR3args), (xdrproc_t)xdr_SETATTR3res, sizeof(SETATTR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_LOOKUP, nfs3_lookup_proc, (xdrproc_t)xdr_LOOKUP3args, sizeof(LOOKUP3args), (xdrproc_t)xdr_LOOKUP3res, sizeof(LOOKUP3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_LOOKUP, block_nfs3_lookup_proc, (xdrproc_t)xdr_LOOKUP3args, sizeof(LOOKUP3args), (xdrproc_t)xdr_LOOKUP3res, sizeof(LOOKUP3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_ACCESS, nfs3_access_proc, (xdrproc_t)xdr_ACCESS3args, sizeof(ACCESS3args), (xdrproc_t)xdr_ACCESS3res, sizeof(ACCESS3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_ACCESS, block_nfs3_access_proc, (xdrproc_t)xdr_ACCESS3args, sizeof(ACCESS3args), (xdrproc_t)xdr_ACCESS3res, sizeof(ACCESS3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READLINK, nfs3_readlink_proc, (xdrproc_t)xdr_READLINK3args, sizeof(READLINK3args), (xdrproc_t)xdr_READLINK3res, sizeof(READLINK3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_READLINK, block_nfs3_readlink_proc, (xdrproc_t)xdr_READLINK3args, sizeof(READLINK3args), (xdrproc_t)xdr_READLINK3res, sizeof(READLINK3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READ, nfs3_read_proc, (xdrproc_t)xdr_READ3args, sizeof(READ3args), (xdrproc_t)xdr_READ3res, sizeof(READ3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_READ, block_nfs3_read_proc, (xdrproc_t)xdr_READ3args, sizeof(READ3args), (xdrproc_t)xdr_READ3res, sizeof(READ3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_WRITE, nfs3_write_proc, (xdrproc_t)xdr_WRITE3args, sizeof(WRITE3args), (xdrproc_t)xdr_WRITE3res, sizeof(WRITE3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_WRITE, block_nfs3_write_proc, (xdrproc_t)xdr_WRITE3args, sizeof(WRITE3args), (xdrproc_t)xdr_WRITE3res, sizeof(WRITE3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_CREATE, nfs3_create_proc, (xdrproc_t)xdr_CREATE3args, sizeof(CREATE3args), (xdrproc_t)xdr_CREATE3res, sizeof(CREATE3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_CREATE, block_nfs3_create_proc, (xdrproc_t)xdr_CREATE3args, sizeof(CREATE3args), (xdrproc_t)xdr_CREATE3res, sizeof(CREATE3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_MKDIR, nfs3_mkdir_proc, (xdrproc_t)xdr_MKDIR3args, sizeof(MKDIR3args), (xdrproc_t)xdr_MKDIR3res, sizeof(MKDIR3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_MKDIR, block_nfs3_mkdir_proc, (xdrproc_t)xdr_MKDIR3args, sizeof(MKDIR3args), (xdrproc_t)xdr_MKDIR3res, sizeof(MKDIR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_SYMLINK, nfs3_symlink_proc, (xdrproc_t)xdr_SYMLINK3args, sizeof(SYMLINK3args), (xdrproc_t)xdr_SYMLINK3res, sizeof(SYMLINK3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_SYMLINK, block_nfs3_symlink_proc, (xdrproc_t)xdr_SYMLINK3args, sizeof(SYMLINK3args), (xdrproc_t)xdr_SYMLINK3res, sizeof(SYMLINK3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_MKNOD, nfs3_mknod_proc, (xdrproc_t)xdr_MKNOD3args, sizeof(MKNOD3args), (xdrproc_t)xdr_MKNOD3res, sizeof(MKNOD3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_MKNOD, block_nfs3_mknod_proc, (xdrproc_t)xdr_MKNOD3args, sizeof(MKNOD3args), (xdrproc_t)xdr_MKNOD3res, sizeof(MKNOD3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_REMOVE, nfs3_remove_proc, (xdrproc_t)xdr_REMOVE3args, sizeof(REMOVE3args), (xdrproc_t)xdr_REMOVE3res, sizeof(REMOVE3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_REMOVE, block_nfs3_remove_proc, (xdrproc_t)xdr_REMOVE3args, sizeof(REMOVE3args), (xdrproc_t)xdr_REMOVE3res, sizeof(REMOVE3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_RMDIR, nfs3_rmdir_proc, (xdrproc_t)xdr_RMDIR3args, sizeof(RMDIR3args), (xdrproc_t)xdr_RMDIR3res, sizeof(RMDIR3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_RMDIR, block_nfs3_rmdir_proc, (xdrproc_t)xdr_RMDIR3args, sizeof(RMDIR3args), (xdrproc_t)xdr_RMDIR3res, sizeof(RMDIR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_RENAME, nfs3_rename_proc, (xdrproc_t)xdr_RENAME3args, sizeof(RENAME3args), (xdrproc_t)xdr_RENAME3res, sizeof(RENAME3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_RENAME, block_nfs3_rename_proc, (xdrproc_t)xdr_RENAME3args, sizeof(RENAME3args), (xdrproc_t)xdr_RENAME3res, sizeof(RENAME3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_LINK, nfs3_link_proc, (xdrproc_t)xdr_LINK3args, sizeof(LINK3args), (xdrproc_t)xdr_LINK3res, sizeof(LINK3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_LINK, block_nfs3_link_proc, (xdrproc_t)xdr_LINK3args, sizeof(LINK3args), (xdrproc_t)xdr_LINK3res, sizeof(LINK3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READDIR, nfs3_readdir_proc, (xdrproc_t)xdr_READDIR3args, sizeof(READDIR3args), (xdrproc_t)xdr_READDIR3res, sizeof(READDIR3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_READDIR, block_nfs3_readdir_proc, (xdrproc_t)xdr_READDIR3args, sizeof(READDIR3args), (xdrproc_t)xdr_READDIR3res, sizeof(READDIR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READDIRPLUS, nfs3_readdirplus_proc, (xdrproc_t)xdr_READDIRPLUS3args, sizeof(READDIRPLUS3args), (xdrproc_t)xdr_READDIRPLUS3res, sizeof(READDIRPLUS3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_READDIRPLUS, block_nfs3_readdirplus_proc, (xdrproc_t)xdr_READDIRPLUS3args, sizeof(READDIRPLUS3args), (xdrproc_t)xdr_READDIRPLUS3res, sizeof(READDIRPLUS3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_FSSTAT, nfs3_fsstat_proc, (xdrproc_t)xdr_FSSTAT3args, sizeof(FSSTAT3args), (xdrproc_t)xdr_FSSTAT3res, sizeof(FSSTAT3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_FSSTAT, nfs3_fsstat_proc, (xdrproc_t)xdr_FSSTAT3args, sizeof(FSSTAT3args), (xdrproc_t)xdr_FSSTAT3res, sizeof(FSSTAT3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_FSINFO, nfs3_fsinfo_proc, (xdrproc_t)xdr_FSINFO3args, sizeof(FSINFO3args), (xdrproc_t)xdr_FSINFO3res, sizeof(FSINFO3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_FSINFO, nfs3_fsinfo_proc, (xdrproc_t)xdr_FSINFO3args, sizeof(FSINFO3args), (xdrproc_t)xdr_FSINFO3res, sizeof(FSINFO3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_PATHCONF, nfs3_pathconf_proc, (xdrproc_t)xdr_PATHCONF3args, sizeof(PATHCONF3args), (xdrproc_t)xdr_PATHCONF3res, sizeof(PATHCONF3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_PATHCONF, nfs3_pathconf_proc, (xdrproc_t)xdr_PATHCONF3args, sizeof(PATHCONF3args), (xdrproc_t)xdr_PATHCONF3res, sizeof(PATHCONF3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_COMMIT, nfs3_commit_proc, (xdrproc_t)xdr_COMMIT3args, sizeof(COMMIT3args), (xdrproc_t)xdr_COMMIT3res, sizeof(COMMIT3res), this}, {NFS_PROGRAM, NFS_V3, NFS3_COMMIT, nfs3_commit_proc, (xdrproc_t)xdr_COMMIT3args, sizeof(COMMIT3args), (xdrproc_t)xdr_COMMIT3res, sizeof(COMMIT3res), self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_NULL, nfs3_null_proc, NULL, 0, NULL, 0, this}, {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_NULL, nfs3_null_proc, NULL, 0, NULL, 0, self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_MNT, mount3_mnt_proc, (xdrproc_t)xdr_nfs_dirpath, sizeof(nfs_dirpath), (xdrproc_t)xdr_nfs_mountres3, sizeof(nfs_mountres3), this}, {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_MNT, mount3_mnt_proc, (xdrproc_t)xdr_nfs_dirpath, sizeof(nfs_dirpath), (xdrproc_t)xdr_nfs_mountres3, sizeof(nfs_mountres3), self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_DUMP, mount3_dump_proc, NULL, 0, (xdrproc_t)xdr_nfs_mountlist, sizeof(nfs_mountlist), this}, {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_DUMP, mount3_dump_proc, NULL, 0, (xdrproc_t)xdr_nfs_mountlist, sizeof(nfs_mountlist), self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNT, mount3_umnt_proc, (xdrproc_t)xdr_nfs_dirpath, sizeof(nfs_dirpath), NULL, 0, this}, {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNT, mount3_umnt_proc, (xdrproc_t)xdr_nfs_dirpath, sizeof(nfs_dirpath), NULL, 0, self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNTALL, mount3_umntall_proc, NULL, 0, NULL, 0, this}, {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNTALL, mount3_umntall_proc, NULL, 0, NULL, 0, self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_EXPORT, mount3_export_proc, NULL, 0, (xdrproc_t)xdr_nfs_exports, sizeof(nfs_exports), this}, {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_EXPORT, mount3_export_proc, NULL, 0, (xdrproc_t)xdr_nfs_exports, sizeof(nfs_exports), self},
}; };
for (int i = 0; i < sizeof(pt)/sizeof(pt[0]); i++) for (int i = 0; i < sizeof(pt)/sizeof(pt[0]); i++)
{ {
proc_table.insert(pt[i]); self->proc_table.insert(pt[i]);
} }
} }
nfs_client_t::~nfs_client_t()
{
}

57
src/nfs_block.h Normal file
View File

@ -0,0 +1,57 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over Vitastor block images - header
#pragma once
struct nfs_dir_t
{
uint64_t id;
uint64_t mod_rev;
timespec mtime;
};
struct extend_size_t
{
inode_t inode;
uint64_t new_size;
};
inline bool operator < (const extend_size_t &a, const extend_size_t &b)
{
return a.inode < b.inode || a.inode == b.inode && a.new_size < b.new_size;
}
struct extend_write_t
{
rpc_op_t *rop;
int resize_res, write_res; // 1 = started, 0 = completed OK, -errno = completed with error
};
struct extend_inode_t
{
uint64_t cur_extend = 0, next_extend = 0;
};
struct block_fs_state_t
{
// filehandle = "S"+base64(sha256(full name with prefix)) or "roothandle" for mount root)
uint64_t next_dir_id = 2;
// filehandle => dir with name_prefix
std::map<std::string, std::string> dir_by_hash;
// dir with name_prefix => dir info
std::map<std::string, nfs_dir_t> dir_info;
// filehandle => inode ID
std::map<std::string, inode_t> inode_by_hash;
// inode ID => filehandle
std::map<inode_t, std::string> hash_by_inode;
// inode extend requests in progress
std::map<inode_t, extend_inode_t> extends;
std::multimap<extend_size_t, extend_write_t> extend_writes;
void init(nfs_proxy_t *proxy);
};
nfsstat3 vitastor_nfs_map_err(int err);

22
src/nfs_common.h Normal file
View File

@ -0,0 +1,22 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy - common functions
#pragma once
#include "nfs/nfs.h"
void nfs_block_procs(nfs_client_t *self);
void nfs_kv_procs(nfs_client_t *self);
int nfs3_fsstat_proc(void *opaque, rpc_op_t *rop);
int nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop);
int nfs3_pathconf_proc(void *opaque, rpc_op_t *rop);
int nfs3_access_proc(void *opaque, rpc_op_t *rop);
int nfs3_null_proc(void *opaque, rpc_op_t *rop);
int nfs3_commit_proc(void *opaque, rpc_op_t *rop);
int mount3_mnt_proc(void *opaque, rpc_op_t *rop);
int mount3_dump_proc(void *opaque, rpc_op_t *rop);
int mount3_umnt_proc(void *opaque, rpc_op_t *rop);
int mount3_umntall_proc(void *opaque, rpc_op_t *rop);
int mount3_export_proc(void *opaque, rpc_op_t *rop);

124
src/nfs_fsstat.cpp Normal file
View File

@ -0,0 +1,124 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy - common FSSTAT, FSINFO, PATHCONF
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
// Get file system statistics
int nfs3_fsstat_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
//FSSTAT3args *args = (FSSTAT3args*)rop->request;
if (self->parent->trace)
fprintf(stderr, "[%d] FSSTAT\n", self->nfs_fd);
FSSTAT3res *reply = (FSSTAT3res*)rop->reply;
uint64_t tbytes = 0, fbytes = 0;
auto pst_it = self->parent->pool_stats.find(self->parent->default_pool_id);
if (pst_it != self->parent->pool_stats.end())
{
auto ttb = pst_it->second["total_raw_tb"].number_value();
auto ftb = (pst_it->second["total_raw_tb"].number_value() - pst_it->second["used_raw_tb"].number_value());
tbytes = ttb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
fbytes = ftb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
}
*reply = (FSSTAT3res){
.status = NFS3_OK,
.resok = (FSSTAT3resok){
.obj_attributes = {
.attributes_follow = 0,
//.attributes = get_root_attributes(self),
},
.tbytes = tbytes, // total bytes
.fbytes = fbytes, // free bytes
.abytes = fbytes, // available bytes
.tfiles = (size3)1 << (63-POOL_ID_BITS), // maximum total files
.ffiles = (size3)1 << (63-POOL_ID_BITS), // free files
.afiles = (size3)1 << (63-POOL_ID_BITS), // available files
.invarsec = 0,
},
};
rpc_queue_reply(rop);
return 0;
}
int nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
FSINFO3args *args = (FSINFO3args*)rop->request;
FSINFO3res *reply = (FSINFO3res*)rop->reply;
if (self->parent->trace)
fprintf(stderr, "[%d] FSINFO %s\n", self->nfs_fd, std::string(args->fsroot).c_str());
if (args->fsroot != NFS_ROOT_HANDLE)
{
*reply = (FSINFO3res){ .status = NFS3ERR_INVAL };
}
else
{
// Fill info
*reply = (FSINFO3res){
.status = NFS3_OK,
.resok = (FSINFO3resok){
.obj_attributes = {
.attributes_follow = 0,
//.attributes = get_root_attributes(self),
},
.rtmax = 128*1024*1024,
.rtpref = 128*1024*1024,
.rtmult = 4096,
.wtmax = 128*1024*1024,
.wtpref = 128*1024*1024,
.wtmult = 4096,
.dtpref = 128,
.maxfilesize = 0x7fffffffffffffff,
.time_delta = {
.seconds = 1,
.nseconds = 0,
},
.properties = FSF3_SYMLINK | FSF3_HOMOGENEOUS,
},
};
}
rpc_queue_reply(rop);
return 0;
}
int nfs3_pathconf_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
PATHCONF3args *args = (PATHCONF3args*)rop->request;
PATHCONF3res *reply = (PATHCONF3res*)rop->reply;
if (self->parent->trace)
fprintf(stderr, "[%d] PATHCONF %s\n", self->nfs_fd, std::string(args->object).c_str());
if (args->object != NFS_ROOT_HANDLE)
{
*reply = (PATHCONF3res){ .status = NFS3ERR_INVAL };
}
else
{
// Fill info
*reply = (PATHCONF3res){
.status = NFS3_OK,
.resok = (PATHCONF3resok){
.obj_attributes = {
// Without at least one reference to a non-constant value (local variable or something else),
// with gcc 8 we get "internal compiler error: side-effects element in no-side-effects CONSTRUCTOR" here
// FIXME: get rid of this after raising compiler requirement
.attributes_follow = 0,
//.attributes = get_root_attributes(self),
},
.linkmax = 0,
.name_max = 255,
.no_trunc = TRUE,
.chown_restricted = FALSE,
.case_insensitive = FALSE,
.case_preserving = TRUE,
},
};
}
rpc_queue_reply(rop);
return 0;
}

174
src/nfs_kv.cpp Normal file
View File

@ -0,0 +1,174 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - common functions
#include <sys/time.h>
#include "str_util.h"
#include "nfs_proxy.h"
#include "nfs_common.h"
#include "nfs_kv.h"
nfstime3 nfstime_from_str(const std::string & s)
{
nfstime3 t;
auto p = s.find(".");
if (p != std::string::npos)
{
t.seconds = stoull_full(s.substr(0, p), 10);
t.nseconds = stoull_full(s.substr(p+1), 10);
p = s.size()-p-1;
for (; p < 9; p++)
t.nseconds *= 10;
for (; p > 9; p--)
t.nseconds /= 10;
}
else
t.seconds = stoull_full(s, 10);
return t;
}
std::string nfstime_to_str(nfstime3 t)
{
char buf[32];
snprintf(buf, sizeof(buf), "%u.%09u", t.seconds, t.nseconds);
return buf;
}
int kv_map_type(const std::string & type)
{
return (type == "" || type == "file" ? NF3REG :
(type == "dir" ? NF3DIR :
(type == "blk" ? NF3BLK :
(type == "chr" ? NF3CHR :
(type == "link" ? NF3LNK :
(type == "sock" ? NF3SOCK :
(type == "fifo" ? NF3FIFO : -1)))))));
}
fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
{
auto type = kv_map_type(attrs["type"].string_value());
auto mode = attrs["mode"].uint64_value();
auto nlink = attrs["nlink"].uint64_value();
nfstime3 mtime = nfstime_from_str(attrs["mtime"].string_value());
nfstime3 atime = attrs["atime"].is_null() ? mtime : nfstime_from_str(attrs["atime"].string_value());
// FIXME In theory we could store the binary structure itself instead of JSON
return (fattr3){
.type = (type == 0 ? NF3REG : (ftype3)type),
.mode = (attrs["mode"].is_null() ? (type == NF3DIR ? 0755 : 0644) : (uint32_t)mode),
.nlink = (nlink == 0 ? 1 : (uint32_t)nlink),
.uid = (uint32_t)attrs["uid"].uint64_value(),
.gid = (uint32_t)attrs["gid"].uint64_value(),
.size = (type == NF3DIR ? 4096 : attrs["size"].uint64_value()),
.used = (type == NF3DIR ? 4096 : attrs["alloc"].uint64_value()),
.rdev = (type == NF3BLK || type == NF3CHR
? (specdata3){ (uint32_t)attrs["major"].uint64_value(), (uint32_t)attrs["minor"].uint64_value() }
: (specdata3){}),
.fsid = self->parent->fsid,
.fileid = ino,
.atime = atime,
.mtime = mtime,
.ctime = mtime,
};
}
std::string kv_direntry_key(uint64_t dir_ino, const std::string & filename)
{
// encode as: d <length> <hex dir_ino> / <filename>
char key[24] = { 0 };
snprintf(key, sizeof(key), "d-%lx/", dir_ino);
int n = strnlen(key, sizeof(key)-1) - 3;
if (n < 10)
key[1] = '0'+n;
else
key[1] = 'A'+(n-10);
return (char*)key + filename;
}
std::string kv_direntry_filename(const std::string & key)
{
// decode as: d <length> <hex dir_ino> / <filename>
auto pos = key.find("/");
if (pos != std::string::npos)
return key.substr(pos+1);
return key;
}
std::string kv_inode_key(uint64_t ino)
{
char key[24] = { 0 };
snprintf(key, sizeof(key), "i-%lx", ino);
int n = strnlen(key, sizeof(key)-1) - 2;
if (n < 10)
key[1] = '0'+n;
else
key[1] = 'A'+(n-10);
return std::string(key, n+2);
}
std::string kv_fh(uint64_t ino)
{
return "S"+std::string((char*)&ino, 8);
}
uint64_t kv_fh_inode(const std::string & fh)
{
if (fh.size() == 1 && fh[0] == 'R')
{
return 1;
}
else if (fh.size() == 9 && fh[0] == 'S')
{
return *(uint64_t*)&fh[1];
}
else if (fh.size() > 17 && fh[0] == 'I')
{
return *(uint64_t*)&fh[fh.size()-8];
}
return 0;
}
bool kv_fh_valid(const std::string & fh)
{
return fh == NFS_ROOT_HANDLE || fh.size() == 9 && fh[0] == 'S' || fh.size() > 17 && fh[0] == 'I';
}
void nfs_kv_procs(nfs_client_t *self)
{
struct rpc_service_proc_t pt[] = {
{NFS_PROGRAM, NFS_V3, NFS3_NULL, nfs3_null_proc, NULL, 0, NULL, 0, self},
{NFS_PROGRAM, NFS_V3, NFS3_GETATTR, kv_nfs3_getattr_proc, (xdrproc_t)xdr_GETATTR3args, sizeof(GETATTR3args), (xdrproc_t)xdr_GETATTR3res, sizeof(GETATTR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_SETATTR, kv_nfs3_setattr_proc, (xdrproc_t)xdr_SETATTR3args, sizeof(SETATTR3args), (xdrproc_t)xdr_SETATTR3res, sizeof(SETATTR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_LOOKUP, kv_nfs3_lookup_proc, (xdrproc_t)xdr_LOOKUP3args, sizeof(LOOKUP3args), (xdrproc_t)xdr_LOOKUP3res, sizeof(LOOKUP3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_ACCESS, nfs3_access_proc, (xdrproc_t)xdr_ACCESS3args, sizeof(ACCESS3args), (xdrproc_t)xdr_ACCESS3res, sizeof(ACCESS3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READLINK, kv_nfs3_readlink_proc, (xdrproc_t)xdr_READLINK3args, sizeof(READLINK3args), (xdrproc_t)xdr_READLINK3res, sizeof(READLINK3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READ, kv_nfs3_read_proc, (xdrproc_t)xdr_READ3args, sizeof(READ3args), (xdrproc_t)xdr_READ3res, sizeof(READ3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_WRITE, kv_nfs3_write_proc, (xdrproc_t)xdr_WRITE3args, sizeof(WRITE3args), (xdrproc_t)xdr_WRITE3res, sizeof(WRITE3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_CREATE, kv_nfs3_create_proc, (xdrproc_t)xdr_CREATE3args, sizeof(CREATE3args), (xdrproc_t)xdr_CREATE3res, sizeof(CREATE3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_MKDIR, kv_nfs3_mkdir_proc, (xdrproc_t)xdr_MKDIR3args, sizeof(MKDIR3args), (xdrproc_t)xdr_MKDIR3res, sizeof(MKDIR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_SYMLINK, kv_nfs3_symlink_proc, (xdrproc_t)xdr_SYMLINK3args, sizeof(SYMLINK3args), (xdrproc_t)xdr_SYMLINK3res, sizeof(SYMLINK3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_MKNOD, kv_nfs3_mknod_proc, (xdrproc_t)xdr_MKNOD3args, sizeof(MKNOD3args), (xdrproc_t)xdr_MKNOD3res, sizeof(MKNOD3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_REMOVE, kv_nfs3_remove_proc, (xdrproc_t)xdr_REMOVE3args, sizeof(REMOVE3args), (xdrproc_t)xdr_REMOVE3res, sizeof(REMOVE3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_RMDIR, kv_nfs3_rmdir_proc, (xdrproc_t)xdr_RMDIR3args, sizeof(RMDIR3args), (xdrproc_t)xdr_RMDIR3res, sizeof(RMDIR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_RENAME, kv_nfs3_rename_proc, (xdrproc_t)xdr_RENAME3args, sizeof(RENAME3args), (xdrproc_t)xdr_RENAME3res, sizeof(RENAME3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_LINK, kv_nfs3_link_proc, (xdrproc_t)xdr_LINK3args, sizeof(LINK3args), (xdrproc_t)xdr_LINK3res, sizeof(LINK3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READDIR, kv_nfs3_readdir_proc, (xdrproc_t)xdr_READDIR3args, sizeof(READDIR3args), (xdrproc_t)xdr_READDIR3res, sizeof(READDIR3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_READDIRPLUS, kv_nfs3_readdirplus_proc, (xdrproc_t)xdr_READDIRPLUS3args, sizeof(READDIRPLUS3args), (xdrproc_t)xdr_READDIRPLUS3res, sizeof(READDIRPLUS3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_FSSTAT, nfs3_fsstat_proc, (xdrproc_t)xdr_FSSTAT3args, sizeof(FSSTAT3args), (xdrproc_t)xdr_FSSTAT3res, sizeof(FSSTAT3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_FSINFO, nfs3_fsinfo_proc, (xdrproc_t)xdr_FSINFO3args, sizeof(FSINFO3args), (xdrproc_t)xdr_FSINFO3res, sizeof(FSINFO3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_PATHCONF, nfs3_pathconf_proc, (xdrproc_t)xdr_PATHCONF3args, sizeof(PATHCONF3args), (xdrproc_t)xdr_PATHCONF3res, sizeof(PATHCONF3res), self},
{NFS_PROGRAM, NFS_V3, NFS3_COMMIT, nfs3_commit_proc, (xdrproc_t)xdr_COMMIT3args, sizeof(COMMIT3args), (xdrproc_t)xdr_COMMIT3res, sizeof(COMMIT3res), self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_NULL, nfs3_null_proc, NULL, 0, NULL, 0, self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_MNT, mount3_mnt_proc, (xdrproc_t)xdr_nfs_dirpath, sizeof(nfs_dirpath), (xdrproc_t)xdr_nfs_mountres3, sizeof(nfs_mountres3), self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_DUMP, mount3_dump_proc, NULL, 0, (xdrproc_t)xdr_nfs_mountlist, sizeof(nfs_mountlist), self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNT, mount3_umnt_proc, (xdrproc_t)xdr_nfs_dirpath, sizeof(nfs_dirpath), NULL, 0, self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNTALL, mount3_umntall_proc, NULL, 0, NULL, 0, self},
{MOUNT_PROGRAM, MOUNT_V3, MOUNT3_EXPORT, mount3_export_proc, NULL, 0, (xdrproc_t)xdr_nfs_exports, sizeof(nfs_exports), self},
};
for (int i = 0; i < sizeof(pt)/sizeof(pt[0]); i++)
{
self->proc_table.insert(pt[i]);
}
}

95
src/nfs_kv.h Normal file
View File

@ -0,0 +1,95 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - header
#pragma once
#include "nfs/nfs.h"
#define KV_ROOT_INODE 1
#define KV_NEXT_ID_KEY "id"
#define SHARED_FILE_MAGIC_V1 0x711A5158A6EDF17E
struct nfs_kv_write_state;
struct list_cookie_t
{
uint64_t dir_ino, cookieverf, cookie;
};
inline bool operator < (const list_cookie_t & a, const list_cookie_t & b)
{
return a.dir_ino < b.dir_ino || a.dir_ino == b.dir_ino &&
(a.cookieverf < b.cookieverf || a.cookieverf == b.cookieverf && a.cookie < b.cookie);
};
struct list_cookie_val_t
{
std::string key;
};
struct shared_alloc_queue_t
{
nfs_kv_write_state *st;
int state;
uint64_t size;
};
struct kv_inode_extend_t
{
int refcnt = 0;
uint64_t cur_extend = 0, next_extend = 0, done_extend = 0;
std::vector<std::function<void()>> waiters;
};
struct kv_fs_state_t
{
std::map<list_cookie_t, list_cookie_val_t> list_cookies;
uint64_t fs_next_id = 0, fs_allocated_id = 0;
std::vector<uint64_t> unallocated_ids;
std::vector<shared_alloc_queue_t> allocating_shared;
uint64_t cur_shared_inode = 0, cur_shared_offset = 0;
std::map<inode_t, kv_inode_extend_t> extends;
};
struct shared_file_header_t
{
uint64_t magic = 0;
uint64_t inode = 0;
uint64_t size = 0;
};
nfsstat3 vitastor_nfs_map_err(int err);
nfstime3 nfstime_from_str(const std::string & s);
std::string nfstime_to_str(nfstime3 t);
int kv_map_type(const std::string & type);
fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs);
std::string kv_direntry_key(uint64_t dir_ino, const std::string & filename);
std::string kv_direntry_filename(const std::string & key);
std::string kv_inode_key(uint64_t ino);
std::string kv_fh(uint64_t ino);
uint64_t kv_fh_inode(const std::string & fh);
bool kv_fh_valid(const std::string & fh);
void allocate_new_id(nfs_client_t *self, std::function<void(int res, uint64_t new_id)> cb);
void kv_read_inode(nfs_client_t *self, uint64_t ino,
std::function<void(int res, const std::string & value, json11::Json ientry)> cb,
bool allow_cache = false);
uint64_t align_shared_size(nfs_client_t *self, uint64_t size);
int kv_nfs3_getattr_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_setattr_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_lookup_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_readlink_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_read_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_write_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_create_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_mkdir_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_symlink_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_mknod_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_remove_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_rmdir_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_rename_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_link_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_readdir_proc(void *opaque, rpc_op_t *rop);
int kv_nfs3_readdirplus_proc(void *opaque, rpc_op_t *rop);

324
src/nfs_kv_create.cpp Normal file
View File

@ -0,0 +1,324 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - CREATE, MKDIR, SYMLINK, MKNOD
#include <sys/time.h>
#include "str_util.h"
#include "nfs_proxy.h"
#include "nfs_kv.h"
void allocate_new_id(nfs_client_t *self, std::function<void(int res, uint64_t new_id)> cb)
{
if (self->parent->kvfs->fs_next_id <= self->parent->kvfs->fs_allocated_id)
{
cb(0, self->parent->kvfs->fs_next_id++);
return;
}
else if (self->parent->kvfs->fs_next_id > self->parent->fs_inode_count)
{
cb(-ENOSPC, 0);
return;
}
self->parent->db->get(KV_NEXT_ID_KEY, [=](int res, const std::string & prev_str)
{
if (res < 0 && res != -ENOENT)
{
cb(res, 0);
return;
}
uint64_t prev_val = stoull_full(prev_str);
if (prev_val >= self->parent->fs_inode_count)
{
cb(-ENOSPC, 0);
return;
}
if (prev_val < 1)
{
prev_val = 1;
}
uint64_t new_val = prev_val + self->parent->id_alloc_batch_size;
if (new_val >= self->parent->fs_inode_count)
{
new_val = self->parent->fs_inode_count;
}
self->parent->db->set(KV_NEXT_ID_KEY, std::to_string(new_val), [=](int res)
{
if (res == -EAGAIN)
{
// CAS failure - retry
allocate_new_id(self, cb);
}
else if (res < 0)
{
cb(res, 0);
}
else
{
self->parent->kvfs->fs_next_id = prev_val+2;
self->parent->kvfs->fs_allocated_id = new_val;
cb(0, prev_val+1);
}
}, [prev_val](int res, const std::string & value)
{
// FIXME: Allow to modify value from CAS callback? ("update" query)
return res < 0 || stoull_full(value) == prev_val;
});
});
}
struct kv_create_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
bool exclusive = false;
uint64_t verf = 0;
uint64_t dir_ino = 0;
std::string filename;
uint64_t new_id = 0;
json11::Json::object attrobj;
json11::Json attrs;
std::string direntry_text;
uint64_t dup_ino = 0;
std::function<void(int res)> cb;
};
static void kv_do_create(kv_create_state *st)
{
if (st->self->parent->trace)
fprintf(stderr, "[%d] CREATE %lu/%s ATTRS %s\n", st->self->nfs_fd, st->dir_ino, st->filename.c_str(), json11::Json(st->attrobj).dump().c_str());
if (st->filename == "" || st->filename.find("/") != std::string::npos)
{
auto cb = std::move(st->cb);
cb(-EINVAL);
return;
}
// Generate inode ID
allocate_new_id(st->self, [st](int res, uint64_t new_id)
{
if (res < 0)
{
auto cb = std::move(st->cb);
cb(res);
return;
}
st->new_id = new_id;
auto direntry = json11::Json::object{ { "ino", st->new_id } };
if (st->attrobj["type"].string_value() == "dir")
{
direntry["type"] = "dir";
}
st->attrs = std::move(st->attrobj);
st->direntry_text = json11::Json(direntry).dump().c_str();
// Set direntry
st->self->parent->db->set(kv_direntry_key(st->dir_ino, st->filename), st->direntry_text, [st](int res)
{
if (res < 0)
{
st->self->parent->kvfs->unallocated_ids.push_back(st->new_id);
if (res == -EAGAIN)
{
if (st->dup_ino)
{
st->new_id = st->dup_ino;
res = 0;
}
else
res = -EEXIST;
}
else
fprintf(stderr, "create %lu/%s failed: %s (code %d)\n", st->dir_ino, st->filename.c_str(), strerror(-res), res);
auto cb = std::move(st->cb);
cb(res);
}
else
{
st->self->parent->db->set(kv_inode_key(st->new_id), st->attrs.dump().c_str(), [st](int res)
{
if (res == -EAGAIN)
{
res = -EEXIST;
}
if (res < 0)
{
st->self->parent->db->del(kv_direntry_key(st->dir_ino, st->filename), [st, res](int del_res)
{
if (!del_res)
{
st->self->parent->kvfs->unallocated_ids.push_back(st->new_id);
}
auto cb = std::move(st->cb);
cb(res);
}, [st](int res, const std::string & value)
{
return res != -ENOENT && value == st->direntry_text;
});
}
else
{
auto cb = std::move(st->cb);
cb(0);
}
}, [st](int res, const std::string & value)
{
return res == -ENOENT;
});
}
}, [st](int res, const std::string & value)
{
// CAS compare - check that the key doesn't exist
if (res == 0)
{
std::string err;
auto direntry = json11::Json::parse(value, err);
if (err != "")
{
fprintf(stderr, "Invalid JSON in direntry %s = %s: %s, overwriting\n",
kv_direntry_key(st->dir_ino, st->filename).c_str(), value.c_str(), err.c_str());
return true;
}
if (st->exclusive && direntry["verf"].uint64_value() == st->verf)
{
st->dup_ino = direntry["ino"].uint64_value();
return false;
}
return false;
}
return true;
});
});
}
static void kv_create_setattr(json11::Json::object & attrobj, sattr3 & sattr)
{
if (sattr.mode.set_it)
attrobj["mode"] = (uint64_t)sattr.mode.mode;
if (sattr.uid.set_it)
attrobj["uid"] = (uint64_t)sattr.uid.uid;
if (sattr.gid.set_it)
attrobj["gid"] = (uint64_t)sattr.gid.gid;
if (sattr.atime.set_it)
attrobj["atime"] = nfstime_to_str(sattr.atime.atime);
if (sattr.mtime.set_it)
attrobj["mtime"] = nfstime_to_str(sattr.mtime.mtime);
}
template<class T, class Tok> static void kv_create_reply(kv_create_state *st, int res)
{
T *reply = (T*)st->rop->reply;
if (res < 0)
{
*reply = (T){ .status = vitastor_nfs_map_err(-res) };
}
else
{
*reply = (T){
.status = NFS3_OK,
.resok = (Tok){
.obj = {
.handle_follows = 1,
.handle = xdr_copy_string(st->rop->xdrs, kv_fh(st->new_id)),
},
.obj_attributes = {
.attributes_follow = 1,
.attributes = get_kv_attributes(st->self, st->new_id, st->attrs),
},
},
};
}
rpc_queue_reply(st->rop);
delete st;
}
int kv_nfs3_create_proc(void *opaque, rpc_op_t *rop)
{
kv_create_state *st = new kv_create_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
auto args = (CREATE3args*)rop->request;
st->exclusive = args->how.mode == NFS_EXCLUSIVE;
st->verf = st->exclusive ? *(uint64_t*)&args->how.verf : 0;
st->dir_ino = kv_fh_inode(args->where.dir);
st->filename = args->where.name;
if (args->how.mode == NFS_EXCLUSIVE)
{
st->attrobj["verf"] = *(uint64_t*)&args->how.verf;
}
else if (args->how.mode == NFS_UNCHECKED)
{
kv_create_setattr(st->attrobj, args->how.obj_attributes);
if (args->how.obj_attributes.size.set_it)
{
st->attrobj["size"] = (uint64_t)args->how.obj_attributes.size.size;
st->attrobj["empty"] = true;
}
}
st->cb = [st](int res) { kv_create_reply<CREATE3res, CREATE3resok>(st, res); };
kv_do_create(st);
return 1;
}
int kv_nfs3_mkdir_proc(void *opaque, rpc_op_t *rop)
{
kv_create_state *st = new kv_create_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
auto args = (MKDIR3args*)rop->request;
st->dir_ino = kv_fh_inode(args->where.dir);
st->filename = args->where.name;
st->attrobj["type"] = "dir";
st->attrobj["parent_ino"] = st->dir_ino;
kv_create_setattr(st->attrobj, args->attributes);
st->cb = [st](int res) { kv_create_reply<MKDIR3res, MKDIR3resok>(st, res); };
kv_do_create(st);
return 1;
}
int kv_nfs3_symlink_proc(void *opaque, rpc_op_t *rop)
{
kv_create_state *st = new kv_create_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
auto args = (SYMLINK3args*)rop->request;
st->dir_ino = kv_fh_inode(args->where.dir);
st->filename = args->where.name;
st->attrobj["type"] = "link";
st->attrobj["symlink"] = (std::string)args->symlink.symlink_data;
kv_create_setattr(st->attrobj, args->symlink.symlink_attributes);
st->cb = [st](int res) { kv_create_reply<SYMLINK3res, SYMLINK3resok>(st, res); };
kv_do_create(st);
return 1;
}
int kv_nfs3_mknod_proc(void *opaque, rpc_op_t *rop)
{
kv_create_state *st = new kv_create_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
auto args = (MKNOD3args*)rop->request;
st->dir_ino = kv_fh_inode(args->where.dir);
st->filename = args->where.name;
if (args->what.type == NF3CHR || args->what.type == NF3BLK)
{
st->attrobj["type"] = (args->what.type == NF3CHR ? "chr" : "blk");
st->attrobj["major"] = (uint64_t)args->what.chr_device.spec.specdata1;
st->attrobj["minor"] = (uint64_t)args->what.chr_device.spec.specdata2;
kv_create_setattr(st->attrobj, args->what.chr_device.dev_attributes);
}
else if (args->what.type == NF3SOCK || args->what.type == NF3FIFO)
{
st->attrobj["type"] = (args->what.type == NF3SOCK ? "sock" : "fifo");
kv_create_setattr(st->attrobj, args->what.sock_attributes);
}
else
{
*(MKNOD3res*)rop->reply = (MKNOD3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
st->cb = [st](int res) { kv_create_reply<MKNOD3res, MKNOD3resok>(st, res); };
kv_do_create(st);
return 1;
}

76
src/nfs_kv_getattr.cpp Normal file
View File

@ -0,0 +1,76 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - GETATTR
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
// Attributes are always stored in the inode
void kv_read_inode(nfs_client_t *self, uint64_t ino,
std::function<void(int res, const std::string & value, json11::Json ientry)> cb,
bool allow_cache)
{
auto key = kv_inode_key(ino);
self->parent->db->get(key, [=](int res, const std::string & value)
{
if (ino == KV_ROOT_INODE && res == -ENOENT)
{
// Allow root inode to not exist
cb(0, "", json11::Json(json11::Json::object{ { "type", "dir" } }));
return;
}
if (res < 0)
{
if (res != -ENOENT)
fprintf(stderr, "Error reading inode %s: %s (code %d)\n", kv_inode_key(ino).c_str(), strerror(-res), res);
cb(res, "", json11::Json());
return;
}
std::string err;
auto attrs = json11::Json::parse(value, err);
if (err != "")
{
fprintf(stderr, "Invalid JSON in inode %s = %s: %s\n", kv_inode_key(ino).c_str(), value.c_str(), err.c_str());
res = -EIO;
}
cb(res, value, attrs);
}, allow_cache);
}
int kv_nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
GETATTR3args *args = (GETATTR3args*)rop->request;
GETATTR3res *reply = (GETATTR3res*)rop->reply;
std::string fh = args->object;
auto ino = kv_fh_inode(fh);
if (self->parent->trace)
fprintf(stderr, "[%d] GETATTR %lu\n", self->nfs_fd, ino);
if (!kv_fh_valid(fh))
{
*reply = (GETATTR3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
return 0;
}
kv_read_inode(self, ino, [=](int res, const std::string & value, json11::Json attrs)
{
if (res < 0)
{
*reply = (GETATTR3res){ .status = vitastor_nfs_map_err(-res) };
}
else
{
*reply = (GETATTR3res){
.status = NFS3_OK,
.resok = (GETATTR3resok){
.obj_attributes = get_kv_attributes(self, ino, attrs),
},
};
}
rpc_queue_reply(rop);
});
return 1;
}

189
src/nfs_kv_link.cpp Normal file
View File

@ -0,0 +1,189 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - LINK
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
struct nfs_kv_link_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
uint64_t ino = 0;
uint64_t dir_ino = 0;
std::string filename;
std::string ientry_text;
json11::Json ientry;
bool retrying = false;
int wait = 0;
int res = 0, res2 = 0;
std::function<void(int)> cb;
};
static void nfs_kv_continue_link(nfs_kv_link_state *st, int state)
{
// 1) Read the source inode
// 2) If it's a directory - fail with -EISDIR
// 3) Create the new direntry with the same inode reference
// 4) Update the inode entry with refcount++
// 5) Retry update if CAS failed but the inode exists
// 6) Otherwise fail and remove the new direntry
// Yeah we may leave a bad direntry if we crash
// But the other option is to possibly leave an inode with too big refcount
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else if (state == 4) goto resume_4;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_link()");
abort();
}
resume_0:
// Check that the source inode exists and is not a directory
st->wait = st->retrying ? 1 : 2;
st->res2 = 0;
kv_read_inode(st->self, st->ino, [st](int res, const std::string & value, json11::Json attrs)
{
st->res = res == 0 ? (attrs["type"].string_value() == "dir" ? -EISDIR : 0) : res;
st->ientry_text = value;
st->ientry = attrs;
if (!--st->wait)
nfs_kv_continue_link(st, 1);
});
if (!st->retrying)
{
// Check that the new directory exists
kv_read_inode(st->self, st->dir_ino, [st](int res, const std::string & value, json11::Json attrs)
{
st->res2 = res == 0 ? (attrs["type"].string_value() == "dir" ? 0 : -ENOTDIR) : res;
if (!--st->wait)
nfs_kv_continue_link(st, 1);
});
}
return;
resume_1:
if (st->res < 0 || st->res2 < 0)
{
auto cb = std::move(st->cb);
cb(st->res < 0 ? st->res : st->res2);
return;
}
// Write the new direntry
if (!st->retrying)
{
st->self->parent->db->set(kv_direntry_key(st->dir_ino, st->filename),
json11::Json(json11::Json::object{ { "ino", st->ino } }).dump(), [st](int res)
{
st->res = res;
nfs_kv_continue_link(st, 2);
}, [st](int res, const std::string & old_value)
{
return res == -ENOENT;
});
return;
resume_2:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
}
// Increase inode refcount
{
auto new_ientry = st->ientry.object_items();
auto nlink = new_ientry["nlink"].uint64_value();
new_ientry["nlink"] = nlink ? nlink+1 : 2;
st->ientry = new_ientry;
}
st->self->parent->db->set(kv_inode_key(st->ino), st->ientry.dump(), [st](int res)
{
st->res = res;
nfs_kv_continue_link(st, 3);
}, [st](int res, const std::string & old_value)
{
st->res2 = res;
return res == 0 && old_value == st->ientry_text;
});
return;
resume_3:
if (st->res2 == -ENOENT)
{
st->res = -ENOENT;
}
if (st->res == -EAGAIN)
{
// Re-read inode and retry
st->retrying = true;
goto resume_0;
}
if (st->res < 0)
{
// Maybe inode was deleted in the meantime, delete our direntry
st->self->parent->db->del(kv_direntry_key(st->dir_ino, st->filename), [st](int res)
{
st->res2 = res;
nfs_kv_continue_link(st, 4);
});
return;
resume_4:
if (st->res2 < 0)
{
fprintf(stderr, "Warning: failed to delete new linked direntry %lu/%s: %s (code %d)\n",
st->dir_ino, st->filename.c_str(), strerror(-st->res2), st->res2);
}
}
auto cb = std::move(st->cb);
cb(st->res);
}
// FIXME: We'll need some tests for the FS
int kv_nfs3_link_proc(void *opaque, rpc_op_t *rop)
{
auto st = new nfs_kv_link_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
LINK3args *args = (LINK3args*)rop->request;
st->ino = kv_fh_inode(args->file);
st->dir_ino = kv_fh_inode(args->link.dir);
st->filename = args->link.name;
if (st->self->parent->trace)
fprintf(stderr, "[%d] LINK %lu -> %lu/%s\n", st->self->nfs_fd, st->ino, st->dir_ino, st->filename.c_str());
if (!st->ino || !st->dir_ino || st->filename == "")
{
LINK3res *reply = (LINK3res*)rop->reply;
*reply = (LINK3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
st->cb = [st](int res)
{
LINK3res *reply = (LINK3res*)st->rop->reply;
if (res < 0)
{
*reply = (LINK3res){ .status = vitastor_nfs_map_err(res) };
}
else
{
*reply = (LINK3res){
.status = NFS3_OK,
.resok = (LINK3resok){
.file_attributes = (post_op_attr){
.attributes_follow = 1,
.attributes = get_kv_attributes(st->self, st->ino, st->ientry),
},
},
};
}
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_link(st, 0);
return 1;
}

104
src/nfs_kv_lookup.cpp Normal file
View File

@ -0,0 +1,104 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - LOOKUP, READLINK
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
int kv_nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
LOOKUP3args *args = (LOOKUP3args*)rop->request;
LOOKUP3res *reply = (LOOKUP3res*)rop->reply;
inode_t dir_ino = kv_fh_inode(args->what.dir);
std::string filename = args->what.name;
if (self->parent->trace)
fprintf(stderr, "[%d] LOOKUP %lu/%s\n", self->nfs_fd, dir_ino, filename.c_str());
if (!dir_ino || filename == "")
{
*reply = (LOOKUP3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
return 0;
}
self->parent->db->get(kv_direntry_key(dir_ino, filename), [=](int res, const std::string & value)
{
if (res < 0)
{
*reply = (LOOKUP3res){ .status = vitastor_nfs_map_err(-res) };
rpc_queue_reply(rop);
return;
}
std::string err;
auto direntry = json11::Json::parse(value, err);
if (err != "")
{
fprintf(stderr, "Invalid JSON in direntry %s = %s: %s\n", kv_direntry_key(dir_ino, filename).c_str(), value.c_str(), err.c_str());
*reply = (LOOKUP3res){ .status = NFS3ERR_IO };
rpc_queue_reply(rop);
return;
}
uint64_t ino = direntry["ino"].uint64_value();
kv_read_inode(self, ino, [=](int res, const std::string & value, json11::Json ientry)
{
if (res < 0)
{
*reply = (LOOKUP3res){ .status = vitastor_nfs_map_err(res == -ENOENT ? -EIO : res) };
rpc_queue_reply(rop);
return;
}
*reply = (LOOKUP3res){
.status = NFS3_OK,
.resok = (LOOKUP3resok){
.object = xdr_copy_string(rop->xdrs, kv_fh(ino)),
.obj_attributes = {
.attributes_follow = 1,
.attributes = get_kv_attributes(self, ino, ientry),
},
},
};
rpc_queue_reply(rop);
});
});
return 1;
}
int kv_nfs3_readlink_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
READLINK3args *args = (READLINK3args*)rop->request;
if (self->parent->trace)
fprintf(stderr, "[%d] READLINK %lu\n", self->nfs_fd, kv_fh_inode(args->symlink));
READLINK3res *reply = (READLINK3res*)rop->reply;
if (!kv_fh_valid(args->symlink) || args->symlink == NFS_ROOT_HANDLE)
{
// Invalid filehandle or trying to read symlink from root entry
*reply = (READLINK3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
return 0;
}
kv_read_inode(self, kv_fh_inode(args->symlink), [=](int res, const std::string & value, json11::Json attrs)
{
if (res < 0)
{
*reply = (READLINK3res){ .status = vitastor_nfs_map_err(-res) };
}
else if (attrs["type"] != "link")
{
*reply = (READLINK3res){ .status = NFS3ERR_INVAL };
}
else
{
*reply = (READLINK3res){
.status = NFS3_OK,
.resok = (READLINK3resok){
.data = xdr_copy_string(rop->xdrs, attrs["symlink"].string_value()),
},
};
}
rpc_queue_reply(rop);
});
return 1;
}

162
src/nfs_kv_read.cpp Normal file
View File

@ -0,0 +1,162 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - READ
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
struct nfs_kv_read_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
bool allow_cache = true;
inode_t ino = 0;
uint64_t offset = 0, size = 0;
std::function<void(int)> cb;
// state
int res = 0;
json11::Json ientry;
uint64_t aligned_size = 0, aligned_offset = 0;
uint8_t *aligned_buf = NULL;
cluster_op_t *op = NULL;
uint8_t *buf = NULL;
};
static void nfs_kv_continue_read(nfs_kv_read_state *st, int state)
{
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_read()");
abort();
}
if (st->offset + sizeof(shared_file_header_t) < st->self->parent->shared_inode_threshold)
{
kv_read_inode(st->self, st->ino, [st](int res, const std::string & value, json11::Json attrs)
{
st->res = res;
st->ientry = attrs;
nfs_kv_continue_read(st, 1);
}, st->allow_cache);
return;
resume_1:
if (st->res < 0 || kv_map_type(st->ientry["type"].string_value()) != NF3REG)
{
auto cb = std::move(st->cb);
cb(st->res < 0 ? st->res : -EINVAL);
return;
}
if (st->ientry["shared_ino"].uint64_value() != 0)
{
st->aligned_size = align_shared_size(st->self, st->offset+st->size);
st->aligned_buf = (uint8_t*)malloc_or_die(st->aligned_size);
st->buf = st->aligned_buf + sizeof(shared_file_header_t) + st->offset;
st->op = new cluster_op_t;
st->op->opcode = OSD_OP_READ;
st->op->inode = st->self->parent->fs_base_inode + st->ientry["shared_ino"].uint64_value();
st->op->offset = st->ientry["shared_offset"].uint64_value();
if (st->offset+st->size > st->ientry["size"].uint64_value())
{
st->op->len = align_shared_size(st->self, st->ientry["size"].uint64_value());
memset(st->aligned_buf+st->op->len, 0, st->aligned_size-st->op->len);
}
else
st->op->len = st->aligned_size;
st->op->iov.push_back(st->aligned_buf, st->op->len);
st->op->callback = [st, state](cluster_op_t *op)
{
st->res = op->retval == op->len ? 0 : op->retval;
delete op;
nfs_kv_continue_read(st, 2);
};
st->self->parent->cli->execute(st->op);
return;
resume_2:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
auto hdr = ((shared_file_header_t*)st->aligned_buf);
if (hdr->magic != SHARED_FILE_MAGIC_V1 || hdr->inode != st->ino ||
align_shared_size(st->self, hdr->size) > align_shared_size(st->self, st->ientry["size"].uint64_value()))
{
// Got unrelated data - retry from the beginning
free(st->aligned_buf);
st->aligned_buf = NULL;
st->allow_cache = false;
nfs_kv_continue_read(st, 0);
return;
}
auto cb = std::move(st->cb);
cb(0);
return;
}
}
st->aligned_offset = (st->offset & ~(st->self->parent->pool_alignment-1));
st->aligned_size = ((st->offset + st->size + st->self->parent->pool_alignment) &
~(st->self->parent->pool_alignment-1)) - st->aligned_offset;
st->aligned_buf = (uint8_t*)malloc_or_die(st->aligned_size);
st->buf = st->aligned_buf + st->offset - st->aligned_offset;
st->op = new cluster_op_t;
st->op->opcode = OSD_OP_READ;
st->op->inode = st->self->parent->fs_base_inode + st->ino;
st->op->offset = st->aligned_offset;
st->op->len = st->aligned_size;
st->op->iov.push_back(st->aligned_buf, st->aligned_size);
st->op->callback = [st](cluster_op_t *op)
{
st->res = op->retval;
delete op;
nfs_kv_continue_read(st, 3);
};
st->self->parent->cli->execute(st->op);
return;
resume_3:
auto cb = std::move(st->cb);
cb(st->res);
return;
}
int kv_nfs3_read_proc(void *opaque, rpc_op_t *rop)
{
READ3args *args = (READ3args*)rop->request;
READ3res *reply = (READ3res*)rop->reply;
auto ino = kv_fh_inode(args->file);
if (args->count > MAX_REQUEST_SIZE || !ino)
{
*reply = (READ3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
return 0;
}
auto st = new nfs_kv_read_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
st->ino = ino;
st->offset = args->offset;
st->size = args->count;
st->cb = [st](int res)
{
READ3res *reply = (READ3res*)st->rop->reply;
*reply = (READ3res){ .status = vitastor_nfs_map_err(res) };
if (res == 0)
{
xdr_add_malloc(st->rop->xdrs, st->aligned_buf);
reply->resok.data.data = (char*)st->buf;
reply->resok.data.size = st->size;
reply->resok.count = st->size;
reply->resok.eof = 0;
}
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_read(st, 0);
return 1;
}

371
src/nfs_kv_readdir.cpp Normal file
View File

@ -0,0 +1,371 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - READDIR, READDIRPLUS
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
static unsigned len_pad4(unsigned len)
{
return len + (len&3 ? 4-(len&3) : 0);
}
struct nfs_kv_readdir_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
// Request:
bool is_plus = false;
uint64_t cookie = 0;
uint64_t cookieverf = 0;
uint64_t dir_ino = 0;
uint64_t maxcount = 0;
std::function<void(int)> cb;
// State:
int res = 0;
std::string prefix, start;
void *list_handle;
uint64_t parent_ino = 0;
std::string ientry_text, parent_ientry_text;
json11::Json ientry, parent_ientry;
std::string cur_key, cur_value;
int reply_size = 0;
int to_skip = 0;
uint64_t offset = 0;
int getattr_running = 0, getattr_cur = 0;
// Result:
bool eof = false;
//uint64_t cookieverf = 0; // same field
std::vector<entryplus3> entries;
};
static void nfs_kv_continue_readdir(nfs_kv_readdir_state *st, int state);
static void kv_getattr_next(nfs_kv_readdir_state *st)
{
while (st->is_plus && st->getattr_cur < st->entries.size() && st->getattr_running < st->self->parent->readdir_getattr_parallel)
{
auto idx = st->getattr_cur++;
st->getattr_running++;
kv_read_inode(st->self, st->entries[idx].fileid, [st, idx](int res, const std::string & value, json11::Json ientry)
{
if (res == 0)
{
st->entries[idx].name_attributes = (post_op_attr){
// FIXME: maybe do not read parent attributes and leave them to a GETATTR?
.attributes_follow = 1,
.attributes = get_kv_attributes(st->self, st->entries[idx].fileid, ientry),
};
}
st->getattr_running--;
kv_getattr_next(st);
if (st->getattr_running == 0 && !st->list_handle)
{
nfs_kv_continue_readdir(st, 4);
}
});
}
}
static void nfs_kv_continue_readdir(nfs_kv_readdir_state *st, int state)
{
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else if (state == 4) goto resume_4;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_readdir()");
abort();
}
// Limit results based on maximum reply size
// Sadly we have to calculate reply size by hand
// reply without entries is 4+4+(dir_attributes ? sizeof(fattr3) : 0)+8+4 bytes
st->reply_size = 20;
if (st->reply_size > st->maxcount)
{
// Error, too small max reply size
auto cb = std::move(st->cb);
cb(-NFS3ERR_TOOSMALL);
return;
}
// Add . and ..
if (st->cookie <= 1)
{
kv_read_inode(st->self, st->dir_ino, [st](int res, const std::string & value, json11::Json ientry)
{
st->res = res;
st->ientry_text = value;
st->ientry = ientry;
nfs_kv_continue_readdir(st, 1);
});
return;
resume_1:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
if (st->cookie == 0)
{
auto fh = kv_fh(st->dir_ino);
auto entry_size = 20 + 4/*len_pad4(".")*/ + (st->is_plus ? 8 + 88 + len_pad4(fh.size()) : 0);
if (st->reply_size + entry_size > st->maxcount)
{
auto cb = std::move(st->cb);
cb(-NFS3ERR_TOOSMALL);
return;
}
entryplus3 dot = {};
dot.name = xdr_copy_string(st->rop->xdrs, ".");
dot.fileid = st->dir_ino;
dot.name_attributes = (post_op_attr){
.attributes_follow = 1,
.attributes = get_kv_attributes(st->self, st->dir_ino, st->ientry),
};
dot.name_handle = (post_op_fh3){
.handle_follows = 1,
.handle = xdr_copy_string(st->rop->xdrs, fh),
};
st->entries.push_back(dot);
st->reply_size += entry_size;
}
st->parent_ino = st->ientry["parent_ino"].uint64_value();
if (st->parent_ino)
{
kv_read_inode(st->self, st->ientry["parent_ino"].uint64_value(), [st](int res, const std::string & value, json11::Json ientry)
{
st->res = res;
st->parent_ientry_text = value;
st->parent_ientry = ientry;
nfs_kv_continue_readdir(st, 2);
});
return;
resume_2:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
}
auto fh = kv_fh(st->parent_ino);
auto entry_size = 20 + 4/*len_pad4("..")*/ + (st->is_plus ? 8 + 88 + len_pad4(fh.size()) : 0);
if (st->reply_size + entry_size > st->maxcount)
{
st->eof = false;
auto cb = std::move(st->cb);
cb(0);
return;
}
entryplus3 dotdot = {};
dotdot.name = xdr_copy_string(st->rop->xdrs, "..");
dotdot.fileid = st->dir_ino;
dotdot.name_attributes = (post_op_attr){
// FIXME: maybe do not read parent attributes and leave them to a GETATTR?
.attributes_follow = 1,
.attributes = get_kv_attributes(st->self,
st->parent_ino ? st->parent_ino : st->dir_ino,
st->parent_ino ? st->parent_ientry : st->ientry),
};
dotdot.name_handle = (post_op_fh3){
.handle_follows = 1,
.handle = xdr_copy_string(st->rop->xdrs, fh),
};
st->entries.push_back(dotdot);
st->reply_size += entry_size;
}
st->prefix = kv_direntry_key(st->dir_ino, "");
st->eof = true;
st->start = st->prefix;
if (st->cookie > 1)
{
auto lc_it = st->self->parent->kvfs->list_cookies.find((list_cookie_t){ st->dir_ino, st->cookieverf, st->cookie });
if (lc_it != st->self->parent->kvfs->list_cookies.end())
{
st->start = lc_it->second.key;
st->to_skip = 1;
st->offset = st->cookie;
}
else
{
st->to_skip = st->cookie-2;
st->offset = 2;
st->cookieverf = ((uint64_t)lrand48() | ((uint64_t)lrand48() << 31) | ((uint64_t)lrand48() << 62));
}
}
else
{
st->to_skip = 0;
st->offset = 2;
st->cookieverf = ((uint64_t)lrand48() | ((uint64_t)lrand48() << 31) | ((uint64_t)lrand48() << 62));
}
{
auto lc_it = st->self->parent->kvfs->list_cookies.lower_bound((list_cookie_t){ st->dir_ino, st->cookieverf, 0 });
if (lc_it != st->self->parent->kvfs->list_cookies.end() &&
lc_it->first.dir_ino == st->dir_ino &&
lc_it->first.cookieverf == st->cookieverf &&
lc_it->first.cookie < st->cookie)
{
auto lc_start = lc_it;
while (lc_it != st->self->parent->kvfs->list_cookies.end() && lc_it->first.cookieverf == st->cookieverf)
{
lc_it++;
}
st->self->parent->kvfs->list_cookies.erase(lc_start, lc_it);
}
}
st->getattr_cur = st->entries.size();
st->list_handle = st->self->parent->db->list_start(st->start);
st->self->parent->db->list_next(st->list_handle, [=](int res, const std::string & key, const std::string & value)
{
st->res = res;
st->cur_key = key;
st->cur_value = value;
nfs_kv_continue_readdir(st, 3);
});
return;
while (st->list_handle)
{
st->self->parent->db->list_next(st->list_handle, NULL);
return;
resume_3:
if (st->res == -ENOENT || st->cur_key.size() > st->prefix.size() || st->cur_key.substr(0, st->prefix.size()) != st->prefix)
{
st->self->parent->db->list_close(st->list_handle);
st->list_handle = NULL;
break;
}
if (st->to_skip > 0)
{
st->to_skip--;
continue;
}
std::string err;
auto direntry = json11::Json::parse(st->cur_value, err);
if (err != "")
{
fprintf(stderr, "readdir: direntry %s contains invalid JSON: %s, skipping\n",
st->cur_key.c_str(), st->cur_value.c_str());
continue;
}
auto ino = direntry["ino"].uint64_value();
auto name = kv_direntry_filename(st->cur_key);
auto fh = kv_fh(ino);
// 1 entry3 is (8+4+(filename_len+3)/4*4+8) bytes
// 1 entryplus3 is (8+4+(filename_len+3)/4*4+8
// + 4+(name_attributes ? (sizeof(fattr3) = 84) : 0)
// + 4+(name_handle ? 4+(handle_len+3)/4*4 : 0)) bytes
auto entry_size = 20 + len_pad4(name.size()) + (st->is_plus ? 8 + 88 + len_pad4(fh.size()) : 0);
if (st->reply_size + entry_size > st->maxcount)
{
st->eof = false;
st->self->parent->db->list_close(st->list_handle);
st->list_handle = NULL;
break;
}
st->reply_size += entry_size;
auto idx = st->entries.size();
st->entries.push_back((entryplus3){});
auto entry = &st->entries[idx];
entry->name = xdr_copy_string(st->rop->xdrs, name);
entry->fileid = ino;
entry->cookie = st->offset++;
st->self->parent->kvfs->list_cookies[(list_cookie_t){ st->dir_ino, st->cookieverf, entry->cookie }] = { .key = entry->name };
if (st->is_plus)
{
entry->name_handle = (post_op_fh3){
.handle_follows = 1,
.handle = xdr_copy_string(st->rop->xdrs, fh),
};
kv_getattr_next(st);
}
st->self->parent->db->list_next(st->list_handle, NULL);
}
resume_4:
while (st->getattr_running > 0)
{
return;
}
void *prev = NULL;
for (int i = 0; i < st->entries.size(); i++)
{
entryplus3 *entry = &st->entries[i];
if (prev)
{
if (st->is_plus)
((entryplus3*)prev)->nextentry = entry;
else
((entry3*)prev)->nextentry = (entry3*)entry;
}
prev = entry;
}
// Send reply
auto cb = std::move(st->cb);
cb(0);
}
static void nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
{
auto st = new nfs_kv_readdir_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
st->is_plus = is_plus;
if (st->is_plus)
{
READDIRPLUS3args *args = (READDIRPLUS3args*)rop->request;
st->dir_ino = kv_fh_inode(args->dir);
st->cookie = args->cookie;
st->cookieverf = *((uint64_t*)args->cookieverf);
st->maxcount = args->maxcount;
}
else
{
READDIR3args *args = ((READDIR3args*)rop->request);
st->dir_ino = kv_fh_inode(args->dir);
st->cookie = args->cookie;
st->cookieverf = *((uint64_t*)args->cookieverf);
st->maxcount = args->count;
}
if (st->self->parent->trace)
fprintf(stderr, "[%d] READDIR %lu VERF %lx OFFSET %lu LIMIT %lu\n", st->self->nfs_fd, st->dir_ino, st->cookieverf, st->cookie, st->maxcount);
st->cb = [st](int res)
{
if (st->is_plus)
{
READDIRPLUS3res *reply = (READDIRPLUS3res*)st->rop->reply;
*reply = (READDIRPLUS3res){ .status = vitastor_nfs_map_err(res) };
*(uint64_t*)(reply->resok.cookieverf) = st->cookieverf;
reply->resok.reply.entries = st->entries.size() ? &st->entries[0] : NULL;
reply->resok.reply.eof = st->eof;
}
else
{
READDIR3res *reply = (READDIR3res*)st->rop->reply;
*reply = (READDIR3res){ .status = vitastor_nfs_map_err(res) };
*(uint64_t*)(reply->resok.cookieverf) = st->cookieverf;
reply->resok.reply.entries = st->entries.size() ? (entry3*)&st->entries[0] : NULL;
reply->resok.reply.eof = st->eof;
}
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_readdir(st, 0);
}
int kv_nfs3_readdir_proc(void *opaque, rpc_op_t *rop)
{
nfs3_readdir_common(opaque, rop, false);
return 0;
}
int kv_nfs3_readdirplus_proc(void *opaque, rpc_op_t *rop)
{
nfs3_readdir_common(opaque, rop, true);
return 0;
}

313
src/nfs_kv_remove.cpp Normal file
View File

@ -0,0 +1,313 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - REMOVE, RMDIR
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
#include "cli.h"
struct kv_del_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
uint64_t dir_ino = 0;
std::string filename;
uint64_t ino = 0;
void *list_handle = NULL;
std::string prefix, list_key, direntry_text, ientry_text;
json11::Json direntry, ientry;
int type = 0;
bool is_rmdir = false;
bool rm_data = false;
int res = 0, res2 = 0;
std::function<void(int)> cb;
};
static void nfs_kv_continue_delete(kv_del_state *st, int state)
{
// Overall algorithm:
// 1) Get inode attributes and check that it's not a directory (REMOVE)
// 2) Get inode attributes and check that it is a directory (RMDIR)
// 3) Delete direntry with CAS
// 4) Check that the directory didn't contain files (RMDIR) and restore it if it did
// 5) Reduce inode refcount by 1 or delete inode
// 6) If regular file and inode is deleted: delete data
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else if (state == 4) goto resume_4;
else if (state == 5) goto resume_5;
else if (state == 6) goto resume_6;
else if (state == 7) goto resume_7;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_delete()");
abort();
}
st->self->parent->db->get(kv_direntry_key(st->dir_ino, st->filename), [st](int res, const std::string & value)
{
st->res = res;
st->direntry_text = value;
nfs_kv_continue_delete(st, 1);
});
return;
resume_1:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
{
std::string err;
st->direntry = json11::Json::parse(st->direntry_text, err);
if (err != "")
{
fprintf(stderr, "Invalid JSON in direntry %s = %s: %s, deleting\n",
kv_direntry_key(st->dir_ino, st->filename).c_str(), st->direntry_text.c_str(), err.c_str());
// Just delete direntry and skip inode
}
else
{
st->ino = st->direntry["ino"].uint64_value();
}
}
// Get inode
st->self->parent->db->get(kv_inode_key(st->ino), [st](int res, const std::string & value)
{
st->res = res;
st->ientry_text = value;
nfs_kv_continue_delete(st, 2);
});
return;
resume_2:
if (st->res < 0)
{
fprintf(stderr, "error reading inode %s: %s (code %d)\n",
kv_inode_key(st->ino).c_str(), strerror(-st->res), st->res);
auto cb = std::move(st->cb);
cb(st->res);
return;
}
{
std::string err;
st->ientry = json11::Json::parse(st->ientry_text, err);
if (err != "")
{
fprintf(stderr, "Invalid JSON in inode %s = %s: %s, treating as a regular file\n",
kv_inode_key(st->ino).c_str(), st->ientry_text.c_str(), err.c_str());
}
}
// (1-2) Check type
st->type = kv_map_type(st->ientry["type"].string_value());
if (st->type == -1 || st->is_rmdir != (st->type == NF3DIR))
{
auto cb = std::move(st->cb);
cb(st->is_rmdir ? -ENOTDIR : -EISDIR);
return;
}
// (3) Delete direntry with CAS
st->self->parent->db->del(kv_direntry_key(st->dir_ino, st->filename), [st](int res)
{
st->res = res;
nfs_kv_continue_delete(st, 3);
}, [st](int res, const std::string & value)
{
return value == st->direntry_text;
});
return;
resume_3:
if (st->res == -EAGAIN)
{
// CAS failure, restart from the beginning
nfs_kv_continue_delete(st, 0);
return;
}
else if (st->res < 0 && st->res != -ENOENT)
{
fprintf(stderr, "failed to remove direntry %s: %s (code %d)\n",
kv_direntry_key(st->dir_ino, st->filename).c_str(), strerror(-st->res), st->res);
auto cb = std::move(st->cb);
cb(st->res);
return;
}
if (!st->ino)
{
// direntry contained invalid JSON and was deleted, finish
auto cb = std::move(st->cb);
cb(0);
return;
}
if (st->is_rmdir)
{
// (4) Check if directory actually is not empty
st->list_handle = st->self->parent->db->list_start(kv_direntry_key(st->ino, ""));
st->self->parent->db->list_next(st->list_handle, [st](int res, const std::string & key, const std::string & value)
{
st->res = res;
st->list_key = key;
st->self->parent->db->list_close(st->list_handle);
nfs_kv_continue_delete(st, 4);
});
return;
resume_4:
st->prefix = kv_direntry_key(st->ino, "");
if (st->res == -ENOENT || st->list_key.size() < st->prefix.size() || st->list_key.substr(0, st->prefix.size()) != st->prefix)
{
// OK, directory is empty
}
else
{
// Not OK, restore direntry
st->self->parent->db->del(kv_direntry_key(st->dir_ino, st->filename), [st](int res)
{
st->res2 = res;
nfs_kv_continue_delete(st, 5);
}, [st](int res, const std::string & value)
{
return res == -ENOENT;
});
return;
resume_5:
if (st->res2 < 0)
{
fprintf(stderr, "failed to restore direntry %s (%s): %s (code %d)",
kv_direntry_key(st->dir_ino, st->filename).c_str(), st->direntry_text.c_str(), strerror(-st->res2), st->res2);
fprintf(stderr, " - inode %lu may be left as garbage\n", st->ino);
}
if (st->res < 0)
{
fprintf(stderr, "failed to list entries from %s: %s (code %d)\n",
kv_direntry_key(st->ino, "").c_str(), strerror(-st->res), st->res);
}
auto cb = std::move(st->cb);
cb(st->res < 0 ? st->res : -ENOTEMPTY);
return;
}
}
// (5) Reduce inode refcount by 1 or delete inode
if (st->ientry["nlink"].uint64_value() > 1)
{
auto copy = st->ientry.object_items();
copy["nlink"] = st->ientry["nlink"].uint64_value()-1;
st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(copy).dump(), [st](int res)
{
st->res = res;
nfs_kv_continue_delete(st, 6);
}, [st](int res, const std::string & old_value)
{
return old_value == st->ientry_text;
});
}
else
{
st->self->parent->db->del(kv_inode_key(st->ino), [st](int res)
{
st->res = res;
nfs_kv_continue_delete(st, 6);
}, [st](int res, const std::string & old_value)
{
return old_value == st->ientry_text;
});
}
return;
resume_6:
if (st->res < 0)
{
// Assume EAGAIN is OK, maybe someone created a hard link in the meantime
auto cb = std::move(st->cb);
cb(st->res == -EAGAIN ? 0 : st->res);
return;
}
// (6) If regular file and inode is deleted: delete data
if ((!st->type || st->type == NF3REG) && st->ientry["nlink"].uint64_value() <= 1 &&
!st->ientry["shared_inode"].uint64_value())
{
// Remove data
st->self->parent->cmd->loop_and_wait(st->self->parent->cmd->start_rm_data(json11::Json::object {
{ "inode", INODE_NO_POOL(st->self->parent->fs_base_inode + st->ino) },
{ "pool", (uint64_t)INODE_POOL(st->self->parent->fs_base_inode + st->ino) },
}), [st](const cli_result_t & r)
{
if (r.err)
{
fprintf(stderr, "Failed to remove inode %lx data: %s (code %d)\n",
st->ino, r.text.c_str(), r.err);
}
st->res = r.err;
nfs_kv_continue_delete(st, 7);
});
return;
resume_7:
auto cb = std::move(st->cb);
cb(st->res);
return;
}
auto cb = std::move(st->cb);
cb(0);
}
int kv_nfs3_remove_proc(void *opaque, rpc_op_t *rop)
{
kv_del_state *st = new kv_del_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
REMOVE3res *reply = (REMOVE3res*)rop->reply;
REMOVE3args *args = (REMOVE3args*)rop->request;
st->dir_ino = kv_fh_inode(args->object.dir);
st->filename = args->object.name;
if (st->self->parent->trace)
fprintf(stderr, "[%d] REMOVE %lu/%s\n", st->self->nfs_fd, st->dir_ino, st->filename.c_str());
if (!st->dir_ino)
{
*reply = (REMOVE3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
st->cb = [st](int res)
{
*((REMOVE3res*)st->rop->reply) = (REMOVE3res){
.status = vitastor_nfs_map_err(res),
};
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_delete(st, 0);
return 1;
}
int kv_nfs3_rmdir_proc(void *opaque, rpc_op_t *rop)
{
kv_del_state *st = new kv_del_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
RMDIR3args *args = (RMDIR3args*)rop->request;
RMDIR3res *reply = (RMDIR3res*)rop->reply;
st->dir_ino = kv_fh_inode(args->object.dir);
st->filename = args->object.name;
st->is_rmdir = true;
if (st->self->parent->trace)
fprintf(stderr, "[%d] RMDIR %lu/%s\n", st->self->nfs_fd, st->dir_ino, st->filename.c_str());
if (!st->dir_ino)
{
*reply = (RMDIR3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
st->cb = [st](int res)
{
*((RMDIR3res*)st->rop->reply) = (RMDIR3res){
.status = vitastor_nfs_map_err(res),
};
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_delete(st, 0);
return 1;
}

206
src/nfs_kv_rename.cpp Normal file
View File

@ -0,0 +1,206 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - RENAME
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
struct nfs_kv_rename_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
uint64_t old_dir_ino = 0, new_dir_ino = 0;
std::string old_name, new_name;
std::string old_direntry_text;
std::string ientry_text;
json11::Json direntry, ientry;
int res = 0, res2 = 0;
std::function<void(int)> cb;
};
static void nfs_kv_continue_rename(nfs_kv_rename_state *st, int state)
{
// Simplified algorithm (non-atomic and without ENOTDIR/EISDIR):
// 1) Check if the target directory exists
// 2) Delete & save (using CAS) the source direntry
// 3) Write the target direntry using CAS, fail if it already exists
// 4) Restore the source direntry on failure
// Atomic version would require something like a journal
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else if (state == 4) goto resume_4;
else if (state == 5) goto resume_5;
else if (state == 6) goto resume_6;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_rename()");
abort();
}
kv_read_inode(st->self, st->new_dir_ino, [st](int res, const std::string & value, json11::Json attrs)
{
st->res = res == 0 ? (attrs["type"].string_value() == "dir" ? 0 : -ENOTDIR) : res;
nfs_kv_continue_rename(st, 1);
});
return;
resume_1:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
// Read & delete the old direntry
st->self->parent->db->del(kv_direntry_key(st->old_dir_ino, st->old_name), [st](int res)
{
st->res = res;
nfs_kv_continue_rename(st, 2);
}, [=](int res, const std::string & old_value)
{
st->res = res;
st->old_direntry_text = old_value;
return true;
});
return;
resume_2:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
{
std::string err;
st->direntry = json11::Json::parse(st->old_direntry_text, err);
if (err != "")
{
fprintf(stderr, "Invalid JSON in direntry %s = %s: %s\n",
kv_direntry_key(st->old_dir_ino, st->old_name).c_str(),
st->old_direntry_text.c_str(), err.c_str());
}
}
if (st->direntry["type"].string_value() == "dir" &&
st->direntry["ino"].uint64_value() != 0 &&
st->new_dir_ino != st->old_dir_ino)
{
// Read & check inode
kv_read_inode(st->self, st->direntry["ino"].uint64_value(), [st](int res, const std::string & value, json11::Json ientry)
{
st->res = res;
st->ientry_text = value;
st->ientry = ientry;
nfs_kv_continue_rename(st, 3);
});
return;
resume_3:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
// Change parent reference
{
auto ientry_new = st->ientry.object_items();
ientry_new["parent_ino"] = st->new_dir_ino;
st->self->parent->db->set(kv_inode_key(st->direntry["ino"].uint64_value()), json11::Json(ientry_new).dump(), [st](int res)
{
st->res = res;
nfs_kv_continue_rename(st, 4);
}, [st](int res, const std::string & old_value)
{
return old_value == st->ientry_text;
});
}
return;
resume_4:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
}
st->self->parent->db->set(kv_direntry_key(st->new_dir_ino, st->new_name), st->old_direntry_text, [st](int res)
{
st->res = res;
nfs_kv_continue_rename(st, 5);
}, [st](int res, const std::string & old_value)
{
return res == -ENOENT;
});
return;
resume_5:
if (st->res < 0)
{
if (st->res == -EAGAIN)
st->res = -EEXIST;
st->res2 = st->res;
st->self->parent->db->set(kv_direntry_key(st->old_dir_ino, st->old_name), st->old_direntry_text, [st](int res)
{
st->res = res;
nfs_kv_continue_rename(st, 6);
}, [st](int res, const std::string & old_value)
{
return res == -ENOENT;
});
return;
resume_6:
if (st->res < 0)
{
if (st->res == -EAGAIN)
st->res = -EEXIST;
fprintf(stderr, "error restoring %s = %s after failed rename: %s (code %d)\n",
kv_direntry_key(st->old_dir_ino, st->old_name).c_str(), st->old_direntry_text.c_str(),
strerror(-st->res), st->res);
}
auto cb = std::move(st->cb);
cb(st->res2);
return;
}
auto cb = std::move(st->cb);
cb(st->res);
}
int kv_nfs3_rename_proc(void *opaque, rpc_op_t *rop)
{
auto st = new nfs_kv_rename_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
RENAME3args *args = (RENAME3args*)rop->request;
st->old_dir_ino = kv_fh_inode(args->from.dir);
st->new_dir_ino = kv_fh_inode(args->to.dir);
st->old_name = args->from.name;
st->new_name = args->to.name;
if (st->self->parent->trace)
fprintf(stderr, "[%d] RENAME %lu/%s -> %lu/%s\n", st->self->nfs_fd, st->old_dir_ino, st->old_name.c_str(), st->new_dir_ino, st->new_name.c_str());
if (!st->old_dir_ino || !st->new_dir_ino || st->old_name == "" || st->new_name == "")
{
RENAME3res *reply = (RENAME3res*)rop->reply;
*reply = (RENAME3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
if (st->old_dir_ino == st->new_dir_ino && st->old_name == st->new_name)
{
RENAME3res *reply = (RENAME3res*)rop->reply;
*reply = (RENAME3res){ .status = NFS3_OK };
rpc_queue_reply(st->rop);
delete st;
return 0;
}
st->cb = [st](int res)
{
RENAME3res *reply = (RENAME3res*)st->rop->reply;
*reply = (RENAME3res){ .status = vitastor_nfs_map_err(res) };
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_rename(st, 0);
return 1;
}

183
src/nfs_kv_setattr.cpp Normal file
View File

@ -0,0 +1,183 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - SETATTR
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
#include "cli.h"
struct nfs_kv_setattr_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
uint64_t ino = 0;
uint64_t old_size = 0, new_size = 0;
json11::Json::object set_attrs;
int res = 0, cas_res = 0;
std::string ientry_text;
json11::Json ientry;
json11::Json::object new_attrs;
std::function<void(int)> cb;
};
static void nfs_kv_continue_setattr(nfs_kv_setattr_state *st, int state)
{
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_setattr()");
abort();
}
resume_0:
kv_read_inode(st->self, st->ino, [st](int res, const std::string & value, json11::Json attrs)
{
st->res = res;
st->ientry_text = value;
st->ientry = attrs;
nfs_kv_continue_setattr(st, 1);
});
return;
resume_1:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
if (st->ientry["type"].string_value() == "link" ||
st->ientry["type"].string_value() != "file" &&
st->ientry["type"].string_value() != "" &&
!st->set_attrs["size"].is_null())
{
auto cb = std::move(st->cb);
cb(-EINVAL);
return;
}
// Now we can update it
st->new_attrs = st->ientry.object_items();
st->old_size = st->ientry["size"].uint64_value();
for (auto & kv: st->set_attrs)
{
if (kv.first == "size")
{
st->new_size = kv.second.uint64_value();
}
st->new_attrs[kv.first] = kv.second;
}
st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(st->new_attrs).dump(), [st](int res)
{
st->res = res;
nfs_kv_continue_setattr(st, 2);
}, [st](int res, const std::string & cas_value)
{
st->cas_res = res;
return (res == 0 || res == -ENOENT && st->ino == KV_ROOT_INODE) && cas_value == st->ientry_text;
});
return;
resume_2:
if (st->cas_res == -ENOENT)
{
st->res = -ENOENT;
}
if (st->res == -EAGAIN)
{
// Retry
fprintf(stderr, "CAS failure during setattr, retrying\n");
goto resume_0;
}
if (st->res < 0)
{
fprintf(stderr, "Failed to update inode %lu: %s (code %d)\n", st->ino, strerror(-st->res), st->res);
auto cb = std::move(st->cb);
cb(st->res);
return;
}
if (!st->set_attrs["size"].is_null() &&
st->ientry["size"].uint64_value() > st->set_attrs["size"].uint64_value())
{
// Delete extra data when downsizing
st->self->parent->cmd->loop_and_wait(st->self->parent->cmd->start_rm_data(json11::Json::object {
{ "inode", INODE_NO_POOL(st->self->parent->fs_base_inode + st->ino) },
{ "pool", (uint64_t)INODE_POOL(st->self->parent->fs_base_inode + st->ino) },
{ "min_offset", st->set_attrs["size"].uint64_value() },
}), [st](const cli_result_t & r)
{
if (r.err)
{
fprintf(stderr, "Failed to truncate inode %lu: %s (code %d)\n",
st->ino, r.text.c_str(), r.err);
}
st->res = r.err;
nfs_kv_continue_setattr(st, 3);
});
return;
}
resume_3:
auto cb = std::move(st->cb);
cb(0);
}
int kv_nfs3_setattr_proc(void *opaque, rpc_op_t *rop)
{
nfs_kv_setattr_state *st = new nfs_kv_setattr_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
auto args = (SETATTR3args*)rop->request;
auto reply = (SETATTR3res*)rop->reply;
std::string fh = args->object;
if (!kv_fh_valid(fh))
{
*reply = (SETATTR3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
st->ino = kv_fh_inode(fh);
if (args->new_attributes.size.set_it)
st->set_attrs["size"] = args->new_attributes.size.size;
if (args->new_attributes.mode.set_it)
st->set_attrs["mode"] = (uint64_t)args->new_attributes.mode.mode;
if (args->new_attributes.uid.set_it)
st->set_attrs["uid"] = (uint64_t)args->new_attributes.uid.uid;
if (args->new_attributes.gid.set_it)
st->set_attrs["gid"] = (uint64_t)args->new_attributes.gid.gid;
if (args->new_attributes.atime.set_it)
st->set_attrs["atime"] = nfstime_to_str(args->new_attributes.atime.atime);
if (args->new_attributes.mtime.set_it)
st->set_attrs["mtime"] = nfstime_to_str(args->new_attributes.mtime.mtime);
fprintf(stderr, "SETATTR %lu ATTRS %s\n", st->ino, json11::Json(st->set_attrs).dump().c_str());
st->cb = [st](int res)
{
auto reply = (SETATTR3res*)st->rop->reply;
if (res < 0)
{
*reply = (SETATTR3res){
.status = vitastor_nfs_map_err(res),
};
}
else
{
*reply = (SETATTR3res){
.status = NFS3_OK,
.resok = (SETATTR3resok){
.obj_wcc = (wcc_data){
.after = (post_op_attr){
.attributes_follow = 1,
.attributes = get_kv_attributes(st->self, st->ino, st->new_attrs),
},
},
},
};
}
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_setattr(st, 0);
return 1;
}

796
src/nfs_kv_write.cpp Normal file
View File

@ -0,0 +1,796 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy over VitastorKV database - WRITE
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs_kv.h"
struct nfs_rmw_t
{
nfs_kv_write_state *st = NULL;
int continue_state = 0;
uint64_t ino = 0;
uint64_t offset = 0;
uint8_t *buf = NULL;
uint64_t size = 0;
uint8_t *part_buf = NULL;
};
struct nfs_kv_write_state
{
nfs_client_t *self = NULL;
rpc_op_t *rop = NULL;
uint64_t ino = 0;
uint64_t offset = 0, size = 0;
bool stable = false;
uint8_t *buf = NULL;
std::function<void(int res)> cb;
// state
bool allow_cache = true;
int res = 0, res2 = 0;
int waiting = 0;
std::string ientry_text;
json11::Json ientry;
uint64_t new_size = 0;
uint64_t aligned_size = 0;
uint8_t *aligned_buf = NULL;
uint64_t shared_inode = 0, shared_offset = 0;
bool was_immediate = false;
nfs_rmw_t rmw[2];
kv_inode_extend_t *ext = NULL;
~nfs_kv_write_state()
{
if (aligned_buf)
{
free(aligned_buf);
aligned_buf = NULL;
}
}
};
static void nfs_kv_continue_write(nfs_kv_write_state *st, int state);
static void finish_allocate_shared(nfs_client_t *self, int res)
{
std::vector<shared_alloc_queue_t> waiting;
waiting.swap(self->parent->kvfs->allocating_shared);
for (auto & w: waiting)
{
w.st->res = res;
if (res == 0)
{
w.st->shared_inode = self->parent->kvfs->cur_shared_inode;
w.st->shared_offset = self->parent->kvfs->cur_shared_offset;
self->parent->kvfs->cur_shared_offset += (w.size + self->parent->pool_alignment-1) & ~(self->parent->pool_alignment-1);
}
nfs_kv_continue_write(w.st, w.state);
}
}
static void allocate_shared_inode(nfs_kv_write_state *st, int state, uint64_t size)
{
if (st->self->parent->kvfs->cur_shared_inode == 0)
{
st->self->parent->kvfs->allocating_shared.push_back({ st, state, size });
if (st->self->parent->kvfs->allocating_shared.size() > 1)
{
return;
}
allocate_new_id(st->self, [st](int res, uint64_t new_id)
{
if (res < 0)
{
finish_allocate_shared(st->self, res);
return;
}
st->self->parent->kvfs->cur_shared_inode = new_id;
st->self->parent->kvfs->cur_shared_offset = 0;
st->self->parent->db->set(
kv_inode_key(new_id), json11::Json(json11::Json::object{ { "type", "shared" } }).dump(),
[st](int res)
{
if (res < 0)
{
st->self->parent->kvfs->cur_shared_inode = 0;
}
finish_allocate_shared(st->self, res);
},
[](int res, const std::string & old_value)
{
return res == -ENOENT;
}
);
});
}
}
uint64_t align_shared_size(nfs_client_t *self, uint64_t size)
{
return (size + sizeof(shared_file_header_t) + self->parent->pool_alignment-1)
& ~(self->parent->pool_alignment-1);
}
static void nfs_do_write(uint64_t ino, uint64_t offset, uint8_t *buf, uint64_t size, nfs_kv_write_state *st, int state)
{
auto op = new cluster_op_t;
op->opcode = OSD_OP_WRITE;
op->inode = st->self->parent->fs_base_inode + ino;
op->offset = offset;
op->len = size;
op->iov.push_back(buf, size);
st->waiting++;
op->callback = [st, state](cluster_op_t *op)
{
if (op->retval != op->len)
{
st->res = op->retval >= 0 ? -EIO : op->retval;
}
delete op;
st->waiting--;
if (!st->waiting)
{
nfs_kv_continue_write(st, state);
}
};
st->self->parent->cli->execute(op);
}
static void nfs_do_shared_write(nfs_kv_write_state *st, int state)
{
nfs_do_write(st->shared_inode, st->shared_offset, st->aligned_buf, st->aligned_size, st, state);
}
static void nfs_do_unshare_write(nfs_kv_write_state *st, int state)
{
nfs_do_write(st->ino, 0, st->aligned_buf + sizeof(shared_file_header_t),
st->aligned_size - sizeof(shared_file_header_t), st, state);
}
static void nfs_do_rmw(nfs_rmw_t *rmw)
{
auto parent = rmw->st->self->parent;
auto align = parent->pool_alignment;
bool is_begin = (rmw->offset % align);
bool is_end = ((rmw->offset+rmw->size) % align);
// RMW either only at beginning or only at end and within a single block
assert(is_begin != is_end);
assert((rmw->offset/parent->pool_block_size) == ((rmw->offset+rmw->size-1)/parent->pool_block_size));
if (!rmw->part_buf)
{
rmw->part_buf = (uint8_t*)malloc_or_die(align);
}
auto op = new cluster_op_t;
op->opcode = OSD_OP_READ;
op->inode = parent->fs_base_inode + rmw->ino;
op->offset = (rmw->offset + (is_begin ? 0 : rmw->size)) & ~(align-1);
op->len = align;
op->iov.push_back(rmw->part_buf, op->len);
rmw->st->waiting++;
op->callback = [rmw](cluster_op_t *rd_op)
{
if (rd_op->retval != rd_op->len)
{
free(rmw->part_buf);
rmw->part_buf = NULL;
rmw->st->res = rd_op->retval >= 0 ? -EIO : rd_op->retval;
rmw->st->waiting--;
if (!rmw->st->waiting)
{
nfs_kv_continue_write(rmw->st, rmw->continue_state);
}
}
else
{
auto parent = rmw->st->self->parent;
auto align = parent->pool_alignment;
bool is_begin = (rmw->offset % align);
auto op = new cluster_op_t;
op->opcode = OSD_OP_WRITE;
op->inode = rmw->st->self->parent->fs_base_inode + rmw->ino;
op->offset = rmw->offset & ~(align-1);
op->len = (rmw->size + align-1) & ~(align-1);
op->version = rd_op->version+1;
if (is_begin)
{
op->iov.push_back(rmw->part_buf, rmw->offset % align);
}
op->iov.push_back(rmw->buf, rmw->size);
if (!is_begin)
{
auto tail = ((rmw->offset+rmw->size) % align);
op->iov.push_back(rmw->part_buf + tail, align - tail);
}
op->callback = [rmw](cluster_op_t *op)
{
if (op->retval == -EAGAIN)
{
// CAS failure - retry
rmw->st->waiting--;
nfs_do_rmw(rmw);
}
else
{
free(rmw->part_buf);
rmw->part_buf = NULL;
if (op->retval != op->len)
{
rmw->st->res = (op->retval >= 0 ? -EIO : op->retval);
}
rmw->st->waiting--;
if (!rmw->st->waiting)
{
nfs_kv_continue_write(rmw->st, rmw->continue_state);
}
}
delete op;
};
parent->cli->execute(op);
}
delete rd_op;
};
parent->cli->execute(op);
}
static void nfs_do_shared_read(nfs_kv_write_state *st, int state)
{
auto op = new cluster_op_t;
op->opcode = OSD_OP_READ;
op->inode = st->self->parent->fs_base_inode + st->ientry["shared_ino"].uint64_value();
op->offset = st->ientry["shared_offset"].uint64_value();
op->len = align_shared_size(st->self, st->ientry["size"].uint64_value());
op->iov.push_back(st->aligned_buf, op->len);
op->callback = [st, state](cluster_op_t *op)
{
st->res = op->retval == op->len ? 0 : op->retval;
delete op;
nfs_kv_continue_write(st, state);
};
st->self->parent->cli->execute(op);
}
static void nfs_do_fsync(nfs_kv_write_state *st, int state)
{
// Client requested a stable write. Add an fsync
auto op = new cluster_op_t;
op->opcode = OSD_OP_SYNC;
op->callback = [st, state](cluster_op_t *op)
{
delete op;
nfs_kv_continue_write(st, state);
};
st->self->parent->cli->execute(op);
}
static bool nfs_do_shared_readmodify(nfs_kv_write_state *st, int base_state, int state, bool unshare)
{
assert(state <= base_state);
if (state < base_state) {}
else if (state == base_state) goto resume_0;
assert(!st->aligned_buf);
st->aligned_size = unshare
? sizeof(shared_file_header_t) + (st->new_size + st->self->parent->pool_alignment-1) & ~(st->self->parent->pool_alignment-1)
: align_shared_size(st->self, st->new_size);
st->aligned_buf = (uint8_t*)malloc_or_die(st->aligned_size);
memset(st->aligned_buf + sizeof(shared_file_header_t), 0, st->offset);
if (st->ientry["shared_ino"].uint64_value() != 0)
{
// Read old data if shared non-empty
nfs_do_shared_read(st, base_state);
return false;
resume_0:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return true;
}
auto hdr = ((shared_file_header_t*)st->aligned_buf);
if (hdr->magic != SHARED_FILE_MAGIC_V1 || hdr->inode != st->ino ||
align_shared_size(st->self, hdr->size) > align_shared_size(st->self, st->ientry["size"].uint64_value()))
{
// Got unrelated data - retry from the beginning
st->allow_cache = false;
nfs_kv_continue_write(st, 0);
return false;
}
}
*((shared_file_header_t*)st->aligned_buf) = {
.magic = SHARED_FILE_MAGIC_V1,
.inode = st->ino,
.size = st->new_size,
};
memcpy(st->aligned_buf + sizeof(shared_file_header_t) + st->offset, st->buf, st->size);
memset(st->aligned_buf + sizeof(shared_file_header_t) + st->offset + st->size, 0,
st->aligned_size - sizeof(shared_file_header_t) - st->offset - st->size);
return true;
}
static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t offset, int state)
{
auto alignment = st->self->parent->pool_alignment;
uint8_t *good_buf = st->buf;
uint64_t good_offset = offset;
uint64_t good_size = st->size;
st->waiting++;
if (offset % alignment)
{
// Requires read-modify-write in the beginning
auto s = (alignment - (offset % alignment));
if (good_size > s)
{
good_buf += s;
good_offset += s;
good_size -= s;
}
else
good_size = 0;
s = s > st->size ? st->size : s;
st->rmw[0] = {
.st = st,
.continue_state = state,
.ino = ino,
.offset = offset,
.buf = st->buf,
.size = s,
};
nfs_do_rmw(&st->rmw[0]);
}
if ((offset+st->size-1) % alignment)
{
// Requires read-modify-write in the end
auto s = ((offset+st->size-1) % alignment);
if (good_size > s)
good_size -= s;
else
good_size = 0;
if (((offset+st->size-1) / alignment) > (offset / alignment))
{
st->rmw[1] = {
.st = st,
.continue_state = state,
.ino = ino,
.offset = offset + st->size-s,
.buf = st->buf + st->size-s,
.size = s,
};
nfs_do_rmw(&st->rmw[1]);
}
}
if (good_size > 0)
{
// Normal write
nfs_do_write(ino, good_offset, good_buf, good_size, st, state);
}
st->waiting--;
if (!st->waiting)
{
nfs_kv_continue_write(st, state);
}
}
static std::string new_normal_ientry(nfs_kv_write_state *st)
{
auto ni = st->ientry.object_items();
ni.erase("empty");
ni.erase("shared_ino");
ni.erase("shared_offset");
ni.erase("shared_alloc");
ni.erase("shared_ver");
ni["size"] = st->ext->cur_extend;
return json11::Json(ni).dump();
}
static std::string new_moved_ientry(nfs_kv_write_state *st)
{
auto ni = st->ientry.object_items();
ni.erase("empty");
ni["shared_ino"] = st->shared_inode;
ni["shared_offset"] = st->shared_offset;
ni["shared_alloc"] = st->aligned_size;
ni.erase("shared_ver");
ni["size"] = st->new_size;
return json11::Json(ni).dump();
}
static std::string new_shared_ientry(nfs_kv_write_state *st)
{
auto ni = st->ientry.object_items();
ni.erase("empty");
ni["size"] = st->new_size;
ni["shared_ver"] = ni["shared_ver"].uint64_value()+1;
return json11::Json(ni).dump();
}
static void nfs_kv_extend_inode(nfs_kv_write_state *st, int state)
{
if (state == 1)
{
goto resume_1;
}
st->ext->cur_extend = st->ext->next_extend;
st->ext->next_extend = 0;
st->res2 = -EAGAIN;
st->self->parent->db->set(kv_inode_key(st->ino), new_normal_ientry(st), [st](int res)
{
st->res = res;
nfs_kv_continue_write(st, 13);
}, [st](int res, const std::string & old_value)
{
if (res != 0)
{
return false;
}
if (old_value == st->ientry_text)
{
return true;
}
std::string err;
auto ientry = json11::Json::parse(old_value, err).object_items();
if (err != "")
{
st->res2 = -EINVAL;
return false;
}
else if (ientry.size() == st->ientry.object_items().size())
{
for (auto & kv: st->ientry.object_items())
{
if (kv.first != "size" && ientry[kv.first] != kv.second)
{
// Something except size changed
return false;
}
}
// OK, only size changed
if (ientry["size"] >= st->new_size)
{
// Already extended
st->res2 = 0;
return false;
}
// size is different but can still be extended, other parameters don't differ
return true;
}
return false;
});
return;
resume_1:
if (st->res == -EAGAIN)
{
// EAGAIN may be OK in fact (see above)
st->res = st->res2;
}
if (st->res == 0)
{
st->ext->done_extend = st->ext->cur_extend;
}
st->ext->cur_extend = 0;
// Wake up other extenders anyway
auto waiters = std::move(st->ext->waiters);
for (auto & cb: waiters)
{
cb();
}
}
// Packing small files into "shared inodes". Insane algorithm...
// Write:
// - If (offset+size <= threshold):
// - Read inode from cache
// - If inode does not exist - stop with -ENOENT
// - If inode is not a regular file - stop with -EINVAL
// - If it's empty (size == 0 || empty == true):
// - If preset size is larger than threshold:
// - Write data into non-shared inode
// - In parallel: clear empty flag
// - If CAS failure: re-read inode and restart
// - Otherwise:
// - Allocate/take a shared inode
// - Allocate space in its end
// - Write data into shared inode
// - If CAS failure: allocate another shared inode and retry
// - Write shared inode reference, set size
// - If CAS failure: free allocated shared space, re-read inode and restart
// - If it's not empty:
// - If non-shared:
// - Write data into non-shared inode
// - In parallel: check if data fits into inode size and extend if it doesn't
// - If CAS failure: re-read inode and retry to extend the size
// - If shared:
// - Read whole file from shared inode
// - If the file header in data doesn't match: re-read inode and restart
// - If data doesn't fit into the same shared inode:
// - Allocate space in a new shared inode
// - Write data into the new shared inode
// - If CAS failure: allocate another shared inode and retry
// - Update inode metadata (set new size and new shared inode)
// - If CAS failure: free allocated shared space, re-read inode and restart
// - If it fits:
// - Write updated data into the shared inode
// - Update inode entry in any case to block parallel non-shared writes
// - If CAS failure: re-read inode and restart
// - Otherwise:
// - Write data into non-shared inode
// - Read inode in parallel
// - If not a regular file:
// - Remove data
// - Stop with -EINVAL
// - If shared:
// - Read whole file from shared inode
// - Write data into non-shared inode
// - If CAS failure (block should not exist): restart
// - Update inode metadata (make non-shared, update size)
// - If CAS failure: restart
// - Zero out the shared inode header
// - If CAS failure: restart
// - Check if size fits
// - Extend if it doesn't
// Read:
// - If (offset+size <= threshold):
// - Read inode from cache
// - If empty: return zeroes
// - If shared:
// - Read the whole file from shared inode, or at least data and shared inode header
// - If the file header in data doesn't match: re-read inode and restart
// - If non-shared:
// - Read data from non-shared inode
// - Otherwise:
// - Read data from non-shared inode
static void nfs_kv_continue_write(nfs_kv_write_state *st, int state)
{
if (state == 0) {}
else if (state == 1) goto resume_1;
else if (state == 2) goto resume_2;
else if (state == 3) goto resume_3;
else if (state == 4) goto resume_4;
else if (state == 5) goto resume_5;
else if (state == 6) goto resume_6;
else if (state == 7) goto resume_7;
else if (state == 8) goto resume_8;
else if (state == 9) goto resume_9;
else if (state == 10) goto resume_10;
else if (state == 11) goto resume_11;
else if (state == 12) goto resume_12;
else if (state == 13) goto resume_13;
else
{
fprintf(stderr, "BUG: invalid state in nfs_kv_continue_write()");
abort();
}
resume_0:
if (!st->size)
{
auto cb = std::move(st->cb);
cb(0);
return;
}
kv_read_inode(st->self, st->ino, [st](int res, const std::string & value, json11::Json attrs)
{
st->res = res;
st->ientry_text = value;
st->ientry = attrs;
nfs_kv_continue_write(st, 1);
}, st->allow_cache);
return;
resume_1:
if (st->res < 0 ||
st->ientry["type"].uint64_value() != 0 &&
st->ientry["type"].uint64_value() != NF3REG)
{
auto cb = std::move(st->cb);
cb(st->res == 0 ? -EINVAL : st->res);
return;
}
st->was_immediate = st->self->parent->cli->get_immediate_commit(st->self->parent->fs_base_inode + st->ino);
st->new_size = st->ientry["size"].uint64_value();
if (st->new_size < st->offset + st->size)
{
st->new_size = st->offset + st->size;
}
if (st->offset + st->size + sizeof(shared_file_header_t) < st->self->parent->shared_inode_threshold)
{
if (st->ientry["size"].uint64_value() == 0 ||
st->ientry["empty"].bool_value() &&
st->ientry["size"].uint64_value() + sizeof(shared_file_header_t) < st->self->parent->shared_inode_threshold ||
st->ientry["shared_ino"].uint64_value() != 0 &&
st->ientry["size"].uint64_value() < st->offset+st->size &&
st->ientry["shared_alloc"].uint64_value() < align_shared_size(st->self, st->offset+st->size))
{
// Either empty, or shared and requires moving into a larger place (redirect-write)
allocate_shared_inode(st, 2, st->new_size);
return;
resume_2:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
resume_3:
if (!nfs_do_shared_readmodify(st, 3, state, false))
return;
nfs_do_shared_write(st, 4); // FIXME assemble from parts, do not copy?
return;
resume_4:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
st->self->parent->db->set(kv_inode_key(st->ino), new_moved_ientry(st), [st](int res)
{
st->res = res;
nfs_kv_continue_write(st, 5);
}, [st](int res, const std::string & old_value)
{
return res == 0 && old_value == st->ientry_text;
});
return;
resume_5:
if (st->res < 0)
{
st->res2 = st->res;
memset(st->aligned_buf, 0, st->aligned_size);
nfs_do_shared_write(st, 6);
return;
resume_6:
free(st->aligned_buf);
st->aligned_buf = NULL;
if (st->res2 == -EAGAIN)
{
goto resume_0;
}
else
{
auto cb = std::move(st->cb);
cb(st->res2);
return;
}
}
auto cb = std::move(st->cb);
cb(0);
return;
}
else if (st->ientry["shared_ino"].uint64_value() > 0)
{
// Non-empty, shared, can be updated in-place
nfs_do_align_write(st, st->ientry["shared_ino"].uint64_value(),
st->ientry["shared_offset"].uint64_value() + sizeof(shared_file_header_t) + st->offset, 7);
return;
resume_7:
if (st->res == 0 && st->stable && !st->was_immediate)
{
nfs_do_fsync(st, 8);
return;
}
// We always have to change inode entry on shared writes
st->self->parent->db->set(kv_inode_key(st->ino), new_shared_ientry(st), [st](int res)
{
st->res = res;
nfs_kv_continue_write(st, 8);
}, [st](int res, const std::string & old_value)
{
return res == 0 && old_value == st->ientry_text;
});
return;
resume_8:
if (st->res == -EAGAIN)
{
goto resume_0;
}
auto cb = std::move(st->cb);
cb(st->res);
return;
}
// Fall through for non-shared
}
// Non-shared write
if (st->ientry["shared_ino"].uint64_value() != 0)
{
// Unshare
resume_9:
if (!nfs_do_shared_readmodify(st, 9, state, true))
return;
nfs_do_unshare_write(st, 10);
return;
}
else
{
// Just write
nfs_do_align_write(st, st->ino, st->offset, 10);
}
resume_10:
if (st->res == 0 && st->stable && !st->was_immediate)
{
nfs_do_fsync(st, 11);
return;
}
resume_11:
if (st->res < 0)
{
auto cb = std::move(st->cb);
cb(st->res);
return;
}
if (st->ientry["empty"].bool_value() ||
st->ientry["size"].uint64_value() < st->new_size ||
st->ientry["shared_ino"].uint64_value() != 0)
{
st->ext = &st->self->parent->kvfs->extends[st->ino];
st->ext->refcnt++;
resume_12:
if (st->ext->next_extend < st->new_size)
{
// Aggregate inode extension requests
st->ext->next_extend = st->new_size;
}
if (st->ext->cur_extend > 0)
{
// Wait for current extend which is already in progress
st->ext->waiters.push_back([st](){ nfs_kv_continue_write(st, 12); });
return;
}
if (st->ext->done_extend < st->new_size)
{
nfs_kv_extend_inode(st, 0);
return;
resume_13:
nfs_kv_extend_inode(st, 1);
}
st->ext->refcnt--;
assert(st->ext->refcnt >= 0);
if (st->ext->refcnt == 0)
{
st->self->parent->kvfs->extends.erase(st->ino);
}
}
if (st->res == -EAGAIN)
{
// Restart
goto resume_0;
}
auto cb = std::move(st->cb);
cb(st->res);
}
int kv_nfs3_write_proc(void *opaque, rpc_op_t *rop)
{
nfs_kv_write_state *st = new nfs_kv_write_state;
st->self = (nfs_client_t*)opaque;
st->rop = rop;
WRITE3args *args = (WRITE3args*)rop->request;
WRITE3res *reply = (WRITE3res*)rop->reply;
st->ino = kv_fh_inode(args->file);
st->offset = args->offset;
st->size = (args->count > args->data.size ? args->data.size : args->count);
if (!st->ino || st->size > MAX_REQUEST_SIZE)
{
*reply = (WRITE3res){ .status = NFS3ERR_INVAL };
rpc_queue_reply(rop);
delete st;
return 0;
}
st->buf = (uint8_t*)args->data.data;
st->stable = (args->stable != UNSTABLE);
st->cb = [st](int res)
{
WRITE3res *reply = (WRITE3res*)st->rop->reply;
*reply = (WRITE3res){ .status = vitastor_nfs_map_err(res) };
if (res == 0)
{
reply->resok.count = (unsigned)st->size;
reply->resok.committed = st->stable || st->was_immediate ? FILE_SYNC : UNSTABLE;
*(uint64_t*)reply->resok.verf = st->self->parent->server_id;
}
rpc_queue_reply(st->rop);
delete st;
};
nfs_kv_continue_write(st, 0);
return 1;
}

119
src/nfs_mount.cpp Normal file
View File

@ -0,0 +1,119 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
//
// NFS proxy - common NULL, ACCESS, COMMIT, DUMP, EXPORT, MNT, UMNT, UMNTALL
#include <sys/time.h>
#include "nfs_proxy.h"
#include "nfs/nfs.h"
nfsstat3 vitastor_nfs_map_err(int err)
{
return (err == EINVAL ? NFS3ERR_INVAL
: (err == ENOENT ? NFS3ERR_NOENT
: (err == ENOSPC ? NFS3ERR_NOSPC
: (err == EEXIST ? NFS3ERR_EXIST
: (err == EIO ? NFS3ERR_IO : (err ? NFS3ERR_IO : NFS3_OK))))));
}
int nfs3_null_proc(void *opaque, rpc_op_t *rop)
{
rpc_queue_reply(rop);
return 0;
}
int nfs3_access_proc(void *opaque, rpc_op_t *rop)
{
//nfs_client_t *self = (nfs_client_t*)opaque;
ACCESS3args *args = (ACCESS3args*)rop->request;
ACCESS3res *reply = (ACCESS3res*)rop->reply;
*reply = (ACCESS3res){
.status = NFS3_OK,
.resok = (ACCESS3resok){
.access = args->access,
},
};
rpc_queue_reply(rop);
return 0;
}
int nfs3_commit_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
//COMMIT3args *args = (COMMIT3args*)rop->request;
cluster_op_t *op = new cluster_op_t;
// fsync. we don't know how to fsync a single inode, so just fsync everything
op->opcode = OSD_OP_SYNC;
op->callback = [self, rop](cluster_op_t *op)
{
COMMIT3res *reply = (COMMIT3res*)rop->reply;
*reply = (COMMIT3res){ .status = vitastor_nfs_map_err(op->retval) };
*(uint64_t*)reply->resok.verf = self->parent->server_id;
rpc_queue_reply(rop);
};
self->parent->cli->execute(op);
return 1;
}
int mount3_mnt_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
//nfs_dirpath *args = (nfs_dirpath*)rop->request;
if (self->parent->trace)
fprintf(stderr, "[%d] MNT\n", self->nfs_fd);
nfs_mountres3 *reply = (nfs_mountres3*)rop->reply;
u_int flavor = RPC_AUTH_NONE;
reply->fhs_status = MNT3_OK;
reply->mountinfo.fhandle = xdr_copy_string(rop->xdrs, NFS_ROOT_HANDLE);
reply->mountinfo.auth_flavors.auth_flavors_len = 1;
reply->mountinfo.auth_flavors.auth_flavors_val = (u_int*)xdr_copy_string(rop->xdrs, (char*)&flavor, sizeof(u_int)).data;
rpc_queue_reply(rop);
return 0;
}
int mount3_dump_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
if (self->parent->trace)
fprintf(stderr, "[%d] DUMP\n", self->nfs_fd);
nfs_mountlist *reply = (nfs_mountlist*)rop->reply;
*reply = (struct nfs_mountbody*)malloc_or_die(sizeof(struct nfs_mountbody));
xdr_add_malloc(rop->xdrs, *reply);
(*reply)->ml_hostname = xdr_copy_string(rop->xdrs, "127.0.0.1");
(*reply)->ml_directory = xdr_copy_string(rop->xdrs, self->parent->export_root);
(*reply)->ml_next = NULL;
rpc_queue_reply(rop);
return 0;
}
int mount3_umnt_proc(void *opaque, rpc_op_t *rop)
{
//nfs_client_t *self = (nfs_client_t*)opaque;
//nfs_dirpath *arg = (nfs_dirpath*)rop->request;
// do nothing
rpc_queue_reply(rop);
return 0;
}
int mount3_umntall_proc(void *opaque, rpc_op_t *rop)
{
// do nothing
rpc_queue_reply(rop);
return 0;
}
int mount3_export_proc(void *opaque, rpc_op_t *rop)
{
nfs_client_t *self = (nfs_client_t*)opaque;
nfs_exports *reply = (nfs_exports*)rop->reply;
*reply = (struct nfs_exportnode*)calloc_or_die(1, sizeof(struct nfs_exportnode) + sizeof(struct nfs_groupnode));
xdr_add_malloc(rop->xdrs, *reply);
(*reply)->ex_dir = xdr_copy_string(rop->xdrs, self->parent->export_root);
(*reply)->ex_groups = (struct nfs_groupnode*)(reply+1);
(*reply)->ex_groups->gr_name = xdr_copy_string(rop->xdrs, "127.0.0.1");
(*reply)->ex_groups->gr_next = NULL;
(*reply)->ex_next = NULL;
rpc_queue_reply(rop);
return 0;
}

View File

@ -21,6 +21,9 @@
#include "addr_util.h" #include "addr_util.h"
#include "str_util.h" #include "str_util.h"
#include "nfs_proxy.h" #include "nfs_proxy.h"
#include "nfs_kv.h"
#include "nfs_block.h"
#include "nfs_common.h"
#include "http_client.h" #include "http_client.h"
#include "cli.h" #include "cli.h"
@ -31,6 +34,8 @@ const char *exe_name = NULL;
nfs_proxy_t::~nfs_proxy_t() nfs_proxy_t::~nfs_proxy_t()
{ {
if (db)
delete db;
if (cmd) if (cmd)
delete cmd; delete cmd;
if (cli) if (cli)
@ -57,6 +62,7 @@ json11::Json::object nfs_proxy_t::parse_args(int narg, const char *args[])
"\n" "\n"
"USAGE:\n" "USAGE:\n"
" %s [STANDARD OPTIONS] [OTHER OPTIONS]\n" " %s [STANDARD OPTIONS] [OTHER OPTIONS]\n"
" --fs <META> mount VitastorFS with metadata in image <META>\n"
" --subdir <DIR> export images prefixed <DIR>/ (default empty - export all images)\n" " --subdir <DIR> export images prefixed <DIR>/ (default empty - export all images)\n"
" --portmap 0 do not listen on port 111 (portmap/rpcbind, requires root)\n" " --portmap 0 do not listen on port 111 (portmap/rpcbind, requires root)\n"
" --bind <IP> bind service to <IP> address (default 0.0.0.0)\n" " --bind <IP> bind service to <IP> address (default 0.0.0.0)\n"
@ -92,6 +98,7 @@ void nfs_proxy_t::run(json11::Json cfg)
srand48(tv.tv_sec*1000000000 + tv.tv_nsec); srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
server_id = (uint64_t)lrand48() | ((uint64_t)lrand48() << 31) | ((uint64_t)lrand48() << 62); server_id = (uint64_t)lrand48() | ((uint64_t)lrand48() << 31) | ((uint64_t)lrand48() << 62);
// Parse options // Parse options
trace = cfg["log_level"].uint64_value() > 5;
bind_address = cfg["bind"].string_value(); bind_address = cfg["bind"].string_value();
if (bind_address == "") if (bind_address == "")
bind_address = "0.0.0.0"; bind_address = "0.0.0.0";
@ -131,67 +138,12 @@ void nfs_proxy_t::run(json11::Json cfg)
cmd->ringloop = ringloop; cmd->ringloop = ringloop;
cmd->epmgr = epmgr; cmd->epmgr = epmgr;
cmd->cli = cli; cmd->cli = cli;
// We need inode name hashes for NFS handles to remain stateless and <= 64 bytes long
dir_info[""] = (nfs_dir_t){
.id = 1,
.mod_rev = 0,
};
clock_gettime(CLOCK_REALTIME, &dir_info[""].mtime);
watch_stats(); watch_stats();
assert(cli->st_cli.on_inode_change_hook == NULL); if (!fs_kv_inode)
cli->st_cli.on_inode_change_hook = [this](inode_t changed_inode, bool removed)
{ {
auto inode_cfg_it = cli->st_cli.inode_config.find(changed_inode); blockfs = new block_fs_state_t();
if (inode_cfg_it == cli->st_cli.inode_config.end()) blockfs->init(this);
{ }
return;
}
auto & inode_cfg = inode_cfg_it->second;
std::string full_name = inode_cfg.name;
if (name_prefix != "" && full_name.substr(0, name_prefix.size()) != name_prefix)
{
return;
}
// Calculate directory modification time and revision (used as "cookie verifier")
timespec now;
clock_gettime(CLOCK_REALTIME, &now);
dir_info[""].mod_rev = dir_info[""].mod_rev < inode_cfg.mod_revision ? inode_cfg.mod_revision : dir_info[""].mod_rev;
dir_info[""].mtime = now;
int pos = full_name.find('/', name_prefix.size());
while (pos >= 0)
{
std::string dir = full_name.substr(0, pos);
auto & dinf = dir_info[dir];
if (!dinf.id)
dinf.id = next_dir_id++;
dinf.mod_rev = dinf.mod_rev < inode_cfg.mod_revision ? inode_cfg.mod_revision : dinf.mod_rev;
dinf.mtime = now;
dir_by_hash["S"+base64_encode(sha256(dir))] = dir;
pos = full_name.find('/', pos+1);
}
// Alter inode_by_hash
if (removed)
{
auto ino_it = hash_by_inode.find(changed_inode);
if (ino_it != hash_by_inode.end())
{
inode_by_hash.erase(ino_it->second);
hash_by_inode.erase(ino_it);
}
}
else
{
std::string hash = "S"+base64_encode(sha256(full_name));
auto hbi_it = hash_by_inode.find(changed_inode);
if (hbi_it != hash_by_inode.end() && hbi_it->second != hash)
{
// inode had a different name, remove old hash=>inode pointer
inode_by_hash.erase(hbi_it->second);
}
inode_by_hash[hash] = changed_inode;
hash_by_inode[changed_inode] = hash;
}
};
// Load image metadata // Load image metadata
while (!cli->is_ready()) while (!cli->is_ready())
{ {
@ -202,6 +154,54 @@ void nfs_proxy_t::run(json11::Json cfg)
} }
// Check default pool // Check default pool
check_default_pool(); check_default_pool();
// Check if we're using VitastorFS
fs_kv_inode = cfg["fs"].uint64_value();
if (!fs_kv_inode && cfg["fs"].is_string())
{
for (auto & ic: cli->st_cli.inode_config)
{
if (ic.second.name == cfg["fs"].string_value())
{
fs_kv_inode = ic.first;
break;
}
}
}
readdir_getattr_parallel = cfg["readdir_getattr_parallel"].uint64_value();
if (!readdir_getattr_parallel)
readdir_getattr_parallel = 8;
id_alloc_batch_size = cfg["id_alloc_batch_size"].uint64_value();
if (!id_alloc_batch_size)
id_alloc_batch_size = 200;
if (fs_kv_inode)
{
// Open DB and wait
int open_res = 0;
bool open_done = false;
db = new kv_dbw_t(cli);
db->open(fs_kv_inode, cfg, [&](int res)
{
open_done = true;
open_res = res;
});
while (!open_done)
{
ringloop->loop();
if (open_done)
break;
ringloop->wait();
}
if (open_res < 0)
{
fprintf(stderr, "Failed to open key/value filesystem metadata index: %s (code %d)\n",
strerror(-open_res), open_res);
exit(1);
}
fs_base_inode = ((uint64_t)default_pool_id << (64-POOL_ID_BITS));
fs_inode_count = ((uint64_t)1 << (64-POOL_ID_BITS)) - 1;
shared_inode_threshold = pool_block_size;
kvfs = new kv_fs_state_t;
}
// Self-register portmap and NFS // Self-register portmap and NFS
pmap.reg_ports.insert((portmap_id_t){ pmap.reg_ports.insert((portmap_id_t){
.prog = PMAP_PROGRAM, .prog = PMAP_PROGRAM,
@ -275,9 +275,13 @@ void nfs_proxy_t::run(json11::Json cfg)
} }
// Destroy the client // Destroy the client
cli->flush(); cli->flush();
delete kvfs;
delete db;
delete cli; delete cli;
delete epmgr; delete epmgr;
delete ringloop; delete ringloop;
kvfs = NULL;
db = NULL;
cli = NULL; cli = NULL;
epmgr = NULL; epmgr = NULL;
ringloop = NULL; ringloop = NULL;
@ -382,8 +386,11 @@ void nfs_proxy_t::check_default_pool()
{ {
if (cli->st_cli.pool_config.size() == 1) if (cli->st_cli.pool_config.size() == 1)
{ {
default_pool = cli->st_cli.pool_config.begin()->second.name; auto pool_it = cli->st_cli.pool_config.begin();
default_pool_id = cli->st_cli.pool_config.begin()->first; default_pool_id = pool_it->first;
default_pool = pool_it->second.name;
pool_block_size = pool_it->second.pg_stripe_size;
pool_alignment = pool_it->second.bitmap_granularity;
} }
else else
{ {
@ -398,6 +405,8 @@ void nfs_proxy_t::check_default_pool()
if (p.second.name == default_pool) if (p.second.name == default_pool)
{ {
default_pool_id = p.first; default_pool_id = p.first;
pool_block_size = p.second.pg_stripe_size;
pool_alignment = p.second.bitmap_granularity;
break; break;
} }
} }
@ -421,6 +430,10 @@ void nfs_proxy_t::do_accept(int listen_fd)
int one = 1; int one = 1;
setsockopt(nfs_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one)); setsockopt(nfs_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
auto cli = new nfs_client_t(); auto cli = new nfs_client_t();
if (fs_kv_inode)
nfs_kv_procs(cli);
else
nfs_block_procs(cli);
cli->parent = this; cli->parent = this;
cli->nfs_fd = nfs_fd; cli->nfs_fd = nfs_fd;
for (auto & fn: pmap.proc_table) for (auto & fn: pmap.proc_table)

View File

@ -4,17 +4,18 @@
#include "epoll_manager.h" #include "epoll_manager.h"
#include "nfs_portmap.h" #include "nfs_portmap.h"
#include "nfs/xdr_impl.h" #include "nfs/xdr_impl.h"
#include "kv_db.h"
#define NFS_ROOT_HANDLE "R"
#define RPC_INIT_BUF_SIZE 32768 #define RPC_INIT_BUF_SIZE 32768
#define MAX_REQUEST_SIZE 128*1024*1024
#define TRUE 1
#define FALSE 0
class cli_tool_t; class cli_tool_t;
struct nfs_dir_t struct kv_fs_state_t;
{ struct block_fs_state_t;
uint64_t id;
uint64_t mod_rev;
timespec mtime;
};
class nfs_proxy_t class nfs_proxy_t
{ {
@ -27,28 +28,28 @@ public:
std::string export_root; std::string export_root;
bool portmap_enabled; bool portmap_enabled;
unsigned nfs_port; unsigned nfs_port;
uint64_t fs_kv_inode = 0;
uint64_t fs_base_inode = 0;
uint64_t fs_inode_count = 0;
int readdir_getattr_parallel = 8, id_alloc_batch_size = 200;
int trace = 0;
pool_id_t default_pool_id; pool_id_t default_pool_id;
uint64_t pool_block_size = 0;
uint64_t pool_alignment = 0;
uint64_t shared_inode_threshold = 0;
portmap_service_t pmap; portmap_service_t pmap;
ring_loop_t *ringloop = NULL; ring_loop_t *ringloop = NULL;
epoll_manager_t *epmgr = NULL; epoll_manager_t *epmgr = NULL;
cluster_client_t *cli = NULL; cluster_client_t *cli = NULL;
cli_tool_t *cmd = NULL; cli_tool_t *cmd = NULL;
kv_dbw_t *db = NULL;
kv_fs_state_t *kvfs = NULL;
block_fs_state_t *blockfs = NULL;
std::vector<XDR*> xdr_pool; std::vector<XDR*> xdr_pool;
// filehandle = "S"+base64(sha256(full name with prefix)) or "roothandle" for mount root)
uint64_t next_dir_id = 2;
// filehandle => dir with name_prefix
std::map<std::string, std::string> dir_by_hash;
// dir with name_prefix => dir info
std::map<std::string, nfs_dir_t> dir_info;
// filehandle => inode ID
std::map<std::string, inode_t> inode_by_hash;
// inode ID => filehandle
std::map<inode_t, std::string> hash_by_inode;
// inode ID => statistics // inode ID => statistics
std::map<inode_t, json11::Json> inode_stats; std::map<inode_t, json11::Json> inode_stats;
// pool ID => statistics // pool ID => statistics
@ -86,28 +87,6 @@ struct rpc_free_buffer_t
unsigned size; unsigned size;
}; };
struct extend_size_t
{
inode_t inode;
uint64_t new_size;
};
inline bool operator < (const extend_size_t &a, const extend_size_t &b)
{
return a.inode < b.inode || a.inode == b.inode && a.new_size < b.new_size;
}
struct extend_write_t
{
rpc_op_t *rop;
int resize_res, write_res; // 1 = started, 0 = completed OK, -errno = completed with error
};
struct extend_inode_t
{
uint64_t cur_extend = 0, next_extend = 0;
};
class nfs_client_t class nfs_client_t
{ {
public: public:
@ -122,8 +101,6 @@ public:
rpc_cur_buffer_t cur_buffer = { 0 }; rpc_cur_buffer_t cur_buffer = { 0 };
std::map<uint8_t*, rpc_used_buffer_t> used_buffers; std::map<uint8_t*, rpc_used_buffer_t> used_buffers;
std::vector<rpc_free_buffer_t> free_buffers; std::vector<rpc_free_buffer_t> free_buffers;
std::map<inode_t, extend_inode_t> extends;
std::multimap<extend_size_t, extend_write_t> extend_writes;
iovec read_iov; iovec read_iov;
msghdr read_msg = { 0 }; msghdr read_msg = { 0 };
@ -133,9 +110,6 @@ public:
std::vector<iovec> send_list, next_send_list; std::vector<iovec> send_list, next_send_list;
std::vector<rpc_op_t*> outbox, next_outbox; std::vector<rpc_op_t*> outbox, next_outbox;
nfs_client_t();
~nfs_client_t();
void select_read_buffer(unsigned wanted_size); void select_read_buffer(unsigned wanted_size);
void submit_read(unsigned wanted_size); void submit_read(unsigned wanted_size);
void handle_read(int result); void handle_read(int result);