client: resend only the unsent tail after a short io_uring send #122
@@ -550,8 +550,8 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
|
||||
int one = 1;
|
||||
setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
|
||||
cl->peer_state = PEER_CONNECTED;
|
||||
// EPOLLOUT is only required without io_uring, to continue partially sent messages
|
||||
tfd->set_fd_handler(peer_fd, use_sync_send_recv, [this](int peer_fd, int epoll_events)
|
||||
// EPOLLOUT is registered on demand when a send returns EAGAIN
|
||||
tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
|
||||
{
|
||||
handle_peer_epoll(peer_fd, epoll_events);
|
||||
});
|
||||
@@ -591,7 +591,7 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
|
||||
}
|
||||
if (epoll_events & EPOLLOUT)
|
||||
{
|
||||
// The socket became writable again - continue the interrupted use_sync_send_recv
|
||||
// The socket became writable again - continue the interrupted send
|
||||
// read_requests() above may have stopped the client, so look it up again
|
||||
auto cl_it = clients.find(cl->client_id);
|
||||
if (cl_it == clients.end())
|
||||
@@ -599,11 +599,28 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
|
||||
return;
|
||||
}
|
||||
cl = cl_it->second;
|
||||
cl->write_ready = true;
|
||||
// send_list is only left non-empty when we're waiting for EPOLLOUT
|
||||
if (cl->send_list.size() && cl->peer_state != PEER_RDMA)
|
||||
if (!cl->write_ready)
|
||||
{
|
||||
try_send(cl);
|
||||
cl->write_ready = true;
|
||||
tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
|
||||
{
|
||||
handle_peer_epoll(peer_fd, epoll_events);
|
||||
});
|
||||
}
|
||||
if (cl->peer_state != PEER_RDMA &&
|
||||
(use_sync_send_recv ? cl->send_list.size() > 0 :
|
||||
(cl->send_plain_tail && !cl->write_msg.msg_iovlen ||
|
||||
cl->write_op || cl->write_ops.size() || cl->hs && cl->hs->out_size())))
|
||||
{
|
||||
if (use_sync_send_recv)
|
||||
try_send(cl);
|
||||
else if (cl->write_state == 0)
|
||||
{
|
||||
cl->write_state = CL_WRITE_READY;
|
||||
write_ready_clients.push_back(cl->client_id);
|
||||
if (ringloop)
|
||||
ringloop->wakeup();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -841,8 +858,8 @@ void osd_messenger_t::accept_connections(int listen_fd)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// Add FD to epoll. EPOLLOUT is only required without io_uring
|
||||
tfd->set_fd_handler(peer_fd, use_sync_send_recv, [this](int peer_fd, int epoll_events)
|
||||
// EPOLLOUT is registered on demand when a send returns EAGAIN
|
||||
tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
|
||||
{
|
||||
handle_peer_epoll(peer_fd, epoll_events);
|
||||
});
|
||||
|
||||
@@ -154,10 +154,19 @@ struct osd_client_t
|
||||
size_t write_op_pos = 0;
|
||||
msghdr write_msg = { 0 };
|
||||
int write_state = 0;
|
||||
// Socket is writable. Only tracked with use_sync_send_recv, io_uring doesn't need it
|
||||
// Socket is writable. Cleared when a send returns EAGAIN; EPOLLOUT sets it again.
|
||||
bool write_ready = true;
|
||||
std::vector<iovec> send_list;
|
||||
size_t send_list_size = 0;
|
||||
// Remainder of a short io_uring send. Resent without zerocopy so the
|
||||
// already accepted prefix can stay on the zerocopy path.
|
||||
bool send_plain_tail = false;
|
||||
// The tail points into send_free[send_tail_from..send_tail_to) of the batch
|
||||
// that was short-written. That range is released only after both the tail
|
||||
// is fully sent and, if it was a zerocopy send, its release notification arrives.
|
||||
bool send_tail_zc_pending = false;
|
||||
uint64_t send_tail_from = 0;
|
||||
uint64_t send_tail_to = 0;
|
||||
uint64_t next_send_free = 0;
|
||||
std::deque<msgr_sendp_t> send_free;
|
||||
op_aes_xts_encrypt_t *xts_enc_ctx = NULL;
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
|
||||
#define GCM_TMP_BUF_SIZE 4096
|
||||
|
||||
// How many times to retry sendmsg() in place before falling back to EPOLLOUT.
|
||||
// Only used with use_sync_send_recv - io_uring waits for the socket by itself
|
||||
// How many times to retry a blocking-style sendmsg() before waiting for EPOLLOUT.
|
||||
// The io_uring path submits once, then waits for EPOLLOUT after EAGAIN.
|
||||
#define MSGR_SEND_RETRIES 4
|
||||
|
||||
class msgr_op_writer_t
|
||||
@@ -508,6 +508,12 @@ restart:
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if (!cl->write_ready)
|
||||
{
|
||||
// Socket buffer is full, the unsent tail is still in send_list.
|
||||
// Do not copy more data into it, just wait for EPOLLOUT
|
||||
return true;
|
||||
}
|
||||
if (!use_sync_send_recv)
|
||||
{
|
||||
if (!ringloop->space_left())
|
||||
@@ -521,14 +527,11 @@ restart:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
else if (!cl->write_ready)
|
||||
{
|
||||
// Socket buffer is full, the unsent tail is still in send_list.
|
||||
// Do not copy more data into it, just wait for EPOLLOUT
|
||||
return true;
|
||||
}
|
||||
// A short send left its tail in send_list. Finish that tail before
|
||||
// appending new ops, and do not zerocopy it (see handle_send).
|
||||
bool resume_plain_tail = cl->send_plain_tail && cl->send_list.size();
|
||||
uint64_t send_free_from = cl->next_send_free + cl->send_free.size();
|
||||
if (cl->hs)
|
||||
if (!resume_plain_tail && cl->hs)
|
||||
{
|
||||
// Send handshake message
|
||||
if (cl->hs->out_size())
|
||||
@@ -546,12 +549,14 @@ restart:
|
||||
goto copy_ops;
|
||||
}
|
||||
}
|
||||
else
|
||||
else if (!resume_plain_tail)
|
||||
{
|
||||
copy_ops:
|
||||
copy_ops_to_with<get_op_writer_t>(cl, NULL, 0);
|
||||
}
|
||||
uint64_t send_free_to = cl->next_send_free + cl->send_free.size();
|
||||
// The tail resend owns no send_free entries, handle_send relies on it
|
||||
assert(!resume_plain_tail || send_free_from == send_free_to);
|
||||
if (cl->io_error)
|
||||
{
|
||||
stop_client(cl->client_id);
|
||||
@@ -588,7 +593,7 @@ copy_ops:
|
||||
{
|
||||
handle_send(data->res, data->prev, data->more, cl, send_free_from, send_free_to);
|
||||
};
|
||||
bool use_zc = has_sendmsg_zc && min_zerocopy_send_size >= 0;
|
||||
bool use_zc = !cl->send_plain_tail && has_sendmsg_zc && min_zerocopy_send_size >= 0;
|
||||
if (use_zc && min_zerocopy_send_size > 0 &&
|
||||
cl->send_list_size/cl->write_msg.msg_iovlen < min_zerocopy_send_size)
|
||||
{
|
||||
@@ -628,6 +633,10 @@ copy_ops:
|
||||
if (cl->send_list.size() && --send_retries <= 0)
|
||||
{
|
||||
cl->write_ready = false;
|
||||
tfd->set_fd_handler(cl->peer_fd, true, [this](int peer_fd, int epoll_events)
|
||||
{
|
||||
handle_peer_epoll(peer_fd, epoll_events);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -771,11 +780,25 @@ bool osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
|
||||
stop_client(cl->client_id);
|
||||
return false;
|
||||
}
|
||||
// EAGAIN queued nothing. EINTR is resent immediately; EAGAIN waits for EPOLLOUT
|
||||
// so a full send buffer cannot spin the ring thread.
|
||||
bool uring_eagain = !use_sync_send_recv && !prev && result == -EAGAIN;
|
||||
bool uring_retry = uring_eagain || (!use_sync_send_recv && !prev && result == -EINTR);
|
||||
if (uring_retry)
|
||||
{
|
||||
result = 0;
|
||||
}
|
||||
if (result >= 0)
|
||||
{
|
||||
if (prev)
|
||||
{
|
||||
// Second (zero-copy release) notification
|
||||
if (cl->send_plain_tail && free_from == cl->send_tail_from && free_to == cl->send_tail_to)
|
||||
{
|
||||
// The unsent tail still points into these buffers, release them after it's sent
|
||||
cl->send_tail_zc_pending = false;
|
||||
return true;
|
||||
}
|
||||
cl->unref_send_free(free_from, free_to);
|
||||
return true;
|
||||
}
|
||||
@@ -783,13 +806,56 @@ bool osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
|
||||
{
|
||||
cl->send_list.clear();
|
||||
cl->send_list_size = 0;
|
||||
if (cl->send_plain_tail)
|
||||
{
|
||||
cl->send_plain_tail = false;
|
||||
// Otherwise the zerocopy notification releases it through the common path
|
||||
if (!cl->send_tail_zc_pending)
|
||||
cl->unref_send_free(cl->send_tail_from, cl->send_tail_to);
|
||||
}
|
||||
}
|
||||
else if (!use_sync_send_recv)
|
||||
{
|
||||
fprintf(stderr, "Client %ju socket write error: expected to send "
|
||||
"%zu bytes with MSG_WAITALL but sent %u. Disconnecting client\n", cl->client_id, cl->send_list_size, result);
|
||||
stop_client(cl->client_id);
|
||||
return false;
|
||||
// io_uring may still return a short write for MSG_WAITALL (depending on the
|
||||
// kernel version or when interrupted), for example with large SEC_LIST replies
|
||||
// filling the socket buffer. Keep the zerocopy prefix and resend only the tail
|
||||
// with a copying sendmsg.
|
||||
if (result <= 0 && !uring_retry)
|
||||
{
|
||||
fprintf(stderr, "Client %ju socket write error: expected to send "
|
||||
"%zu bytes with MSG_WAITALL but sent %u. Disconnecting client\n", cl->client_id, cl->send_list_size, result);
|
||||
stop_client(cl->client_id);
|
||||
return false;
|
||||
}
|
||||
if (!cl->send_plain_tail)
|
||||
{
|
||||
// First short write of the original batch. Repeated short writes of
|
||||
// the tail itself own no send_free entries and must not overwrite it.
|
||||
cl->send_tail_from = free_from;
|
||||
cl->send_tail_to = free_to;
|
||||
cl->send_tail_zc_pending = more;
|
||||
}
|
||||
eat_send_list(cl, result);
|
||||
cl->send_list_size -= result;
|
||||
cl->send_plain_tail = true;
|
||||
if (uring_eagain)
|
||||
{
|
||||
cl->write_ready = false;
|
||||
tfd->set_fd_handler(cl->peer_fd, true, [this](int peer_fd, int epoll_events)
|
||||
{
|
||||
handle_peer_epoll(peer_fd, epoll_events);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
// A positive short write made progress. Submit the tail once;
|
||||
// if that also returns EAGAIN, the branch above waits for EPOLLOUT.
|
||||
if (cl->write_state == 0)
|
||||
{
|
||||
cl->write_state = CL_WRITE_READY;
|
||||
write_ready_clients.push_back(cl->client_id);
|
||||
}
|
||||
ringloop->wakeup();
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@@ -27,6 +27,28 @@ target_link_libraries(stub_uring_osd
|
||||
${RDMACM_LIBRARIES}
|
||||
)
|
||||
|
||||
add_executable(test_msgr_send_reply
|
||||
EXCLUDE_FROM_ALL
|
||||
test_msgr_send_reply.cpp
|
||||
)
|
||||
target_link_libraries(test_msgr_send_reply
|
||||
vitastor_net
|
||||
${LIBURING_LIBRARIES}
|
||||
${IBVERBS_LIBRARIES}
|
||||
${RDMACM_LIBRARIES}
|
||||
)
|
||||
|
||||
# test_msgr_send
|
||||
add_executable(test_msgr_send EXCLUDE_FROM_ALL test_msgr_send.cpp)
|
||||
target_link_libraries(test_msgr_send
|
||||
vitastor_net
|
||||
${LIBURING_LIBRARIES}
|
||||
${IBVERBS_LIBRARIES}
|
||||
${RDMACM_LIBRARIES}
|
||||
)
|
||||
add_dependencies(build_tests test_msgr_send)
|
||||
add_test(NAME test_msgr_send COMMAND test_msgr_send)
|
||||
|
||||
# test_msgr_stop
|
||||
add_executable(test_msgr_stop
|
||||
EXCLUDE_FROM_ALL
|
||||
|
||||
472
src/test/test_msgr_send.cpp
Normal file
472
src/test/test_msgr_send.cpp
Normal file
@@ -0,0 +1,472 @@
|
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// Short io_uring send handling in osd_messenger_t: the unsent tail is resent
|
||||
// without zerocopy and every send_free range is released exactly once, after
|
||||
// both the tail is sent and the zerocopy release notification arrives.
|
||||
|
||||
#undef NDEBUG
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/epoll.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <deque>
|
||||
|
||||
#include "messenger.h"
|
||||
|
||||
#define TEST_DATA_LEN 4096
|
||||
#define TEST_BATCH_LEN (OSD_PACKET_SIZE + TEST_DATA_LEN)
|
||||
|
||||
// Records SQEs and delivers completions by hand, with the same prev/more
|
||||
// semantics as ring_loop_t::loop()
|
||||
class send_ring_t: public ring_loop_i
|
||||
{
|
||||
public:
|
||||
std::deque<io_uring_sqe> sqes;
|
||||
std::deque<ring_data_t> datas;
|
||||
|
||||
void register_consumer(ring_consumer_t *consumer) override {}
|
||||
void unregister_consumer(ring_consumer_t *consumer) override {}
|
||||
int register_eventfd() override { return -1; }
|
||||
io_uring_sqe* get_sqe() override
|
||||
{
|
||||
sqes.emplace_back();
|
||||
datas.emplace_back();
|
||||
io_uring_sqe *sqe = &sqes.back();
|
||||
ring_data_t *d = &datas.back();
|
||||
*sqe = {};
|
||||
d->res = 0;
|
||||
d->prev = d->more = false;
|
||||
io_uring_sqe_set_data(sqe, d);
|
||||
return sqe;
|
||||
}
|
||||
void set_immediate(const std::function<void()> & cb) override { cb(); }
|
||||
int submit() override { return 0; }
|
||||
int wait() override { return 0; }
|
||||
unsigned space_left() override { return 1024; }
|
||||
bool has_work() override { return false; }
|
||||
bool has_sendmsg_zc() override { return true; }
|
||||
void loop() override {}
|
||||
void wakeup() override {}
|
||||
unsigned save() override { return 0; }
|
||||
void restore(unsigned sqe_tail) override {}
|
||||
|
||||
// more = true emulates a CQE with IORING_CQE_F_MORE (first zerocopy completion)
|
||||
void complete(size_t i, int res, bool more)
|
||||
{
|
||||
assert(i < datas.size());
|
||||
ring_data_t *d = &datas[i];
|
||||
assert(d->callback);
|
||||
if (more)
|
||||
{
|
||||
d->res = res;
|
||||
d->more = true;
|
||||
d->callback(d);
|
||||
d->prev = true;
|
||||
d->more = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ring_data_t dl;
|
||||
dl.iov = d->iov;
|
||||
dl.res = res;
|
||||
dl.more = false;
|
||||
dl.prev = d->prev;
|
||||
dl.callback.swap(d->callback);
|
||||
d->prev = false;
|
||||
dl.callback(&dl);
|
||||
}
|
||||
}
|
||||
|
||||
bool is_zc(size_t i)
|
||||
{
|
||||
assert(i < sqes.size());
|
||||
assert(sqes[i].opcode == IORING_OP_SENDMSG_ZC || sqes[i].opcode == IORING_OP_SENDMSG);
|
||||
return sqes[i].opcode == IORING_OP_SENDMSG_ZC;
|
||||
}
|
||||
|
||||
msghdr *msg(size_t i)
|
||||
{
|
||||
return (msghdr*)sqes[i].addr;
|
||||
}
|
||||
};
|
||||
|
||||
class test_msgr_t: public osd_messenger_t
|
||||
{
|
||||
public:
|
||||
void set_zerocopy(bool enabled)
|
||||
{
|
||||
min_zerocopy_send_size = enabled ? 0 : -1;
|
||||
}
|
||||
|
||||
osd_client_t *add_client()
|
||||
{
|
||||
int fds[2];
|
||||
int r = socketpair(AF_UNIX, SOCK_STREAM, 0, fds);
|
||||
assert(r == 0);
|
||||
close(fds[1]);
|
||||
osd_client_t *cl = new osd_client_t();
|
||||
cl->client_id = next_client_id++;
|
||||
cl->peer_fd = fds[0];
|
||||
cl->peer_state = PEER_CONNECTED;
|
||||
clients[cl->client_id] = cl;
|
||||
clients_by_fd[cl->peer_fd] = cl;
|
||||
return cl;
|
||||
}
|
||||
|
||||
// Reply to a SEC_READ with TEST_DATA_LEN bytes of data, sent from op->buf
|
||||
osd_op_t *push_reply(osd_client_t *cl, uint8_t fill)
|
||||
{
|
||||
osd_op_t *op = new osd_op_t();
|
||||
op->op_type = OSD_OP_IN;
|
||||
op->client_id = cl->client_id;
|
||||
op->req.hdr.opcode = OSD_OP_SEC_READ;
|
||||
op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
|
||||
op->reply.hdr.opcode = OSD_OP_SEC_READ;
|
||||
op->reply.hdr.retval = TEST_DATA_LEN;
|
||||
op->buf = malloc_or_die(TEST_DATA_LEN);
|
||||
memset(op->buf, fill, TEST_DATA_LEN);
|
||||
op->iov.push_back(op->buf, TEST_DATA_LEN);
|
||||
cl->write_ops.push_back(op);
|
||||
return op;
|
||||
}
|
||||
|
||||
bool send(osd_client_t *cl)
|
||||
{
|
||||
return try_send(cl);
|
||||
}
|
||||
|
||||
bool is_write_ready(osd_client_t *cl)
|
||||
{
|
||||
for (auto id: write_ready_clients)
|
||||
if (id == cl->client_id)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
void socket_writable(osd_client_t *cl)
|
||||
{
|
||||
handle_peer_epoll(cl->peer_fd, EPOLLOUT);
|
||||
}
|
||||
};
|
||||
|
||||
struct test_env_t
|
||||
{
|
||||
timerfd_manager_t *tfd;
|
||||
send_ring_t *ring;
|
||||
test_msgr_t *msgr;
|
||||
std::map<int, bool> fd_write_watched;
|
||||
|
||||
test_env_t(bool zerocopy)
|
||||
{
|
||||
tfd = new timerfd_manager_t([this](int fd, bool wr, std::function<void(int, int)> callback)
|
||||
{
|
||||
if (callback)
|
||||
fd_write_watched[fd] = wr;
|
||||
else
|
||||
fd_write_watched.erase(fd);
|
||||
});
|
||||
ring = new send_ring_t();
|
||||
msgr = new test_msgr_t();
|
||||
msgr->tfd = tfd;
|
||||
msgr->ringloop = ring;
|
||||
msgr->has_sendmsg_zc = true;
|
||||
msgr->set_zerocopy(zerocopy);
|
||||
}
|
||||
|
||||
~test_env_t()
|
||||
{
|
||||
delete msgr;
|
||||
delete ring;
|
||||
delete tfd;
|
||||
}
|
||||
};
|
||||
|
||||
static void check_released(osd_client_t *cl, uint64_t next_send_free, size_t send_free_size)
|
||||
{
|
||||
if (cl->next_send_free != next_send_free || cl->send_free.size() != send_free_size)
|
||||
{
|
||||
fprintf(stderr, "send_free: expected next=%ju size=%zu, got next=%ju size=%zu\n",
|
||||
next_send_free, send_free_size, cl->next_send_free, cl->send_free.size());
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
static void check_tail(send_ring_t *ring, size_t i, osd_op_t *op, size_t sent)
|
||||
{
|
||||
assert(!ring->is_zc(i));
|
||||
assert(ring->sqes[i].msg_flags & MSG_WAITALL);
|
||||
msghdr *msg = ring->msg(i);
|
||||
size_t total = 0;
|
||||
for (size_t j = 0; j < msg->msg_iovlen; j++)
|
||||
total += msg->msg_iov[j].iov_len;
|
||||
assert(total == TEST_BATCH_LEN - sent);
|
||||
assert(sent >= OSD_PACKET_SIZE);
|
||||
assert(msg->msg_iov[0].iov_base == (uint8_t*)op->buf + (sent - OSD_PACKET_SIZE));
|
||||
}
|
||||
|
||||
// Zerocopy short write, the tail is sent before the release notification arrives
|
||||
static void test_zc_tail_before_notify()
|
||||
{
|
||||
printf("test_zc_tail_before_notify\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
assert(env.ring->is_zc(0));
|
||||
env.ring->complete(0, 1000, true);
|
||||
assert(cl->send_plain_tail);
|
||||
assert(cl->send_tail_zc_pending);
|
||||
assert(cl->send_list_size == TEST_BATCH_LEN - 1000);
|
||||
assert(env.msgr->is_write_ready(cl));
|
||||
env.msgr->send_replies();
|
||||
check_tail(env.ring, 1, a, 1000);
|
||||
env.ring->complete(1, TEST_BATCH_LEN - 1000, false);
|
||||
assert(!cl->send_plain_tail);
|
||||
// The zerocopy prefix is still owned by the kernel
|
||||
check_released(cl, 0, 1);
|
||||
env.ring->complete(0, 0, false);
|
||||
check_released(cl, 1, 0);
|
||||
}
|
||||
|
||||
// Zerocopy short write, the release notification arrives before the tail is sent
|
||||
static void test_zc_notify_before_tail()
|
||||
{
|
||||
printf("test_zc_notify_before_tail\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, 1000, true);
|
||||
// Notification before the tail is even submitted
|
||||
env.ring->complete(0, 0, false);
|
||||
assert(!cl->send_tail_zc_pending);
|
||||
check_released(cl, 0, 1);
|
||||
env.msgr->send_replies();
|
||||
check_tail(env.ring, 1, a, 1000);
|
||||
check_released(cl, 0, 1);
|
||||
env.ring->complete(1, TEST_BATCH_LEN - 1000, false);
|
||||
check_released(cl, 1, 0);
|
||||
}
|
||||
|
||||
// A notification of an older zerocopy batch arrives while the tail is pending.
|
||||
// It must be released at once and must not swallow the notification of the short batch.
|
||||
static void test_zc_other_notify_during_tail()
|
||||
{
|
||||
printf("test_zc_other_notify_during_tail\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
env.msgr->push_reply(cl, 0x11);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, TEST_BATCH_LEN, true);
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
assert(env.ring->is_zc(1));
|
||||
env.ring->complete(1, 1000, true);
|
||||
assert(cl->send_tail_from == 1 && cl->send_tail_to == 2);
|
||||
env.msgr->send_replies();
|
||||
check_tail(env.ring, 2, a, 1000);
|
||||
env.ring->complete(0, 0, false);
|
||||
check_released(cl, 1, 1);
|
||||
env.ring->complete(1, 0, false);
|
||||
check_released(cl, 1, 1);
|
||||
env.ring->complete(2, TEST_BATCH_LEN - 1000, false);
|
||||
check_released(cl, 2, 0);
|
||||
}
|
||||
|
||||
// Copying short write while an older zerocopy notification is still in flight
|
||||
static void test_plain_short_with_zc_notify()
|
||||
{
|
||||
printf("test_plain_short_with_zc_notify\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
env.msgr->push_reply(cl, 0x11);
|
||||
env.msgr->send(cl);
|
||||
assert(env.ring->is_zc(0));
|
||||
env.ring->complete(0, TEST_BATCH_LEN, true);
|
||||
env.msgr->set_zerocopy(false);
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
assert(!env.ring->is_zc(1));
|
||||
env.ring->complete(1, 1000, false);
|
||||
assert(cl->send_plain_tail);
|
||||
assert(!cl->send_tail_zc_pending);
|
||||
env.msgr->send_replies();
|
||||
check_tail(env.ring, 2, a, 1000);
|
||||
env.ring->complete(0, 0, false);
|
||||
check_released(cl, 1, 1);
|
||||
env.ring->complete(2, TEST_BATCH_LEN - 1000, false);
|
||||
check_released(cl, 2, 0);
|
||||
}
|
||||
|
||||
// The tail itself is short-written again: the owner range must be kept
|
||||
static void test_tail_short_again()
|
||||
{
|
||||
printf("test_tail_short_again\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, 1000, true);
|
||||
env.msgr->send_replies();
|
||||
check_tail(env.ring, 1, a, 1000);
|
||||
env.ring->complete(1, 1000, false);
|
||||
assert(cl->send_plain_tail);
|
||||
assert(cl->send_tail_from == 0 && cl->send_tail_to == 1);
|
||||
assert(cl->send_tail_zc_pending);
|
||||
env.msgr->send_replies();
|
||||
check_tail(env.ring, 2, a, 2000);
|
||||
env.ring->complete(0, 0, false);
|
||||
check_released(cl, 0, 1);
|
||||
env.ring->complete(2, TEST_BATCH_LEN - 2000, false);
|
||||
check_released(cl, 1, 0);
|
||||
}
|
||||
|
||||
// io_uring returns -EAGAIN: nothing was queued, so the batch waits for EPOLLOUT
|
||||
// and is then resent through the copying tail path. A second EAGAIN does not
|
||||
// submit another SQE until the socket is writable again.
|
||||
static void test_eagain_resend()
|
||||
{
|
||||
printf("test_eagain_resend\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, -EAGAIN, false);
|
||||
assert(env.msgr->clients.find(cl->client_id) != env.msgr->clients.end());
|
||||
assert(cl->send_plain_tail);
|
||||
assert(!cl->send_tail_zc_pending);
|
||||
assert(cl->send_list_size == TEST_BATCH_LEN);
|
||||
assert(!cl->write_ready);
|
||||
assert(env.fd_write_watched[cl->peer_fd]);
|
||||
assert(!env.msgr->is_write_ready(cl));
|
||||
env.msgr->send_replies();
|
||||
assert(env.ring->sqes.size() == 1);
|
||||
env.msgr->socket_writable(cl);
|
||||
assert(cl->write_ready);
|
||||
assert(!env.fd_write_watched[cl->peer_fd]);
|
||||
assert(env.msgr->is_write_ready(cl));
|
||||
env.msgr->send_replies();
|
||||
assert(env.ring->sqes.size() == 2);
|
||||
assert(!env.ring->is_zc(1));
|
||||
assert(env.ring->msg(1)->msg_iov[0].iov_base == &a->reply);
|
||||
env.ring->complete(1, -EAGAIN, false);
|
||||
assert(!cl->write_ready);
|
||||
assert(env.fd_write_watched[cl->peer_fd]);
|
||||
assert(!env.msgr->is_write_ready(cl));
|
||||
env.msgr->send_replies();
|
||||
assert(env.ring->sqes.size() == 2);
|
||||
env.msgr->socket_writable(cl);
|
||||
assert(!env.fd_write_watched[cl->peer_fd]);
|
||||
env.msgr->send_replies();
|
||||
assert(env.ring->sqes.size() == 3);
|
||||
env.ring->complete(2, TEST_BATCH_LEN, false);
|
||||
assert(!cl->send_plain_tail);
|
||||
check_released(cl, 1, 0);
|
||||
}
|
||||
|
||||
// Failed zerocopy send: -EAGAIN carries IORING_CQE_F_MORE. The release
|
||||
// notification must not free the buffer, and the resend still waits for EPOLLOUT.
|
||||
static void test_eagain_zc_more()
|
||||
{
|
||||
printf("test_eagain_zc_more\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, -EAGAIN, true);
|
||||
assert(cl->send_plain_tail);
|
||||
assert(cl->send_tail_zc_pending);
|
||||
assert(!cl->write_ready);
|
||||
assert(cl->send_list_size == TEST_BATCH_LEN);
|
||||
env.ring->complete(0, 0, false);
|
||||
assert(!cl->send_tail_zc_pending);
|
||||
check_released(cl, 0, 1);
|
||||
assert(env.ring->sqes.size() == 1);
|
||||
env.msgr->socket_writable(cl);
|
||||
env.msgr->send_replies();
|
||||
assert(!env.ring->is_zc(1));
|
||||
assert(env.ring->msg(1)->msg_iov[0].iov_base == &a->reply);
|
||||
env.ring->complete(1, TEST_BATCH_LEN, false);
|
||||
assert(!cl->send_plain_tail);
|
||||
check_released(cl, 1, 0);
|
||||
}
|
||||
|
||||
// A zero-byte send without an error is still a broken socket
|
||||
static void test_zero_bytes_disconnects()
|
||||
{
|
||||
printf("test_zero_bytes_disconnects\n");
|
||||
test_env_t env(false);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
uint64_t client_id = cl->client_id;
|
||||
env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, 0, false);
|
||||
assert(env.msgr->clients.find(client_id) == env.msgr->clients.end());
|
||||
}
|
||||
|
||||
// New replies queued while the tail is pending must go after it, in a separate send
|
||||
static void test_new_ops_wait_for_tail()
|
||||
{
|
||||
printf("test_new_ops_wait_for_tail\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
osd_op_t *a = env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
env.ring->complete(0, 1000, true);
|
||||
osd_op_t *b = env.msgr->push_reply(cl, 0xbb);
|
||||
env.msgr->send(cl);
|
||||
check_tail(env.ring, 1, a, 1000);
|
||||
assert(cl->write_ops.size() == 1);
|
||||
assert(env.ring->sqes.size() == 2);
|
||||
env.ring->complete(1, TEST_BATCH_LEN - 1000, false);
|
||||
assert(env.msgr->is_write_ready(cl));
|
||||
env.msgr->send_replies();
|
||||
assert(env.ring->sqes.size() == 3);
|
||||
assert(env.ring->is_zc(2));
|
||||
assert(env.ring->msg(2)->msg_iov[0].iov_base == &b->reply);
|
||||
env.ring->complete(2, TEST_BATCH_LEN, true);
|
||||
env.ring->complete(2, 0, false);
|
||||
// The first batch still waits for its notification and blocks the head
|
||||
check_released(cl, 0, 2);
|
||||
env.ring->complete(0, 0, false);
|
||||
check_released(cl, 2, 0);
|
||||
}
|
||||
|
||||
// A spurious EPOLLOUT while a normal io_uring send is in flight must not
|
||||
// enqueue the client into write_ready_clients
|
||||
static void test_inflight_epollout_ignored()
|
||||
{
|
||||
printf("test_inflight_epollout_ignored\n");
|
||||
test_env_t env(true);
|
||||
osd_client_t *cl = env.msgr->add_client();
|
||||
env.msgr->push_reply(cl, 0xaa);
|
||||
env.msgr->send(cl);
|
||||
assert(cl->write_msg.msg_iovlen > 0);
|
||||
assert(cl->send_list.size() > 0);
|
||||
env.msgr->socket_writable(cl);
|
||||
assert(!env.msgr->is_write_ready(cl));
|
||||
env.ring->complete(0, TEST_BATCH_LEN, true);
|
||||
env.ring->complete(0, 0, false);
|
||||
check_released(cl, 1, 0);
|
||||
}
|
||||
|
||||
int main(int narg, char *args[])
|
||||
{
|
||||
test_zc_tail_before_notify();
|
||||
test_zc_notify_before_tail();
|
||||
test_zc_other_notify_during_tail();
|
||||
test_plain_short_with_zc_notify();
|
||||
test_tail_short_again();
|
||||
test_eagain_resend();
|
||||
test_eagain_zc_more();
|
||||
test_zero_bytes_disconnects();
|
||||
test_new_ops_wait_for_tail();
|
||||
test_inflight_epollout_ignored();
|
||||
printf("OK\n");
|
||||
return 0;
|
||||
}
|
||||
714
src/test/test_msgr_send_reply.cpp
Normal file
714
src/test/test_msgr_send_reply.cpp
Normal file
@@ -0,0 +1,714 @@
|
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// One-shot transfer through osd_messenger_t over TCP or RDMA.
|
||||
//
|
||||
// test_msgr_send_reply server --ip ADDR [--port N] [--net tcp|rdma] [--zerocopy]
|
||||
// test_msgr_send_reply client --ip ADDR [--port N] [--net tcp|rdma]
|
||||
// [--send_size SIZE] [--reply_size SIZE] [--zerocopy]
|
||||
//
|
||||
// --ip ADDR required. Server bind address, or the address the client
|
||||
// connects to. --net rdma selects a RoCEv2 GID in the /24
|
||||
// that contains this address.
|
||||
// --port N TCP port. Default 11223.
|
||||
// --net tcp|rdma transport. Default tcp.
|
||||
// --send_size SIZE client only. OSD_OP_SEC_WRITE payload. Default 1 MiB.
|
||||
// --reply_size SIZE client only. OSD_OP_SEC_READ reply payload. Default 1 MiB.
|
||||
// --zerocopy send with io_uring sendmsg_zc. The default is a copying send.
|
||||
//
|
||||
// SIZE is a number of MiB, with an optional m/MiB suffix.
|
||||
// The server takes the lengths from the requests. It checks the write pattern and
|
||||
// fills the read reply with the same pattern.
|
||||
|
||||
#include <time.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdint.h>
|
||||
#include <signal.h>
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include "json11/json11.hpp"
|
||||
#include "addr_util.h"
|
||||
#include "ringloop.h"
|
||||
#include "epoll_manager.h"
|
||||
#include "messenger.h"
|
||||
|
||||
static const uint32_t DEFAULT_PAYLOAD_MIB = 1;
|
||||
static const uint32_t MAX_PAYLOAD_MIB = UINT32_MAX / (1024u * 1024u);
|
||||
static uint32_t g_send_mib = DEFAULT_PAYLOAD_MIB;
|
||||
static uint32_t g_send_bytes = DEFAULT_PAYLOAD_MIB * 1024u * 1024u;
|
||||
static uint32_t g_reply_mib = DEFAULT_PAYLOAD_MIB;
|
||||
static uint32_t g_reply_bytes = DEFAULT_PAYLOAD_MIB * 1024u * 1024u;
|
||||
static const int DEFAULT_PORT = 11223;
|
||||
static const osd_num_t SERVER_OSD = 2;
|
||||
static const osd_num_t CLIENT_OSD = 1;
|
||||
static const int TEST_TIMEOUT_SEC = 180;
|
||||
// Copying sendmsg is the default. --zerocopy opts into io_uring sendmsg_zc.
|
||||
static bool g_force_copy = true;
|
||||
static bool g_use_rdma = false;
|
||||
|
||||
enum
|
||||
{
|
||||
PHASE_WAIT = 0,
|
||||
PHASE_WRITE = 1,
|
||||
PHASE_READ = 2,
|
||||
PHASE_DONE = 3,
|
||||
PHASE_FAIL = 4,
|
||||
};
|
||||
|
||||
static osd_messenger_t *g_msgr = NULL;
|
||||
static int g_phase = PHASE_WAIT;
|
||||
static int g_exit = -1;
|
||||
static bool g_server_role = false;
|
||||
static bool g_seen_client = false;
|
||||
static bool g_write_ok = false;
|
||||
static bool g_read_ok = false;
|
||||
static time_t g_start = 0;
|
||||
static timespec g_t0;
|
||||
static uint8_t *g_write_buf = NULL;
|
||||
|
||||
static uint64_t pattern_word(size_t i)
|
||||
{
|
||||
return 0xC0FFEE0000000000ull + (uint64_t)i;
|
||||
}
|
||||
|
||||
static void fill_pattern(uint8_t *buf, size_t len)
|
||||
{
|
||||
uint64_t *p = (uint64_t*)buf;
|
||||
size_t n = len / sizeof(uint64_t);
|
||||
for (size_t i = 0; i < n; i++)
|
||||
p[i] = pattern_word(i);
|
||||
}
|
||||
|
||||
// Returns the byte offset of the first mismatch, or len if the buffer matches.
|
||||
static size_t check_pattern(const uint8_t *buf, size_t len)
|
||||
{
|
||||
const uint64_t *p = (const uint64_t*)buf;
|
||||
size_t n = len / sizeof(uint64_t);
|
||||
for (size_t i = 0; i < n; i++)
|
||||
{
|
||||
if (p[i] != pattern_word(i))
|
||||
return i * sizeof(uint64_t);
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
static double elapsed_ms(const timespec & t0)
|
||||
{
|
||||
timespec t1;
|
||||
clock_gettime(CLOCK_MONOTONIC, &t1);
|
||||
return (t1.tv_sec - t0.tv_sec) * 1000.0 + (t1.tv_nsec - t0.tv_nsec) / 1e6;
|
||||
}
|
||||
|
||||
static const char *peer_mode(osd_client_t *cl)
|
||||
{
|
||||
if (cl->peer_state == PEER_RDMA)
|
||||
return "rdma";
|
||||
if (cl->peer_state == PEER_CONNECTED)
|
||||
return "tcp";
|
||||
return "other";
|
||||
}
|
||||
|
||||
static void reply_op(osd_messenger_t *msgr, osd_op_t *op, int64_t retval)
|
||||
{
|
||||
op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
|
||||
op->reply.hdr.id = op->req.hdr.id;
|
||||
op->reply.hdr.opcode = op->req.hdr.opcode;
|
||||
op->reply.hdr.retval = retval;
|
||||
msgr->outbox_push(op);
|
||||
}
|
||||
|
||||
static void fail_test(const char *who, const char *why)
|
||||
{
|
||||
fprintf(stderr, "[%s] FAIL: %s\n", who, why);
|
||||
g_phase = PHASE_FAIL;
|
||||
g_exit = 1;
|
||||
}
|
||||
|
||||
static void start_read();
|
||||
|
||||
static void on_write_reply(osd_op_t *op)
|
||||
{
|
||||
double ms = elapsed_ms(g_t0);
|
||||
int64_t retval = op->reply.hdr.retval;
|
||||
delete op;
|
||||
if (g_phase != PHASE_WRITE)
|
||||
return;
|
||||
if (retval != (int64_t)g_send_bytes)
|
||||
{
|
||||
fprintf(stderr, "[client] SEC_WRITE retval=%jd (%s), %.1f ms\n",
|
||||
retval, retval < 0 ? strerror(-(int)retval) : "unexpected", ms);
|
||||
fail_test("client", "SEC_WRITE was not accepted");
|
||||
return;
|
||||
}
|
||||
double mib_s = (g_send_bytes / 1024.0 / 1024.0) / (ms / 1000.0);
|
||||
fprintf(stderr, "[client] SEC_WRITE %u bytes ok, %.1f ms, %.1f MiB/s\n",
|
||||
g_send_bytes, ms, mib_s);
|
||||
start_read();
|
||||
}
|
||||
|
||||
static void on_read_reply(osd_op_t *op)
|
||||
{
|
||||
double ms = elapsed_ms(g_t0);
|
||||
int64_t retval = op->reply.hdr.retval;
|
||||
if (g_phase != PHASE_READ)
|
||||
{
|
||||
delete op;
|
||||
return;
|
||||
}
|
||||
if (retval != (int64_t)g_reply_bytes)
|
||||
{
|
||||
fprintf(stderr, "[client] SEC_READ retval=%jd (%s), %.1f ms\n",
|
||||
retval, retval < 0 ? strerror(-(int)retval) : "unexpected", ms);
|
||||
delete op;
|
||||
fail_test("client", "SEC_READ was not accepted");
|
||||
return;
|
||||
}
|
||||
size_t bad = check_pattern((uint8_t*)op->buf, g_reply_bytes);
|
||||
delete op;
|
||||
if (bad != g_reply_bytes)
|
||||
{
|
||||
fprintf(stderr, "[client] SEC_READ pattern mismatch at offset %zu\n", bad);
|
||||
fail_test("client", "SEC_READ payload mismatch");
|
||||
return;
|
||||
}
|
||||
double mib_s = (g_reply_bytes / 1024.0 / 1024.0) / (ms / 1000.0);
|
||||
fprintf(stderr, "[client] SEC_READ %u bytes ok, %.1f ms, %.1f MiB/s\n",
|
||||
g_reply_bytes, ms, mib_s);
|
||||
fprintf(stderr, "[client] PASS\n");
|
||||
g_phase = PHASE_DONE;
|
||||
g_exit = 0;
|
||||
}
|
||||
|
||||
static void start_read()
|
||||
{
|
||||
auto cl_it = g_msgr->osd_peers.find(SERVER_OSD);
|
||||
if (cl_it == g_msgr->osd_peers.end())
|
||||
{
|
||||
fail_test("client", "peer gone before SEC_READ");
|
||||
return;
|
||||
}
|
||||
osd_op_t *op = new osd_op_t();
|
||||
op->op_type = OSD_OP_OUT;
|
||||
op->client_id = cl_it->second->client_id;
|
||||
op->req.hdr.magic = SECONDARY_OSD_OP_MAGIC;
|
||||
op->req.hdr.opcode = OSD_OP_SEC_READ;
|
||||
op->req.sec_rw.oid = { .inode = 1, .stripe = 0 };
|
||||
op->req.sec_rw.version = 1;
|
||||
op->req.sec_rw.len = g_reply_bytes;
|
||||
op->buf = memalign_or_die(MEM_ALIGNMENT, g_reply_bytes);
|
||||
op->iov.push_back(op->buf, g_reply_bytes);
|
||||
op->callback = on_read_reply;
|
||||
g_phase = PHASE_READ;
|
||||
clock_gettime(CLOCK_MONOTONIC, &g_t0);
|
||||
fprintf(stderr, "[client] sending one SEC_READ of %u bytes\n", g_reply_bytes);
|
||||
g_msgr->outbox_push(op);
|
||||
}
|
||||
|
||||
static void start_write()
|
||||
{
|
||||
auto cl_it = g_msgr->osd_peers.find(SERVER_OSD);
|
||||
if (cl_it == g_msgr->osd_peers.end())
|
||||
{
|
||||
fail_test("client", "connected but peer is missing");
|
||||
return;
|
||||
}
|
||||
osd_client_t *cl = cl_it->second;
|
||||
fprintf(stderr, "[client] connected to OSD %ju client_id=%ju mode=%s\n",
|
||||
SERVER_OSD, cl->client_id, peer_mode(cl));
|
||||
if (g_use_rdma && cl->peer_state != PEER_RDMA)
|
||||
{
|
||||
fail_test("client", "RDMA requested but the peer stayed on TCP");
|
||||
return;
|
||||
}
|
||||
if (!g_use_rdma && cl->peer_state == PEER_RDMA)
|
||||
{
|
||||
fail_test("client", "TCP requested but the peer switched to RDMA");
|
||||
return;
|
||||
}
|
||||
osd_op_t *op = new osd_op_t();
|
||||
op->op_type = OSD_OP_OUT;
|
||||
op->client_id = cl->client_id;
|
||||
op->req.hdr.magic = SECONDARY_OSD_OP_MAGIC;
|
||||
op->req.hdr.opcode = OSD_OP_SEC_WRITE;
|
||||
op->req.sec_rw.oid = { .inode = 1, .stripe = 0 };
|
||||
op->req.sec_rw.version = 1;
|
||||
op->req.sec_rw.len = g_send_bytes;
|
||||
op->buf = g_write_buf;
|
||||
g_write_buf = NULL;
|
||||
op->iov.push_back(op->buf, g_send_bytes);
|
||||
op->callback = on_write_reply;
|
||||
g_phase = PHASE_WRITE;
|
||||
clock_gettime(CLOCK_MONOTONIC, &g_t0);
|
||||
fprintf(stderr, "[client] sending one SEC_WRITE of %u bytes\n", g_send_bytes);
|
||||
g_msgr->outbox_push(op);
|
||||
}
|
||||
|
||||
static void exec_show_config(osd_op_t *op)
|
||||
{
|
||||
std::string json_err;
|
||||
json11::Json req = op->req.show_conf.json_len
|
||||
? json11::Json::parse(std::string((char*)op->buf), json_err)
|
||||
: json11::Json();
|
||||
auto cl = g_msgr->clients.at(op->client_id);
|
||||
cl->in_osd_num = req["osd_num"].uint64_value();
|
||||
if (req["features"]["check_sequencing"].bool_value())
|
||||
cl->check_sequencing = true;
|
||||
json11::Json::object features;
|
||||
int peer_csums = (int)req["features"]["proto_checksums"].uint64_value();
|
||||
if (g_msgr->use_proto_checksums && peer_csums)
|
||||
{
|
||||
if (g_msgr->use_proto_checksums == MSGR_CSUM_FULL && peer_csums == MSGR_CSUM_FULL)
|
||||
cl->proto_csum_status = MSGR_CSUM_FULL | MSGR_CSUM_NEG;
|
||||
else
|
||||
cl->proto_csum_status = MSGR_CSUM_PAYLOAD | MSGR_CSUM_NEG;
|
||||
features["proto_checksums"] = (uint64_t)g_msgr->use_proto_checksums;
|
||||
}
|
||||
json11::Json::object wire = {
|
||||
{ "osd_num", (uint64_t)SERVER_OSD },
|
||||
{ "protocol_version", (uint64_t)OSD_PROTOCOL_VERSION },
|
||||
{ "features", features },
|
||||
};
|
||||
#ifdef WITH_RDMA
|
||||
if (g_msgr->is_rdma_enabled())
|
||||
{
|
||||
wire["rdma_enabled"] = true;
|
||||
if (req["connect_rdma"].is_string())
|
||||
{
|
||||
json11::Json data = g_msgr->connect_rdma(
|
||||
op->client_id, req["connect_rdma"].string_value(), req["rdma_max_msg"].uint64_value());
|
||||
if (!data.is_null())
|
||||
{
|
||||
for (auto & kv: data.object_items())
|
||||
wire[kv.first] = kv.second;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (op->buf)
|
||||
free(op->buf);
|
||||
std::string cfg = json11::Json(wire).dump();
|
||||
op->buf = malloc_or_die(cfg.size() + 1);
|
||||
memcpy(op->buf, cfg.c_str(), cfg.size() + 1);
|
||||
op->iov.push_back(op->buf, cfg.size() + 1);
|
||||
fprintf(stderr, "[server] SHOW_CONFIG from OSD %ju, checksum=%d, reply %s\n",
|
||||
cl->in_osd_num, cl->proto_csum_status, cfg.c_str());
|
||||
reply_op(g_msgr, op, cfg.size() + 1);
|
||||
}
|
||||
|
||||
static void exec_sec_write(osd_op_t *op)
|
||||
{
|
||||
auto cl = g_msgr->clients.at(op->client_id);
|
||||
uint32_t len = op->req.sec_rw.len;
|
||||
if (!len || !op->buf)
|
||||
{
|
||||
fprintf(stderr, "[server] SEC_WRITE empty payload\n");
|
||||
g_write_ok = false;
|
||||
reply_op(g_msgr, op, -EINVAL);
|
||||
return;
|
||||
}
|
||||
g_send_bytes = len;
|
||||
bool mode_ok = g_use_rdma ? cl->peer_state == PEER_RDMA : cl->peer_state == PEER_CONNECTED;
|
||||
size_t bad = check_pattern((uint8_t*)op->buf, len);
|
||||
if (!mode_ok || bad != len)
|
||||
{
|
||||
fprintf(stderr, "[server] SEC_WRITE %s at mode=%s (want %s), mismatch_off=%zu\n",
|
||||
mode_ok ? "pattern mismatch" : "transport mismatch",
|
||||
peer_mode(cl), g_use_rdma ? "rdma" : "tcp", bad);
|
||||
g_write_ok = false;
|
||||
reply_op(g_msgr, op, -EIO);
|
||||
return;
|
||||
}
|
||||
g_write_ok = true;
|
||||
fprintf(stderr, "[server] SEC_WRITE %u bytes pattern ok (mode=%s)\n",
|
||||
len, peer_mode(cl));
|
||||
reply_op(g_msgr, op, len);
|
||||
}
|
||||
|
||||
static void exec_sec_read(osd_op_t *op)
|
||||
{
|
||||
auto cl = g_msgr->clients.at(op->client_id);
|
||||
uint32_t len = op->req.sec_rw.len;
|
||||
if (!len)
|
||||
{
|
||||
fprintf(stderr, "[server] SEC_READ empty payload\n");
|
||||
reply_op(g_msgr, op, -EINVAL);
|
||||
return;
|
||||
}
|
||||
g_reply_bytes = len;
|
||||
op->buf = memalign_or_die(MEM_ALIGNMENT, len);
|
||||
fprintf(stderr, "[server] filling %u byte SEC_READ reply\n", len);
|
||||
fill_pattern((uint8_t*)op->buf, len);
|
||||
op->iov.push_back(op->buf, len);
|
||||
op->reply.sec_rw.attr_len = 0;
|
||||
g_read_ok = true;
|
||||
fprintf(stderr, "[server] sending one SEC_READ reply of %u bytes (mode=%s)\n",
|
||||
len, peer_mode(cl));
|
||||
reply_op(g_msgr, op, len);
|
||||
}
|
||||
|
||||
static void server_exec_op(osd_op_t *op)
|
||||
{
|
||||
g_seen_client = true;
|
||||
if (op->req.hdr.opcode == OSD_OP_SHOW_CONFIG)
|
||||
exec_show_config(op);
|
||||
else if (op->req.hdr.opcode == OSD_OP_PING)
|
||||
reply_op(g_msgr, op, 0);
|
||||
else if (op->req.hdr.opcode == OSD_OP_SEC_WRITE || op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
|
||||
exec_sec_write(op);
|
||||
else if (op->req.hdr.opcode == OSD_OP_SEC_READ)
|
||||
exec_sec_read(op);
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "[server] unsupported opcode %ju\n", op->req.hdr.opcode);
|
||||
reply_op(g_msgr, op, -EINVAL);
|
||||
}
|
||||
}
|
||||
|
||||
static json11::Json msgr_config(const std::string & ip)
|
||||
{
|
||||
json11::Json::object cfg = {
|
||||
{ "log_level", 3 },
|
||||
{ "use_rdma", g_use_rdma },
|
||||
{ "osd_idle_timeout", TEST_TIMEOUT_SEC },
|
||||
{ "osd_ping_timeout", TEST_TIMEOUT_SEC },
|
||||
{ "peer_connect_timeout", 10 },
|
||||
{ "peer_connect_interval", 1 },
|
||||
{ "proto_checksums", "payload" },
|
||||
{ "use_sync_send_recv", false },
|
||||
};
|
||||
|
||||
if (g_use_rdma)
|
||||
cfg["osd_network"] = ip + "/24";
|
||||
cfg["min_zerocopy_send_size"] = g_force_copy ? -1 : 0;
|
||||
return cfg;
|
||||
}
|
||||
|
||||
static int setup_msgr(ring_loop_t *ringloop, epoll_manager_t *epmgr, osd_num_t osd_num, const std::string & ip)
|
||||
{
|
||||
g_msgr = new osd_messenger_t();
|
||||
g_msgr->osd_num = osd_num;
|
||||
g_msgr->tfd = epmgr->tfd;
|
||||
g_msgr->ringloop = ringloop;
|
||||
g_msgr->repeer_pgs = [](osd_num_t peer)
|
||||
{
|
||||
if (!g_server_role && peer == SERVER_OSD && g_phase == PHASE_WAIT)
|
||||
start_write();
|
||||
};
|
||||
g_msgr->break_pg_locks = [](osd_num_t) {};
|
||||
if (g_server_role)
|
||||
g_msgr->exec_op = [](osd_op_t *op) { server_exec_op(op); };
|
||||
g_msgr->parse_config(msgr_config(ip), true);
|
||||
uint32_t op_bytes = g_server_role
|
||||
? MAX_PAYLOAD_MIB * 1024u * 1024u
|
||||
: (g_send_bytes > g_reply_bytes ? g_send_bytes : g_reply_bytes);
|
||||
g_msgr->bs_block_size = op_bytes;
|
||||
g_msgr->max_write_request_size = op_bytes;
|
||||
g_msgr->init();
|
||||
#ifdef WITH_RDMA
|
||||
bool rdma_on = g_msgr->is_rdma_enabled();
|
||||
#else
|
||||
bool rdma_on = false;
|
||||
#endif
|
||||
if (g_server_role)
|
||||
{
|
||||
fprintf(stderr, "[server] osd=%ju transport=%s zerocopy=%d sendmsg_zc=%d rdma_ready=%d\n",
|
||||
osd_num, g_use_rdma ? "rdma" : "tcp", (int)!g_force_copy, (int)g_msgr->has_sendmsg_zc, (int)rdma_on);
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "[client] osd=%ju send=%u MiB (%u bytes) reply=%u MiB (%u bytes) transport=%s zerocopy=%d sendmsg_zc=%d rdma_ready=%d\n",
|
||||
osd_num, g_send_mib, g_send_bytes, g_reply_mib, g_reply_bytes,
|
||||
g_use_rdma ? "rdma" : "tcp", (int)!g_force_copy, (int)g_msgr->has_sendmsg_zc, (int)rdma_on);
|
||||
}
|
||||
if (g_use_rdma && !rdma_on)
|
||||
{
|
||||
fprintf(stderr, "[%s] FAIL: RDMA requested but no device matched %s\n",
|
||||
g_server_role ? "server" : "client", ip.c_str());
|
||||
g_exit = 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void arm_timeout(timerfd_manager_t *tfd)
|
||||
{
|
||||
tfd->set_timer(1000, true, [](int)
|
||||
{
|
||||
if (g_exit >= 0)
|
||||
return;
|
||||
if (g_server_role && g_seen_client && g_msgr->clients.empty())
|
||||
{
|
||||
if (g_write_ok && g_read_ok)
|
||||
{
|
||||
fprintf(stderr, "[server] PASS write %u bytes + read %u bytes, peer disconnected\n",
|
||||
g_send_bytes, g_reply_bytes);
|
||||
g_exit = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "[server] FAIL peer disconnected write_ok=%d read_ok=%d\n",
|
||||
(int)g_write_ok, (int)g_read_ok);
|
||||
g_exit = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (time(NULL) - g_start >= TEST_TIMEOUT_SEC)
|
||||
{
|
||||
fprintf(stderr, "[%s] FAIL timeout after %d s phase=%d write_ok=%d read_ok=%d\n",
|
||||
g_server_role ? "server" : "client", TEST_TIMEOUT_SEC, g_phase,
|
||||
(int)g_write_ok, (int)g_read_ok);
|
||||
g_exit = 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
static int run_loop(ring_loop_t *ringloop)
|
||||
{
|
||||
while (g_exit < 0)
|
||||
{
|
||||
ringloop->loop();
|
||||
if (g_exit >= 0)
|
||||
break;
|
||||
ringloop->wait();
|
||||
}
|
||||
return g_exit;
|
||||
}
|
||||
|
||||
static int run_server(const std::string & bind_ip, int port)
|
||||
{
|
||||
g_server_role = true;
|
||||
ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
|
||||
epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
|
||||
setup_msgr(ringloop, epmgr, SERVER_OSD, bind_ip);
|
||||
if (g_exit > 0)
|
||||
return g_exit;
|
||||
int listen_fd = create_and_bind_socket(bind_ip, port, 128, NULL);
|
||||
if (listen_fd < 0)
|
||||
{
|
||||
fprintf(stderr, "listen failed\n");
|
||||
return 1;
|
||||
}
|
||||
fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
|
||||
epmgr->set_fd_handler(listen_fd, false, [listen_fd](int, int)
|
||||
{
|
||||
g_msgr->accept_connections(listen_fd);
|
||||
});
|
||||
ring_consumer_t looper;
|
||||
looper.loop = [ringloop]()
|
||||
{
|
||||
g_msgr->read_requests();
|
||||
g_msgr->send_replies();
|
||||
ringloop->submit();
|
||||
};
|
||||
ringloop->register_consumer(&looper);
|
||||
arm_timeout(epmgr->tfd);
|
||||
fprintf(stderr, "[server] listening on %s:%d\n", bind_ip.c_str(), port);
|
||||
int rc = run_loop(ringloop);
|
||||
delete g_msgr;
|
||||
delete epmgr;
|
||||
delete ringloop;
|
||||
return rc < 0 ? 1 : rc;
|
||||
}
|
||||
|
||||
static int run_client(const std::string & host, int port)
|
||||
{
|
||||
g_server_role = false;
|
||||
fprintf(stderr, "[client] filling %u byte pattern\n", g_send_bytes);
|
||||
g_write_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, g_send_bytes);
|
||||
fill_pattern(g_write_buf, g_send_bytes);
|
||||
ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
|
||||
epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
|
||||
setup_msgr(ringloop, epmgr, CLIENT_OSD, host);
|
||||
if (g_exit > 0)
|
||||
return g_exit;
|
||||
ring_consumer_t looper;
|
||||
looper.loop = [ringloop]()
|
||||
{
|
||||
g_msgr->read_requests();
|
||||
g_msgr->send_replies();
|
||||
ringloop->submit();
|
||||
};
|
||||
ringloop->register_consumer(&looper);
|
||||
arm_timeout(epmgr->tfd);
|
||||
g_msgr->connect_peer(SERVER_OSD, json11::Json::object {
|
||||
{ "addresses", json11::Json::array { host } },
|
||||
{ "port", port },
|
||||
});
|
||||
fprintf(stderr, "[client] connecting to %s:%d\n", host.c_str(), port);
|
||||
int rc = run_loop(ringloop);
|
||||
delete g_msgr;
|
||||
delete epmgr;
|
||||
delete ringloop;
|
||||
return rc < 0 ? 1 : rc;
|
||||
}
|
||||
|
||||
static void usage(const char *argv0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"USAGE:\n"
|
||||
" %s server --ip ADDR [--port N] [--net tcp|rdma] [--zerocopy]\n"
|
||||
" %s client --ip ADDR [--port N] [--net tcp|rdma]\n"
|
||||
" [--send_size SIZE] [--reply_size SIZE] [--zerocopy]\n"
|
||||
"The client sends one SEC_WRITE of --send_size, then one SEC_READ of --reply_size.\n"
|
||||
"The server uses those request lengths. SIZE is MiB, with an optional m/MiB\n"
|
||||
"suffix (default %u, max %u).\n"
|
||||
"--ip is required. Defaults: --port %d --net tcp, copying send.\n"
|
||||
"--zerocopy uses io_uring sendmsg_zc. --net rdma selects a RoCEv2 GID in the /24 of --ip.\n",
|
||||
argv0, argv0, DEFAULT_PAYLOAD_MIB, MAX_PAYLOAD_MIB, DEFAULT_PORT);
|
||||
}
|
||||
|
||||
static bool parse_u32(const char *begin, const char *end, uint32_t *out)
|
||||
{
|
||||
if (begin >= end)
|
||||
return false;
|
||||
uint64_t v = 0;
|
||||
for (const char *p = begin; p < end; p++)
|
||||
{
|
||||
if (*p < '0' || *p > '9')
|
||||
return false;
|
||||
v = v * 10u + (uint32_t)(*p - '0');
|
||||
if (v > UINT32_MAX)
|
||||
return false;
|
||||
}
|
||||
*out = (uint32_t)v;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool suffix_is_mib(const char *s)
|
||||
{
|
||||
char buf[8];
|
||||
size_t n = strlen(s);
|
||||
if (n == 0 || n >= sizeof(buf))
|
||||
return false;
|
||||
for (size_t i = 0; i < n; i++)
|
||||
buf[i] = (s[i] >= 'A' && s[i] <= 'Z') ? (char)(s[i] - 'A' + 'a') : s[i];
|
||||
buf[n] = 0;
|
||||
return !strcmp(buf, "m") || !strcmp(buf, "mib");
|
||||
}
|
||||
|
||||
static uint32_t parse_mib_size(const char *arg, const char *what)
|
||||
{
|
||||
const char *end = arg;
|
||||
while (*end >= '0' && *end <= '9')
|
||||
end++;
|
||||
uint32_t mib = 0;
|
||||
if (!parse_u32(arg, end, &mib) || !(end[0] == 0 || suffix_is_mib(end)) ||
|
||||
mib < 1 || mib > MAX_PAYLOAD_MIB)
|
||||
{
|
||||
char msg[96];
|
||||
snprintf(msg, sizeof(msg), "bad %s (1..%u MiB): %s", what, MAX_PAYLOAD_MIB, arg);
|
||||
throw std::runtime_error(msg);
|
||||
}
|
||||
return mib;
|
||||
}
|
||||
|
||||
static void set_send_mib(uint32_t mib)
|
||||
{
|
||||
g_send_mib = mib;
|
||||
g_send_bytes = mib * 1024u * 1024u;
|
||||
}
|
||||
|
||||
static void set_reply_mib(uint32_t mib)
|
||||
{
|
||||
g_reply_mib = mib;
|
||||
g_reply_bytes = mib * 1024u * 1024u;
|
||||
}
|
||||
|
||||
static int parse_port(const char *arg)
|
||||
{
|
||||
uint32_t p = 0;
|
||||
if (!parse_u32(arg, arg + strlen(arg), &p) || p < 1 || p > 65535)
|
||||
throw std::runtime_error(std::string("bad port: ") + arg);
|
||||
return (int)p;
|
||||
}
|
||||
|
||||
static void set_transport(const char *value)
|
||||
{
|
||||
if (!strcmp(value, "rdma"))
|
||||
g_use_rdma = true;
|
||||
else if (!strcmp(value, "tcp"))
|
||||
g_use_rdma = false;
|
||||
else
|
||||
throw std::runtime_error(std::string("bad --net (tcp or rdma): ") + value);
|
||||
}
|
||||
|
||||
// Returns the value of --name VALUE or --name=VALUE, or NULL if arg is another flag.
|
||||
static const char *flag_value(int argc, char **argv, int & i, const char *name)
|
||||
{
|
||||
const char *arg = argv[i];
|
||||
size_t n = strlen(name);
|
||||
if (!strncmp(arg, name, n) && arg[n] == '=')
|
||||
return arg + n + 1;
|
||||
if (strcmp(arg, name))
|
||||
return NULL;
|
||||
if (i + 1 >= argc)
|
||||
throw std::runtime_error(std::string(name) + " needs a value");
|
||||
return argv[++i];
|
||||
}
|
||||
|
||||
static void apply_arg(int argc, char **argv, int & i, bool server, std::string & host, int & port)
|
||||
{
|
||||
const char *value;
|
||||
if ((value = flag_value(argc, argv, i, "--ip")))
|
||||
host = value;
|
||||
else if ((value = flag_value(argc, argv, i, "--net")))
|
||||
set_transport(value);
|
||||
else if ((value = flag_value(argc, argv, i, "--send_size")))
|
||||
{
|
||||
if (server)
|
||||
throw std::runtime_error("server does not take --send_size; the client request sets it");
|
||||
set_send_mib(parse_mib_size(value, "send_size"));
|
||||
}
|
||||
else if ((value = flag_value(argc, argv, i, "--reply_size")))
|
||||
{
|
||||
if (server)
|
||||
throw std::runtime_error("server does not take --reply_size; the client request sets it");
|
||||
set_reply_mib(parse_mib_size(value, "reply_size"));
|
||||
}
|
||||
else if ((value = flag_value(argc, argv, i, "--port")))
|
||||
port = parse_port(value);
|
||||
else if (!strcmp(argv[i], "--zerocopy"))
|
||||
g_force_copy = false;
|
||||
else
|
||||
throw std::runtime_error(std::string("unknown argument: ") + argv[i]);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
g_start = time(NULL);
|
||||
if (argc < 2)
|
||||
{
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
std::string mode = argv[1];
|
||||
std::string host;
|
||||
int port = DEFAULT_PORT;
|
||||
try
|
||||
{
|
||||
if (mode == "server" || mode == "client")
|
||||
{
|
||||
for (int i = 2; i < argc; i++)
|
||||
apply_arg(argc, argv, i, mode == "server", host, port);
|
||||
if (host.empty())
|
||||
throw std::runtime_error("--ip is required");
|
||||
if (mode == "server")
|
||||
return run_server(host, port);
|
||||
return run_client(host, port);
|
||||
}
|
||||
}
|
||||
catch (std::exception & e)
|
||||
{
|
||||
fprintf(stderr, "test_msgr_send_reply: %s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
Reference in New Issue
Block a user