Subscribe to EPOLLOUT only while a send waits for it in no-io_uring mode #144

Open
koolkhel wants to merge 1 commits from koolkhel/vitastor:no-uring-epollout-on-demand into master
3 changed files with 179 additions and 4 deletions

View File

@@ -550,8 +550,7 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
int one = 1;
setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
cl->peer_state = PEER_CONNECTED;
// EPOLLOUT is only required without io_uring, to continue partially sent messages
tfd->set_fd_handler(peer_fd, use_sync_send_recv, [this](int peer_fd, int epoll_events)
tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
{
handle_peer_epoll(peer_fd, epoll_events);
});
@@ -600,6 +599,10 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
}
cl = cl_it->second;
cl->write_ready = true;
tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
{
handle_peer_epoll(peer_fd, epoll_events);
});
// send_list is only left non-empty when we're waiting for EPOLLOUT
if (cl->send_list.size() && cl->peer_state != PEER_RDMA)
{
@@ -841,8 +844,8 @@ void osd_messenger_t::accept_connections(int listen_fd)
{
continue;
}
// Add FD to epoll. EPOLLOUT is only required without io_uring
tfd->set_fd_handler(peer_fd, use_sync_send_recv, [this](int peer_fd, int epoll_events)
// Add FD to epoll
tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
{
handle_peer_epoll(peer_fd, epoll_events);
});

View File

@@ -627,7 +627,12 @@ copy_ops:
}
if (cl->send_list.size() && --send_retries <= 0)
{
// EPOLLOUT only while waiting: QEMU's fd handlers are level-triggered
cl->write_ready = false;
tfd->set_fd_handler(cl->peer_fd, true, [this](int peer_fd, int epoll_events)
{
handle_peer_epoll(peer_fd, epoll_events);
});
return true;
}
}

View File

@@ -3,8 +3,11 @@
// Stopping an OSD peer must cancel its operations at once, also while a send to the
// peer is stuck on a full socket, as with a powered-off host.
// Without io_uring, in a level-triggered loop like QEMU's, an idle connection must not
// wake the loop for EPOLLOUT.
// Uses the real messenger over io_uring. Run: test_msgr_stop
#include <sys/epoll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
@@ -15,6 +18,7 @@
#include <stdio.h>
#include <assert.h>
#include <map>
#include <stdexcept>
#include <vector>
@@ -242,10 +246,173 @@ static void test_keepalive_after_stall()
delete tfd;
}
struct level_fd_t
{
bool wr;
std::function<void(int, int)> cb;
};
// One pass of a level-triggered loop, as QEMU runs the fd handlers that vitastor_c gives it.
// Returns how many times a write handler was called
static int level_loop_pass(std::map<int, level_fd_t> & fds)
{
std::vector<pollfd> pfds;
for (auto & p: fds)
pfds.push_back((pollfd){ .fd = p.first, .events = (short)(POLLIN | (p.second.wr ? POLLOUT : 0)) });
int r = poll(pfds.data(), pfds.size(), 0);
assert(r >= 0);
int write_calls = 0;
for (auto & pfd: pfds)
{
auto it = fds.find(pfd.fd);
if (it != fds.end() && (pfd.revents & POLLIN))
{
auto cb = it->second.cb;
cb(pfd.fd, EPOLLIN);
}
it = fds.find(pfd.fd);
if (it != fds.end() && it->second.wr && (pfd.revents & POLLOUT))
{
write_calls++;
auto cb = it->second.cb;
cb(pfd.fd, EPOLLOUT);
}
}
return write_calls;
}
static size_t read_all(int fd)
{
static char buf[65536];
size_t total = 0;
ssize_t r;
while ((r = read(fd, buf, sizeof(buf))) > 0)
total += r;
return total;
}
static void test_no_uring_epollout_on_demand()
{
printf("test_no_uring_epollout_on_demand\n");
std::map<int, level_fd_t> fds;
timerfd_manager_t *tfd = new timerfd_manager_t([&](int fd, bool wr, std::function<void(int, int)> cb)
{
if (cb)
fds[fd] = { .wr = wr, .cb = cb };
else
fds.erase(fd);
});
osd_messenger_t *msgr = new osd_messenger_t();
msgr->osd_num = 1;
msgr->tfd = tfd;
msgr->parse_config(json11::Json::object {}, true);
// A client connects, the messenger accepts it. Small buffers fill up quickly
int small = 16384;
int listen_fd = socket(AF_INET, SOCK_STREAM, 0);
assert(listen_fd >= 0);
setsockopt(listen_fd, SOL_SOCKET, SO_SNDBUF, &small, sizeof(small));
sockaddr_in addr = {};
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
int r = bind(listen_fd, (sockaddr*)&addr, sizeof(addr));
assert(r == 0);
r = listen(listen_fd, 1);
assert(r == 0);
socklen_t addr_len = sizeof(addr);
r = getsockname(listen_fd, (sockaddr*)&addr, &addr_len);
assert(r == 0);
int peer_fd = socket(AF_INET, SOCK_STREAM, 0);
assert(peer_fd >= 0);
setsockopt(peer_fd, SOL_SOCKET, SO_RCVBUF, &small, sizeof(small));
r = connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
assert(r == 0);
fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
msgr->accept_connections(listen_fd);
assert(msgr->clients.size() == 1);
osd_client_t *cl = msgr->clients.begin()->second;
const int our_fd = cl->peer_fd;
assert(fds.find(our_fd) != fds.end());
uint64_t op_id = 0;
auto push_replies = [&](int n, int len)
{
for (int i = 0; i < n; i++)
{
osd_op_t *op = new osd_op_t();
op->op_type = OSD_OP_IN;
op->client_id = cl->client_id;
op->req.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op->req.hdr.id = ++op_id;
op->req.hdr.opcode = OSD_OP_READ;
op->req.rw.inode = 1;
op->req.rw.offset = op_id*len;
op->req.rw.len = len;
op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
op->reply.hdr.id = op->req.hdr.id;
op->reply.hdr.opcode = OSD_OP_READ;
op->reply.hdr.retval = len;
op->buf = malloc_or_die(len);
memset(op->buf, 0xab, len);
op->iov.push_back(op->buf, len);
cl->received_ops.push_back(op);
msgr->outbox_push(op);
}
};
auto idle_write_calls = [&]()
{
int calls = 0;
for (int i = 0; i < 100; i++)
calls += level_loop_pass(fds);
return calls;
};
// (1) Replies that fit into the socket buffer are sent at once, then the connection is idle
push_replies(4, 1024);
assert(read_all(peer_fd) == 4*(OSD_PACKET_SIZE+1024));
int calls = idle_write_calls();
printf(" idle: EPOLLOUT subscribed %d, write handler called %d times in 100 passes\n", fds[our_fd].wr, calls);
assert(!cl->send_list.size() && cl->write_ready);
assert(!fds[our_fd].wr);
assert(calls == 0);
// (2) The client doesn't read: the send parks on the full socket and waits for EPOLLOUT
const int n_big = 16, big_len = 65536;
const size_t total = n_big*(OSD_PACKET_SIZE+big_len);
push_replies(n_big, big_len);
printf(" parked: %zu iovecs unsent, EPOLLOUT subscribed %d\n", cl->send_list.size(), fds[our_fd].wr);
assert(cl->send_list.size() > 0 && !cl->write_ready);
assert(fds[our_fd].wr);
// The client reads, the loop calls the write handler and the send completes
size_t got = 0;
calls = 0;
for (int i = 0; i < 100000 && got < total; i++)
{
got += read_all(peer_fd);
calls += level_loop_pass(fds);
}
printf(" received %zu of %zu bytes, write handler called %d times\n", got, total, calls);
assert(got == total);
assert(calls > 0);
assert(!cl->send_list.size() && cl->write_ready);
assert(!fds[our_fd].wr);
calls = idle_write_calls();
assert(calls == 0);
delete msgr;
assert(fds.find(our_fd) == fds.end());
close(peer_fd);
close(listen_fd);
delete tfd;
printf("test_no_uring_epollout_on_demand passed\n");
}
int main(int narg, char *args[])
{
test_keepalive_catch_up();
test_keepalive_after_stall();
test_no_uring_epollout_on_demand();
ring_loop_t *ringloop;
try
{