forked from vitalif/vitastor
Allow to turn synchronous recvmsg/sendmsg on with a config option
parent
9abaf5b735
commit
62343c8022
|
@ -46,6 +46,8 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
|
||||||
msgr.stop_client(op->peer_fd);
|
msgr.stop_client(op->peer_fd);
|
||||||
delete op;
|
delete op;
|
||||||
};
|
};
|
||||||
|
msgr.use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
|
||||||
|
config["use_sync_send_recv"].uint64_value();
|
||||||
|
|
||||||
st_cli.tfd = tfd;
|
st_cli.tfd = tfd;
|
||||||
st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
|
st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
|
||||||
|
|
|
@ -3,17 +3,17 @@
|
||||||
// Random write:
|
// Random write:
|
||||||
//
|
//
|
||||||
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -fsync=16 -iodepth=16 -rw=randwrite \
|
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -fsync=16 -iodepth=16 -rw=randwrite \
|
||||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/microceph] -size=1000M
|
// -etcd=127.0.0.1:2379 [-etcd_prefix=/microceph] -inode=1 -size=1000M
|
||||||
//
|
//
|
||||||
// Linear write:
|
// Linear write:
|
||||||
//
|
//
|
||||||
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=128k -direct=1 -fsync=32 -iodepth=32 -rw=write \
|
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=128k -direct=1 -fsync=32 -iodepth=32 -rw=write \
|
||||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/microceph] -size=1000M
|
// -etcd=127.0.0.1:2379 [-etcd_prefix=/microceph] -inode=1 -size=1000M
|
||||||
//
|
//
|
||||||
// Random read (run with -iodepth=32 or -iodepth=1):
|
// Random read (run with -iodepth=32 or -iodepth=1):
|
||||||
//
|
//
|
||||||
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -iodepth=32 -rw=randread \
|
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -iodepth=32 -rw=randread \
|
||||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/microceph] -size=1000M
|
// -etcd=127.0.0.1:2379 [-etcd_prefix=/microceph] -inode=1 -size=1000M
|
||||||
|
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
|
|
@ -169,7 +169,10 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
|
||||||
if (cl.read_ready == 1)
|
if (cl.read_ready == 1)
|
||||||
{
|
{
|
||||||
read_ready_clients.push_back(cl.peer_fd);
|
read_ready_clients.push_back(cl.peer_fd);
|
||||||
ringloop->wakeup();
|
if (ringloop)
|
||||||
|
ringloop->wakeup();
|
||||||
|
else
|
||||||
|
read_requests();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -233,6 +233,7 @@ struct osd_messenger_t
|
||||||
int peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
int peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
||||||
int peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
int peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
||||||
int log_level = 0;
|
int log_level = 0;
|
||||||
|
bool use_sync_send_recv = false;
|
||||||
|
|
||||||
std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
|
std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
|
||||||
std::map<uint64_t, int> osd_peer_fds;
|
std::map<uint64_t, int> osd_peer_fds;
|
||||||
|
|
|
@ -6,13 +6,6 @@ void osd_messenger_t::read_requests()
|
||||||
{
|
{
|
||||||
int peer_fd = read_ready_clients[i];
|
int peer_fd = read_ready_clients[i];
|
||||||
auto & cl = clients[peer_fd];
|
auto & cl = clients[peer_fd];
|
||||||
io_uring_sqe* sqe = ringloop->get_sqe();
|
|
||||||
if (!sqe)
|
|
||||||
{
|
|
||||||
read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
ring_data_t* data = ((ring_data_t*)sqe->user_data);
|
|
||||||
if (cl.read_remaining < receive_buffer_size)
|
if (cl.read_remaining < receive_buffer_size)
|
||||||
{
|
{
|
||||||
cl.read_iov.iov_base = cl.in_buf;
|
cl.read_iov.iov_base = cl.in_buf;
|
||||||
|
@ -27,8 +20,27 @@ void osd_messenger_t::read_requests()
|
||||||
cl.read_msg.msg_iov = cl.recv_list.get_iovec();
|
cl.read_msg.msg_iov = cl.recv_list.get_iovec();
|
||||||
cl.read_msg.msg_iovlen = cl.recv_list.get_size();
|
cl.read_msg.msg_iovlen = cl.recv_list.get_size();
|
||||||
}
|
}
|
||||||
data->callback = [this, peer_fd](ring_data_t *data) { handle_read(data->res, peer_fd); };
|
if (ringloop && !use_sync_send_recv)
|
||||||
my_uring_prep_recvmsg(sqe, peer_fd, &cl.read_msg, 0);
|
{
|
||||||
|
io_uring_sqe* sqe = ringloop->get_sqe();
|
||||||
|
if (!sqe)
|
||||||
|
{
|
||||||
|
read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ring_data_t* data = ((ring_data_t*)sqe->user_data);
|
||||||
|
data->callback = [this, peer_fd](ring_data_t *data) { handle_read(data->res, peer_fd); };
|
||||||
|
my_uring_prep_recvmsg(sqe, peer_fd, &cl.read_msg, 0);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
int result = recvmsg(peer_fd, &cl.read_msg, 0);
|
||||||
|
if (result < 0)
|
||||||
|
{
|
||||||
|
result = -errno;
|
||||||
|
}
|
||||||
|
handle_read(result, peer_fd);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
read_ready_clients.clear();
|
read_ready_clients.clear();
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,12 +42,6 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
|
||||||
bool osd_messenger_t::try_send(osd_client_t & cl)
|
bool osd_messenger_t::try_send(osd_client_t & cl)
|
||||||
{
|
{
|
||||||
int peer_fd = cl.peer_fd;
|
int peer_fd = cl.peer_fd;
|
||||||
io_uring_sqe* sqe = ringloop->get_sqe();
|
|
||||||
if (!sqe)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
ring_data_t* data = ((ring_data_t*)sqe->user_data);
|
|
||||||
if (!cl.write_op)
|
if (!cl.write_op)
|
||||||
{
|
{
|
||||||
// pick next command
|
// pick next command
|
||||||
|
@ -103,8 +97,26 @@ bool osd_messenger_t::try_send(osd_client_t & cl)
|
||||||
}
|
}
|
||||||
cl.write_msg.msg_iov = cl.send_list.get_iovec();
|
cl.write_msg.msg_iov = cl.send_list.get_iovec();
|
||||||
cl.write_msg.msg_iovlen = cl.send_list.get_size();
|
cl.write_msg.msg_iovlen = cl.send_list.get_size();
|
||||||
data->callback = [this, peer_fd](ring_data_t *data) { handle_send(data->res, peer_fd); };
|
if (ringloop && !use_sync_send_recv)
|
||||||
my_uring_prep_sendmsg(sqe, peer_fd, &cl.write_msg, 0);
|
{
|
||||||
|
io_uring_sqe* sqe = ringloop->get_sqe();
|
||||||
|
if (!sqe)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ring_data_t* data = ((ring_data_t*)sqe->user_data);
|
||||||
|
data->callback = [this, peer_fd](ring_data_t *data) { handle_send(data->res, peer_fd); };
|
||||||
|
my_uring_prep_sendmsg(sqe, peer_fd, &cl.write_msg, 0);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
int result = sendmsg(peer_fd, &cl.write_msg, MSG_NOSIGNAL);
|
||||||
|
if (result < 0)
|
||||||
|
{
|
||||||
|
result = -errno;
|
||||||
|
}
|
||||||
|
handle_send(result, peer_fd);
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue