forked from vitalif/vitastor
Change default RDMA settings to 128x 129K buffers
129K to leave extra space for the header The problem with 8x 1M buffers is that the following happens with, for example, 2 OSDs and 4M T1Q1 write: - Server posts 8 receives - Client posts 8 sends - WRs are processed by the RDMA stack, but the OSD doesn't have the time to handle them and doesn't refill buffers - Client posts 1 more send - RNR retransmission happens and performance drops to zero Overall it seems that RDMA support should be reworked to use real 'RDMA' operations i.e. operations writing into remote memory. This has an additional advantage of avoiding a copy at the receive side of the OSD.nbd-vmsplice
parent
f0ebfae3b8
commit
660c3f7b0d
|
@ -147,13 +147,13 @@ void osd_messenger_t::parse_config(const json11::Json & config)
|
||||||
this->rdma_max_sge = 128;
|
this->rdma_max_sge = 128;
|
||||||
this->rdma_max_send = config["rdma_max_send"].uint64_value();
|
this->rdma_max_send = config["rdma_max_send"].uint64_value();
|
||||||
if (!this->rdma_max_send)
|
if (!this->rdma_max_send)
|
||||||
this->rdma_max_send = 32;
|
this->rdma_max_send = 1;
|
||||||
this->rdma_max_recv = config["rdma_max_recv"].uint64_value();
|
this->rdma_max_recv = config["rdma_max_recv"].uint64_value();
|
||||||
if (!this->rdma_max_recv)
|
if (!this->rdma_max_recv)
|
||||||
this->rdma_max_recv = 8;
|
this->rdma_max_recv = 128;
|
||||||
this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
|
this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
|
||||||
if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
|
if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
|
||||||
this->rdma_max_msg = 1024*1024;
|
this->rdma_max_msg = 129*1024;
|
||||||
#endif
|
#endif
|
||||||
this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value();
|
this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value();
|
||||||
if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024)
|
if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024)
|
||||||
|
|
|
@ -133,7 +133,7 @@ protected:
|
||||||
std::string rdma_device;
|
std::string rdma_device;
|
||||||
uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
|
uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
|
||||||
msgr_rdma_context_t *rdma_context = NULL;
|
msgr_rdma_context_t *rdma_context = NULL;
|
||||||
uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 8;
|
uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
|
||||||
uint64_t rdma_max_msg = 0;
|
uint64_t rdma_max_msg = 0;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue