Enable TCP_NODELAY

fio_sec_osd.cpp

@@ -20,6 +20,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 
 #include <vector>
 #include <unordered_map>
@@ -136,6 +137,8 @@ static int sec_init(struct thread_data *td)
         perror("connect");
         return 1;
     }
+    int one = 1;
+    setsockopt(bsd->connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
 
     // FIXME: read config (block size) from OSD
 

osd.cpp

@@ -2,6 +2,7 @@
 #include <sys/epoll.h>
 #include <sys/poll.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <arpa/inet.h>
 
 #include "osd.h"
@@ -197,6 +198,8 @@ void osd_t::handle_connect_result(int peer_fd)
         callback(-result);
         return;
     }
+    int one = 1;
+    setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
     // Disable EPOLLOUT on this fd
     cl.connect_callback = NULL;
     cl.peer_state = PEER_CONNECTED;
@@ -227,6 +230,8 @@ int osd_t::handle_epoll_events()
                 char peer_str[256];
                 printf("osd: new client %d: connection from %s port %d\n", peer_fd, inet_ntop(AF_INET, &addr.sin_addr, peer_str, 256), ntohs(addr.sin_port));
                 fcntl(peer_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
+                int one = 1;
+                setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
                 clients[peer_fd] = {
                     .peer_addr = addr,
                     .peer_port = ntohs(addr.sin_port),

osd.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <sys/types.h>
+#include <sys/time.h>
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <fcntl.h>

osd_send.cpp

@@ -35,6 +35,7 @@ void osd_t::send_replies()
         cl.write_iov.iov_len = cl.write_remaining;
         cl.write_msg.msg_iov = &cl.write_iov;
         cl.write_msg.msg_iovlen = 1;
+        // FIXME: This is basically a busy-loop. It's probably better to add epoll here
         data->callback = [this, peer_fd](ring_data_t *data) { handle_send(data, peer_fd); };
         my_uring_prep_sendmsg(sqe, peer_fd, &cl.write_msg, 0);
         cl.write_state = cl.write_state | SQE_SENT;