From 522a9db0e2aaf5ccaab7ecbca742fc0021115e88 Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Thu, 9 Jan 2020 20:20:56 +0300
Subject: [PATCH] Enable TCP_NODELAY

---
 fio_sec_osd.cpp | 3 +++
 osd.cpp         | 5 +++++
 osd.h           | 1 +
 osd_send.cpp    | 1 +
 4 files changed, 10 insertions(+)

diff --git a/fio_sec_osd.cpp b/fio_sec_osd.cpp
index ac546648f..400c1c663 100644
--- a/fio_sec_osd.cpp
+++ b/fio_sec_osd.cpp
@@ -20,6 +20,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 
 #include <vector>
 #include <unordered_map>
@@ -136,6 +137,8 @@ static int sec_init(struct thread_data *td)
         perror("connect");
         return 1;
     }
+    int one = 1;
+    setsockopt(bsd->connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
 
     // FIXME: read config (block size) from OSD
 
diff --git a/osd.cpp b/osd.cpp
index c1fe45b71..aa55a07b5 100644
--- a/osd.cpp
+++ b/osd.cpp
@@ -2,6 +2,7 @@
 #include <sys/epoll.h>
 #include <sys/poll.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <arpa/inet.h>
 
 #include "osd.h"
@@ -197,6 +198,8 @@ void osd_t::handle_connect_result(int peer_fd)
         callback(-result);
         return;
     }
+    int one = 1;
+    setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
     // Disable EPOLLOUT on this fd
     cl.connect_callback = NULL;
     cl.peer_state = PEER_CONNECTED;
@@ -227,6 +230,8 @@ int osd_t::handle_epoll_events()
                 char peer_str[256];
                 printf("osd: new client %d: connection from %s port %d\n", peer_fd, inet_ntop(AF_INET, &addr.sin_addr, peer_str, 256), ntohs(addr.sin_port));
                 fcntl(peer_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
+                int one = 1;
+                setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
                 clients[peer_fd] = {
                     .peer_addr = addr,
                     .peer_port = ntohs(addr.sin_port),
diff --git a/osd.h b/osd.h
index 9b3c3dcd3..7879eaa10 100644
--- a/osd.h
+++ b/osd.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <sys/types.h>
+#include <sys/time.h>
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <fcntl.h>
diff --git a/osd_send.cpp b/osd_send.cpp
index 5685c3fc5..7b03a5a2b 100644
--- a/osd_send.cpp
+++ b/osd_send.cpp
@@ -35,6 +35,7 @@ void osd_t::send_replies()
         cl.write_iov.iov_len = cl.write_remaining;
         cl.write_msg.msg_iov = &cl.write_iov;
         cl.write_msg.msg_iovlen = 1;
+        // FIXME: This is basically a busy-loop. It's probably better to add epoll here
         data->callback = [this, peer_fd](ring_data_t *data) { handle_send(data, peer_fd); };
         my_uring_prep_sendmsg(sqe, peer_fd, &cl.write_msg, 0);
         cl.write_state = cl.write_state | SQE_SENT;