Close client FDs only when destroying the client, after handling all async reads/writes

Fixes "Client XX command out of sync" sometimes happening on reconnections

src/messenger.cpp

@@ -251,6 +251,10 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
         return;
     }
     clients[peer_fd] = new osd_client_t();
+    if (log_level > 0)
+    {
+        fprintf(stderr, "Connecting to OSD %lu at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
+    }
     clients[peer_fd]->peer_addr = addr;
     clients[peer_fd]->peer_port = peer_port;
     clients[peer_fd]->peer_fd = peer_fd;
@@ -313,7 +317,10 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
     if (epoll_events & EPOLLRDHUP)
     {
         // Stop client
-        fprintf(stderr, "[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
+        if (log_level > 0)
+        {
+            fprintf(stderr, "[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
+        }
         stop_client(peer_fd, true);
     }
     else if (epoll_events & EPOLLIN)

src/messenger.h

@@ -50,7 +50,7 @@ struct osd_client_t
 
     sockaddr_storage peer_addr;
     int peer_port;
-    int peer_fd;
+    int peer_fd = -1;
     int peer_state;
     int connect_timeout_id = -1;
     int ping_time_remaining = 0;
@@ -87,11 +87,7 @@ struct osd_client_t
     std::vector<iovec> send_list, next_send_list;
     std::vector<msgr_sendp_t> outbox, next_outbox;
 
-    ~osd_client_t()
-    {
-        free(in_buf);
-        in_buf = NULL;
-    }
+    ~osd_client_t();
 };
 
 struct osd_wanted_peer_t

src/msgr_stop.cpp

@@ -122,17 +122,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         // Cancel outbound operations
         cancel_osd_ops(cl);
     }
-#ifndef __MOCK__
-    // And close the FD only when everything is done
-    // ...because peer_fd number can get reused after close()
-    close(peer_fd);
-#ifdef WITH_RDMA
-    if (cl->rdma_conn)
-    {
-        delete cl->rdma_conn;
-    }
-#endif
-#endif
     // Find the item again because it can be invalidated at this point
     it = clients.find(peer_fd);
     if (it != clients.end())
@@ -145,3 +134,25 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         delete cl;
     }
 }
+
+osd_client_t::~osd_client_t()
+{
+    free(in_buf);
+    in_buf = NULL;
+    if (peer_fd >= 0)
+    {
+        // Close the FD only when the client is actually destroyed
+        // Which only happens when all references are cleared
+        close(peer_fd);
+        peer_fd = -1;
+    }
+#ifndef __MOCK__
+#ifdef WITH_RDMA
+    if (rdma_conn)
+    {
+        delete rdma_conn;
+        rdma_conn = NULL;
+    }
+#endif
+#endif
+}