Shiz01
/
vitastor
forked from vitalif/vitastor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Close client FDs only when destroying the client, after handling all async reads/writes 

Fixes "Client XX command out of sync" sometimes happening on reconnections
Vitaliy Filippov 2023-05-24 01:38:48 +03:00
parent 3f60fecd7c
commit cdfc74665b
3 changed files with 32 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/messenger.cpp
View File

@ -251,6 +251,10 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
return;
}
clients[peer_fd] = new osd_client_t();
if (log_level > 0)
{
fprintf(stderr, "Connecting to OSD %lu at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
}
clients[peer_fd]->peer_addr = addr;
clients[peer_fd]->peer_port = peer_port;
clients[peer_fd]->peer_fd = peer_fd;
@ -313,7 +317,10 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
if (epoll_events & EPOLLRDHUP)
{
// Stop client
fprintf(stderr, "[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
if (log_level > 0)
{
fprintf(stderr, "[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
}
stop_client(peer_fd, true);
}
else if (epoll_events & EPOLLIN)

8
src/messenger.h
View File

@ -50,7 +50,7 @@ struct osd_client_t
sockaddr_storage peer_addr;
int peer_port;
int peer_fd;
int peer_fd = -1;
int peer_state;
int connect_timeout_id = -1;
int ping_time_remaining = 0;
@ -87,11 +87,7 @@ struct osd_client_t
std::vector<iovec> send_list, next_send_list;
std::vector<msgr_sendp_t> outbox, next_outbox;
~osd_client_t()
{
free(in_buf);
in_buf = NULL;
}
~osd_client_t();
};
struct osd_wanted_peer_t

33
src/msgr_stop.cpp
View File

@ -122,17 +122,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
// Cancel outbound operations
cancel_osd_ops(cl);
}
#ifndef __MOCK__
// And close the FD only when everything is done
// ...because peer_fd number can get reused after close()
close(peer_fd);
#ifdef WITH_RDMA
if (cl->rdma_conn)
{
delete cl->rdma_conn;
}
#endif
#endif
// Find the item again because it can be invalidated at this point
it = clients.find(peer_fd);
if (it != clients.end())
@ -145,3 +134,25 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
delete cl;
}
}
osd_client_t::~osd_client_t()
{
free(in_buf);
in_buf = NULL;
if (peer_fd >= 0)
{
// Close the FD only when the client is actually destroyed
// Which only happens when all references are cleared
close(peer_fd);
peer_fd = -1;
}
#ifndef __MOCK__
#ifdef WITH_RDMA
if (rdma_conn)
{
delete rdma_conn;
rdma_conn = NULL;
}
#endif
#endif
}