forked from vitalif/vitastor
Close client FDs only when destroying the client, after handling all async reads/writes
Fixes "Client XX command out of sync" sometimes happening on reconnections
parent
3f60fecd7c
commit
cdfc74665b
|
@ -251,6 +251,10 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
clients[peer_fd] = new osd_client_t();
|
clients[peer_fd] = new osd_client_t();
|
||||||
|
if (log_level > 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Connecting to OSD %lu at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
|
||||||
|
}
|
||||||
clients[peer_fd]->peer_addr = addr;
|
clients[peer_fd]->peer_addr = addr;
|
||||||
clients[peer_fd]->peer_port = peer_port;
|
clients[peer_fd]->peer_port = peer_port;
|
||||||
clients[peer_fd]->peer_fd = peer_fd;
|
clients[peer_fd]->peer_fd = peer_fd;
|
||||||
|
@ -313,7 +317,10 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
|
||||||
if (epoll_events & EPOLLRDHUP)
|
if (epoll_events & EPOLLRDHUP)
|
||||||
{
|
{
|
||||||
// Stop client
|
// Stop client
|
||||||
|
if (log_level > 0)
|
||||||
|
{
|
||||||
fprintf(stderr, "[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
|
fprintf(stderr, "[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
|
||||||
|
}
|
||||||
stop_client(peer_fd, true);
|
stop_client(peer_fd, true);
|
||||||
}
|
}
|
||||||
else if (epoll_events & EPOLLIN)
|
else if (epoll_events & EPOLLIN)
|
||||||
|
|
|
@ -50,7 +50,7 @@ struct osd_client_t
|
||||||
|
|
||||||
sockaddr_storage peer_addr;
|
sockaddr_storage peer_addr;
|
||||||
int peer_port;
|
int peer_port;
|
||||||
int peer_fd;
|
int peer_fd = -1;
|
||||||
int peer_state;
|
int peer_state;
|
||||||
int connect_timeout_id = -1;
|
int connect_timeout_id = -1;
|
||||||
int ping_time_remaining = 0;
|
int ping_time_remaining = 0;
|
||||||
|
@ -87,11 +87,7 @@ struct osd_client_t
|
||||||
std::vector<iovec> send_list, next_send_list;
|
std::vector<iovec> send_list, next_send_list;
|
||||||
std::vector<msgr_sendp_t> outbox, next_outbox;
|
std::vector<msgr_sendp_t> outbox, next_outbox;
|
||||||
|
|
||||||
~osd_client_t()
|
~osd_client_t();
|
||||||
{
|
|
||||||
free(in_buf);
|
|
||||||
in_buf = NULL;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct osd_wanted_peer_t
|
struct osd_wanted_peer_t
|
||||||
|
|
|
@ -122,17 +122,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
|
||||||
// Cancel outbound operations
|
// Cancel outbound operations
|
||||||
cancel_osd_ops(cl);
|
cancel_osd_ops(cl);
|
||||||
}
|
}
|
||||||
#ifndef __MOCK__
|
|
||||||
// And close the FD only when everything is done
|
|
||||||
// ...because peer_fd number can get reused after close()
|
|
||||||
close(peer_fd);
|
|
||||||
#ifdef WITH_RDMA
|
|
||||||
if (cl->rdma_conn)
|
|
||||||
{
|
|
||||||
delete cl->rdma_conn;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
// Find the item again because it can be invalidated at this point
|
// Find the item again because it can be invalidated at this point
|
||||||
it = clients.find(peer_fd);
|
it = clients.find(peer_fd);
|
||||||
if (it != clients.end())
|
if (it != clients.end())
|
||||||
|
@ -145,3 +134,25 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
|
||||||
delete cl;
|
delete cl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
osd_client_t::~osd_client_t()
|
||||||
|
{
|
||||||
|
free(in_buf);
|
||||||
|
in_buf = NULL;
|
||||||
|
if (peer_fd >= 0)
|
||||||
|
{
|
||||||
|
// Close the FD only when the client is actually destroyed
|
||||||
|
// Which only happens when all references are cleared
|
||||||
|
close(peer_fd);
|
||||||
|
peer_fd = -1;
|
||||||
|
}
|
||||||
|
#ifndef __MOCK__
|
||||||
|
#ifdef WITH_RDMA
|
||||||
|
if (rdma_conn)
|
||||||
|
{
|
||||||
|
delete rdma_conn;
|
||||||
|
rdma_conn = NULL;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue