2019-12-11 14:18:19 +03:00
|
|
|
#include <sys/socket.h>
|
|
|
|
#include <sys/epoll.h>
|
2020-05-30 10:24:50 +03:00
|
|
|
#include <sys/eventfd.h>
|
2019-12-14 20:51:41 +03:00
|
|
|
#include <sys/poll.h>
|
2019-12-11 14:18:19 +03:00
|
|
|
#include <netinet/in.h>
|
2020-01-09 20:20:56 +03:00
|
|
|
#include <netinet/tcp.h>
|
2019-12-11 14:18:19 +03:00
|
|
|
#include <arpa/inet.h>
|
|
|
|
|
2019-12-15 01:11:51 +03:00
|
|
|
#include "osd.h"
|
2019-12-11 14:18:19 +03:00
|
|
|
|
2020-06-07 00:29:09 +03:00
|
|
|
#define MAX_EPOLL_EVENTS 64
|
|
|
|
|
2020-03-31 17:50:50 +03:00
|
|
|
const char* osd_op_names[] = {
|
2020-02-28 12:25:54 +03:00
|
|
|
"",
|
|
|
|
"read",
|
|
|
|
"write",
|
|
|
|
"sync",
|
|
|
|
"stabilize",
|
|
|
|
"rollback",
|
|
|
|
"delete",
|
|
|
|
"sync_stab_all",
|
|
|
|
"list",
|
|
|
|
"show_config",
|
|
|
|
"primary_read",
|
|
|
|
"primary_write",
|
|
|
|
"primary_sync",
|
2020-05-05 00:16:01 +03:00
|
|
|
"primary_delete",
|
2020-02-28 12:25:54 +03:00
|
|
|
};
|
|
|
|
|
2019-12-15 14:49:10 +03:00
|
|
|
osd_t::osd_t(blockstore_config_t & config, blockstore_t *bs, ring_loop_t *ringloop)
|
2019-12-13 20:12:31 +03:00
|
|
|
{
|
2020-01-21 14:04:58 +03:00
|
|
|
this->config = config;
|
|
|
|
this->bs = bs;
|
|
|
|
this->ringloop = ringloop;
|
2020-04-02 22:16:46 +03:00
|
|
|
|
2020-01-30 22:06:46 +03:00
|
|
|
this->bs_block_size = bs->get_block_size();
|
|
|
|
// FIXME: use bitmap granularity instead
|
|
|
|
this->bs_disk_alignment = bs->get_disk_alignment();
|
2020-01-21 14:04:58 +03:00
|
|
|
|
2020-04-02 22:16:46 +03:00
|
|
|
parse_config(config);
|
|
|
|
|
2020-04-20 17:44:03 +03:00
|
|
|
epoll_fd = epoll_create(1);
|
|
|
|
if (epoll_fd < 0)
|
|
|
|
{
|
|
|
|
throw std::runtime_error(std::string("epoll_create: ") + strerror(errno));
|
|
|
|
}
|
2020-05-30 10:24:50 +03:00
|
|
|
event_fd = eventfd(0, EFD_NONBLOCK);
|
|
|
|
if (event_fd < 0)
|
|
|
|
{
|
|
|
|
throw std::runtime_error(std::string("eventfd: ") + strerror(errno));
|
|
|
|
}
|
2020-04-02 22:16:46 +03:00
|
|
|
|
2020-05-30 00:07:44 +03:00
|
|
|
this->tfd = new timerfd_manager_t(ringloop);
|
|
|
|
this->tfd->set_fd_handler = [this](int fd, std::function<void(int, int)> handler) { set_fd_handler(fd, handler); };
|
2020-04-24 01:07:19 +03:00
|
|
|
this->tfd->set_timer(print_stats_interval*1000, true, [this](int timer_id)
|
2020-04-02 22:16:46 +03:00
|
|
|
{
|
|
|
|
print_stats();
|
|
|
|
});
|
|
|
|
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.tfd = this->tfd;
|
|
|
|
c_cli.ringloop = this->ringloop;
|
|
|
|
c_cli.exec_op = [this](osd_op_t *op) { exec_op(op); };
|
|
|
|
c_cli.repeer_pgs = [this](osd_num_t peer_osd) { repeer_pgs(peer_osd); };
|
|
|
|
|
2020-04-17 01:59:06 +03:00
|
|
|
init_cluster();
|
|
|
|
|
2020-04-02 22:16:46 +03:00
|
|
|
consumer.loop = [this]() { loop(); };
|
|
|
|
ringloop->register_consumer(&consumer);
|
2020-05-30 10:24:50 +03:00
|
|
|
epoll_thread = new std::thread([this]()
|
|
|
|
{
|
|
|
|
int nfds;
|
|
|
|
epoll_event events[MAX_EPOLL_EVENTS];
|
|
|
|
while (1)
|
|
|
|
{
|
|
|
|
nfds = epoll_wait(epoll_fd, events, MAX_EPOLL_EVENTS, -1);
|
|
|
|
{
|
|
|
|
std::lock_guard<std::mutex> guard(epoll_mutex);
|
|
|
|
for (int i = 0; i < nfds; i++)
|
|
|
|
{
|
|
|
|
int fd = events[i].data.fd;
|
|
|
|
int ev = events[i].events;
|
|
|
|
epoll_ready[fd] |= ev;
|
|
|
|
}
|
|
|
|
uint64_t n = 1;
|
|
|
|
write(event_fd, &n, 8);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
osd_t::~osd_t()
|
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
close(epoll_fd);
|
|
|
|
epoll_thread->join();
|
|
|
|
delete epoll_thread;
|
2020-04-15 15:33:18 +03:00
|
|
|
if (tfd)
|
|
|
|
{
|
|
|
|
delete tfd;
|
|
|
|
tfd = NULL;
|
|
|
|
}
|
2020-04-02 22:16:46 +03:00
|
|
|
ringloop->unregister_consumer(&consumer);
|
2020-05-30 10:24:50 +03:00
|
|
|
close(event_fd);
|
2020-04-02 22:16:46 +03:00
|
|
|
close(listen_fd);
|
|
|
|
}
|
|
|
|
|
|
|
|
void osd_t::parse_config(blockstore_config_t & config)
|
|
|
|
{
|
2020-04-20 17:44:03 +03:00
|
|
|
// Initial startup configuration
|
2020-06-06 16:32:35 +03:00
|
|
|
json11::Json json_config = json11::Json(config);
|
|
|
|
st_cli.parse_config(json_config);
|
2020-04-24 01:03:55 +03:00
|
|
|
etcd_report_interval = strtoull(config["etcd_report_interval"].c_str(), NULL, 10);
|
|
|
|
if (etcd_report_interval <= 0)
|
|
|
|
etcd_report_interval = 30;
|
2020-01-21 14:04:58 +03:00
|
|
|
osd_num = strtoull(config["osd_num"].c_str(), NULL, 10);
|
|
|
|
if (!osd_num)
|
|
|
|
throw std::runtime_error("osd_num is required in the configuration");
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.osd_num = osd_num;
|
2020-04-20 17:44:03 +03:00
|
|
|
run_primary = config["run_primary"] != "false" && config["run_primary"] != "0" && config["run_primary"] != "no";
|
|
|
|
// Cluster configuration
|
|
|
|
bind_address = config["bind_address"];
|
|
|
|
if (bind_address == "")
|
|
|
|
bind_address = "0.0.0.0";
|
|
|
|
bind_port = stoull_full(config["bind_port"]);
|
|
|
|
if (bind_port <= 0 || bind_port > 65535)
|
|
|
|
bind_port = 0;
|
2020-03-10 02:05:32 +03:00
|
|
|
if (config["immediate_commit"] == "all")
|
|
|
|
immediate_commit = IMMEDIATE_ALL;
|
|
|
|
else if (config["immediate_commit"] == "small")
|
|
|
|
immediate_commit = IMMEDIATE_SMALL;
|
2020-05-23 15:11:33 +03:00
|
|
|
if (config.find("autosync_interval") != config.end())
|
|
|
|
{
|
|
|
|
autosync_interval = strtoull(config["autosync_interval"].c_str(), NULL, 10);
|
|
|
|
if (autosync_interval > MAX_AUTOSYNC_INTERVAL)
|
|
|
|
autosync_interval = DEFAULT_AUTOSYNC_INTERVAL;
|
|
|
|
}
|
2020-05-25 15:28:32 +03:00
|
|
|
if (config.find("client_queue_depth") != config.end())
|
|
|
|
{
|
|
|
|
client_queue_depth = strtoull(config["client_queue_depth"].c_str(), NULL, 10);
|
|
|
|
if (client_queue_depth < 128)
|
|
|
|
client_queue_depth = 128;
|
|
|
|
}
|
|
|
|
if (config.find("pg_stripe_size") != config.end())
|
|
|
|
{
|
|
|
|
pg_stripe_size = strtoull(config["pg_stripe_size"].c_str(), NULL, 10);
|
|
|
|
if (!pg_stripe_size || !bs_block_size || pg_stripe_size < bs_block_size || (pg_stripe_size % bs_block_size) != 0)
|
|
|
|
pg_stripe_size = DEFAULT_PG_STRIPE_SIZE;
|
|
|
|
}
|
2020-04-04 02:18:29 +03:00
|
|
|
recovery_queue_depth = strtoull(config["recovery_queue_depth"].c_str(), NULL, 10);
|
|
|
|
if (recovery_queue_depth < 1 || recovery_queue_depth > MAX_RECOVERY_QUEUE)
|
|
|
|
recovery_queue_depth = DEFAULT_RECOVERY_QUEUE;
|
2020-04-11 12:05:28 +03:00
|
|
|
if (config["readonly"] == "true" || config["readonly"] == "1" || config["readonly"] == "yes")
|
|
|
|
readonly = true;
|
2020-04-14 14:37:50 +03:00
|
|
|
print_stats_interval = strtoull(config["print_stats_interval"].c_str(), NULL, 10);
|
|
|
|
if (!print_stats_interval)
|
|
|
|
print_stats_interval = 3;
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.peer_connect_interval = strtoull(config["peer_connect_interval"].c_str(), NULL, 10);
|
|
|
|
if (!c_cli.peer_connect_interval)
|
2020-06-06 01:39:58 +03:00
|
|
|
c_cli.peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.peer_connect_timeout = strtoull(config["peer_connect_timeout"].c_str(), NULL, 10);
|
|
|
|
if (!c_cli.peer_connect_timeout)
|
2020-06-06 01:39:58 +03:00
|
|
|
c_cli.peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
2020-04-27 14:32:59 +03:00
|
|
|
log_level = strtoull(config["log_level"].c_str(), NULL, 10);
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.log_level = log_level;
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
2019-12-11 14:18:19 +03:00
|
|
|
|
2020-04-02 22:16:46 +03:00
|
|
|
void osd_t::bind_socket()
|
|
|
|
{
|
2019-12-11 14:18:19 +03:00
|
|
|
listen_fd = socket(AF_INET, SOCK_STREAM, 0);
|
|
|
|
if (listen_fd < 0)
|
|
|
|
{
|
|
|
|
throw std::runtime_error(std::string("socket: ") + strerror(errno));
|
|
|
|
}
|
2019-12-13 14:05:11 +03:00
|
|
|
int enable = 1;
|
|
|
|
setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable));
|
2019-12-11 14:18:19 +03:00
|
|
|
|
|
|
|
sockaddr_in addr;
|
2019-12-14 20:51:41 +03:00
|
|
|
int r;
|
|
|
|
if ((r = inet_pton(AF_INET, bind_address.c_str(), &addr.sin_addr)) != 1)
|
2019-12-11 14:18:19 +03:00
|
|
|
{
|
|
|
|
close(listen_fd);
|
|
|
|
throw std::runtime_error("bind address "+bind_address+(r == 0 ? " is not valid" : ": no ipv4 support"));
|
|
|
|
}
|
|
|
|
addr.sin_family = AF_INET;
|
|
|
|
|
2020-04-21 00:10:59 +03:00
|
|
|
addr.sin_port = htons(bind_port);
|
|
|
|
if (bind(listen_fd, (sockaddr*)&addr, sizeof(addr)) < 0)
|
2019-12-11 14:18:19 +03:00
|
|
|
{
|
2020-04-21 00:10:59 +03:00
|
|
|
close(listen_fd);
|
|
|
|
throw std::runtime_error(std::string("bind: ") + strerror(errno));
|
|
|
|
}
|
|
|
|
if (bind_port == 0)
|
|
|
|
{
|
|
|
|
socklen_t len = sizeof(addr);
|
|
|
|
if (getsockname(listen_fd, (sockaddr *)&addr, &len) == -1)
|
2020-04-20 17:44:03 +03:00
|
|
|
{
|
|
|
|
close(listen_fd);
|
2020-04-21 00:10:59 +03:00
|
|
|
throw std::runtime_error(std::string("getsockname: ") + strerror(errno));
|
2020-04-20 17:44:03 +03:00
|
|
|
}
|
2020-04-21 00:10:59 +03:00
|
|
|
listening_port = ntohs(addr.sin_port);
|
2020-04-20 17:44:03 +03:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2020-04-21 00:10:59 +03:00
|
|
|
listening_port = bind_port;
|
2019-12-11 14:18:19 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if (listen(listen_fd, listen_backlog) < 0)
|
|
|
|
{
|
|
|
|
close(listen_fd);
|
|
|
|
throw std::runtime_error(std::string("listen: ") + strerror(errno));
|
|
|
|
}
|
|
|
|
|
2019-12-12 11:32:20 +03:00
|
|
|
fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
|
|
|
|
|
2019-12-13 22:53:59 +03:00
|
|
|
epoll_event ev;
|
2019-12-11 14:18:19 +03:00
|
|
|
ev.data.fd = listen_fd;
|
2020-02-29 01:46:12 +03:00
|
|
|
ev.events = EPOLLIN | EPOLLET;
|
2019-12-12 11:32:20 +03:00
|
|
|
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, listen_fd, &ev) < 0)
|
2019-12-11 14:18:19 +03:00
|
|
|
{
|
2020-01-09 18:39:58 +03:00
|
|
|
close(listen_fd);
|
|
|
|
close(epoll_fd);
|
2020-05-30 10:24:50 +03:00
|
|
|
throw std::runtime_error(std::string("epoll_ctl (add listen_fd): ") + strerror(errno));
|
2019-12-11 14:18:19 +03:00
|
|
|
}
|
2020-05-30 10:24:50 +03:00
|
|
|
|
|
|
|
epoll_handlers[listen_fd] = [this](int peer_fd, int epoll_events)
|
|
|
|
{
|
|
|
|
c_cli.accept_connections(listen_fd);
|
|
|
|
};
|
2019-12-23 21:56:03 +03:00
|
|
|
}
|
|
|
|
|
2019-12-15 01:52:08 +03:00
|
|
|
bool osd_t::shutdown()
|
|
|
|
{
|
2019-12-19 22:16:04 +03:00
|
|
|
stopping = true;
|
|
|
|
if (inflight_ops > 0)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return bs->is_safe_to_stop();
|
2019-12-15 01:52:08 +03:00
|
|
|
}
|
|
|
|
|
2019-12-11 14:18:19 +03:00
|
|
|
void osd_t::loop()
|
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
std::map<int,int> cur_epoll;
|
2019-12-16 21:37:13 +03:00
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
std::lock_guard<std::mutex> guard(epoll_mutex);
|
|
|
|
cur_epoll.swap(epoll_ready);
|
|
|
|
}
|
|
|
|
for (auto p: cur_epoll)
|
|
|
|
{
|
|
|
|
auto cb_it = epoll_handlers.find(p.first);
|
|
|
|
if (cb_it != epoll_handlers.end())
|
|
|
|
{
|
|
|
|
cb_it->second(p.first, p.second);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!(wait_state & 2))
|
|
|
|
{
|
|
|
|
handle_eventfd();
|
|
|
|
wait_state = wait_state | 2;
|
2019-12-16 21:37:13 +03:00
|
|
|
}
|
2020-01-21 14:04:58 +03:00
|
|
|
handle_peers();
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.read_requests();
|
|
|
|
c_cli.send_replies();
|
2019-12-11 14:18:19 +03:00
|
|
|
ringloop->submit();
|
|
|
|
}
|
|
|
|
|
2020-05-21 01:23:43 +03:00
|
|
|
void osd_t::set_fd_handler(int fd, std::function<void(int, int)> handler)
|
|
|
|
{
|
|
|
|
if (handler != NULL)
|
|
|
|
{
|
2020-05-26 21:23:01 +03:00
|
|
|
bool exists = epoll_handlers.find(fd) != epoll_handlers.end();
|
2020-05-21 01:23:43 +03:00
|
|
|
epoll_event ev;
|
|
|
|
ev.data.fd = fd;
|
|
|
|
ev.events = EPOLLOUT | EPOLLIN | EPOLLRDHUP | EPOLLET;
|
2020-05-26 21:23:01 +03:00
|
|
|
if (epoll_ctl(epoll_fd, exists ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev) < 0)
|
2020-05-21 01:23:43 +03:00
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
throw std::runtime_error(std::string(exists ? "epoll_ctl (mod fd): " : "epoll_ctl (add fd): ") + strerror(errno));
|
2020-05-21 01:23:43 +03:00
|
|
|
}
|
|
|
|
epoll_handlers[fd] = handler;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2020-05-23 15:01:47 +03:00
|
|
|
if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, NULL) < 0 && errno != ENOENT)
|
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
throw std::runtime_error(std::string("epoll_ctl (remove fd): ") + strerror(errno));
|
2020-05-23 15:01:47 +03:00
|
|
|
}
|
2020-05-21 01:23:43 +03:00
|
|
|
epoll_handlers.erase(fd);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-30 10:24:50 +03:00
|
|
|
void osd_t::handle_eventfd()
|
2019-12-11 14:18:19 +03:00
|
|
|
{
|
2020-03-03 01:47:56 +03:00
|
|
|
io_uring_sqe *sqe = ringloop->get_sqe();
|
|
|
|
if (!sqe)
|
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
throw std::runtime_error("can't get SQE, will fall out of sync with eventfd");
|
2020-03-03 01:47:56 +03:00
|
|
|
}
|
|
|
|
ring_data_t *data = ((ring_data_t*)sqe->user_data);
|
2020-05-30 10:24:50 +03:00
|
|
|
my_uring_prep_poll_add(sqe, event_fd, POLLIN);
|
2020-03-03 01:47:56 +03:00
|
|
|
data->callback = [this](ring_data_t *data)
|
|
|
|
{
|
|
|
|
if (data->res < 0)
|
|
|
|
{
|
|
|
|
throw std::runtime_error(std::string("epoll failed: ") + strerror(-data->res));
|
|
|
|
}
|
2020-05-30 10:24:50 +03:00
|
|
|
handle_eventfd();
|
2020-03-03 01:47:56 +03:00
|
|
|
};
|
|
|
|
ringloop->submit();
|
2020-05-30 10:24:50 +03:00
|
|
|
uint64_t n = 0;
|
|
|
|
size_t res = read(event_fd, &n, 8);
|
|
|
|
if (res == 8)
|
2020-02-29 01:46:12 +03:00
|
|
|
{
|
2020-05-30 10:24:50 +03:00
|
|
|
// No need to do anything, the loop has already woken up
|
|
|
|
ringloop->wakeup();
|
2020-02-29 01:46:12 +03:00
|
|
|
}
|
2019-12-11 14:18:19 +03:00
|
|
|
}
|
2019-12-13 14:05:11 +03:00
|
|
|
|
2019-12-28 01:25:55 +03:00
|
|
|
void osd_t::exec_op(osd_op_t *cur_op)
|
2019-12-15 15:30:51 +03:00
|
|
|
{
|
2020-03-03 00:54:42 +03:00
|
|
|
clock_gettime(CLOCK_REALTIME, &cur_op->tv_begin);
|
2019-12-19 22:16:04 +03:00
|
|
|
if (stopping)
|
|
|
|
{
|
|
|
|
// Throw operation away
|
|
|
|
delete cur_op;
|
|
|
|
return;
|
|
|
|
}
|
2020-04-03 13:03:42 +03:00
|
|
|
inflight_ops++;
|
2020-02-29 02:27:10 +03:00
|
|
|
cur_op->send_list.push_back(cur_op->reply.buf, OSD_PACKET_SIZE);
|
2020-02-23 19:03:06 +03:00
|
|
|
if (cur_op->req.hdr.magic != SECONDARY_OSD_OP_MAGIC ||
|
|
|
|
cur_op->req.hdr.opcode < OSD_OP_MIN || cur_op->req.hdr.opcode > OSD_OP_MAX ||
|
2020-02-25 01:20:45 +03:00
|
|
|
(cur_op->req.hdr.opcode == OSD_OP_SECONDARY_READ || cur_op->req.hdr.opcode == OSD_OP_SECONDARY_WRITE) &&
|
2020-04-14 19:19:56 +03:00
|
|
|
(cur_op->req.sec_rw.len > OSD_RW_MAX || cur_op->req.sec_rw.len % bs_disk_alignment || cur_op->req.sec_rw.offset % bs_disk_alignment) ||
|
2020-05-05 00:16:01 +03:00
|
|
|
(cur_op->req.hdr.opcode == OSD_OP_READ || cur_op->req.hdr.opcode == OSD_OP_WRITE || cur_op->req.hdr.opcode == OSD_OP_DELETE) &&
|
2020-04-14 19:19:56 +03:00
|
|
|
(cur_op->req.rw.len > OSD_RW_MAX || cur_op->req.rw.len % bs_disk_alignment || cur_op->req.rw.offset % bs_disk_alignment))
|
2019-12-13 20:12:31 +03:00
|
|
|
{
|
|
|
|
// Bad command
|
2020-03-31 17:50:50 +03:00
|
|
|
finish_op(cur_op, -EINVAL);
|
2019-12-15 15:30:51 +03:00
|
|
|
return;
|
|
|
|
}
|
2020-05-01 00:54:34 +03:00
|
|
|
if (readonly &&
|
|
|
|
cur_op->req.hdr.opcode != OSD_OP_SECONDARY_READ &&
|
|
|
|
cur_op->req.hdr.opcode != OSD_OP_SECONDARY_LIST &&
|
|
|
|
cur_op->req.hdr.opcode != OSD_OP_READ &&
|
|
|
|
cur_op->req.hdr.opcode != OSD_OP_SHOW_CONFIG)
|
|
|
|
{
|
|
|
|
// Readonly mode
|
|
|
|
finish_op(cur_op, -EROFS);
|
|
|
|
return;
|
|
|
|
}
|
2020-02-23 19:03:06 +03:00
|
|
|
if (cur_op->req.hdr.opcode == OSD_OP_TEST_SYNC_STAB_ALL)
|
2019-12-15 15:30:51 +03:00
|
|
|
{
|
2019-12-28 01:25:55 +03:00
|
|
|
exec_sync_stab_all(cur_op);
|
2019-12-13 20:12:31 +03:00
|
|
|
}
|
2020-02-23 19:03:06 +03:00
|
|
|
else if (cur_op->req.hdr.opcode == OSD_OP_SHOW_CONFIG)
|
2019-12-23 21:56:03 +03:00
|
|
|
{
|
2019-12-28 01:25:55 +03:00
|
|
|
exec_show_config(cur_op);
|
2019-12-23 21:56:03 +03:00
|
|
|
}
|
2020-02-23 19:03:06 +03:00
|
|
|
else if (cur_op->req.hdr.opcode == OSD_OP_READ)
|
2019-12-26 14:06:03 +03:00
|
|
|
{
|
2020-02-24 02:40:48 +03:00
|
|
|
continue_primary_read(cur_op);
|
2020-01-30 22:06:46 +03:00
|
|
|
}
|
2020-02-23 19:03:06 +03:00
|
|
|
else if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
|
2020-01-30 22:06:46 +03:00
|
|
|
{
|
2020-02-24 02:40:48 +03:00
|
|
|
continue_primary_write(cur_op);
|
2019-12-26 14:06:03 +03:00
|
|
|
}
|
2020-02-25 20:10:17 +03:00
|
|
|
else if (cur_op->req.hdr.opcode == OSD_OP_SYNC)
|
|
|
|
{
|
|
|
|
continue_primary_sync(cur_op);
|
|
|
|
}
|
2020-05-05 00:16:01 +03:00
|
|
|
else if (cur_op->req.hdr.opcode == OSD_OP_DELETE)
|
|
|
|
{
|
|
|
|
continue_primary_del(cur_op);
|
|
|
|
}
|
2019-12-23 21:56:03 +03:00
|
|
|
else
|
|
|
|
{
|
2019-12-28 01:25:55 +03:00
|
|
|
exec_secondary(cur_op);
|
2019-12-13 14:05:11 +03:00
|
|
|
}
|
|
|
|
}
|
2020-04-02 22:16:46 +03:00
|
|
|
|
2020-04-14 14:37:50 +03:00
|
|
|
void osd_t::reset_stats()
|
2020-04-02 22:16:46 +03:00
|
|
|
{
|
2020-05-26 21:23:01 +03:00
|
|
|
c_cli.stats = { 0 };
|
|
|
|
prev_stats = { 0 };
|
|
|
|
memset(recovery_stat_count, 0, sizeof(recovery_stat_count));
|
|
|
|
memset(recovery_stat_bytes, 0, sizeof(recovery_stat_bytes));
|
2020-04-14 14:37:50 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void osd_t::print_stats()
|
|
|
|
{
|
2020-04-02 22:16:46 +03:00
|
|
|
for (int i = 0; i <= OSD_OP_MAX; i++)
|
|
|
|
{
|
2020-05-26 21:23:01 +03:00
|
|
|
if (c_cli.stats.op_stat_count[i] != prev_stats.op_stat_count[i])
|
2020-04-02 22:16:46 +03:00
|
|
|
{
|
2020-05-26 21:23:01 +03:00
|
|
|
uint64_t avg = (c_cli.stats.op_stat_sum[i] - prev_stats.op_stat_sum[i])/(c_cli.stats.op_stat_count[i] - prev_stats.op_stat_count[i]);
|
|
|
|
uint64_t bw = (c_cli.stats.op_stat_bytes[i] - prev_stats.op_stat_bytes[i]) / print_stats_interval;
|
|
|
|
if (c_cli.stats.op_stat_bytes[i] != 0)
|
2020-05-23 15:19:51 +03:00
|
|
|
{
|
|
|
|
printf(
|
|
|
|
"[OSD %lu] avg latency for op %d (%s): %lu us, B/W: %.2f %s\n", osd_num, i, osd_op_names[i], avg,
|
|
|
|
(bw > 1024*1024*1024 ? bw/1024.0/1024/1024 : (bw > 1024*1024 ? bw/1024.0/1024 : bw/1024.0)),
|
|
|
|
(bw > 1024*1024*1024 ? "GB/s" : (bw > 1024*1024 ? "MB/s" : "KB/s"))
|
|
|
|
);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
printf("[OSD %lu] avg latency for op %d (%s): %lu us\n", osd_num, i, osd_op_names[i], avg);
|
|
|
|
}
|
2020-05-26 21:23:01 +03:00
|
|
|
prev_stats.op_stat_count[i] = c_cli.stats.op_stat_count[i];
|
|
|
|
prev_stats.op_stat_sum[i] = c_cli.stats.op_stat_sum[i];
|
|
|
|
prev_stats.op_stat_bytes[i] = c_cli.stats.op_stat_bytes[i];
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
|
|
|
}
|
2020-04-14 14:37:50 +03:00
|
|
|
for (int i = 0; i <= OSD_OP_MAX; i++)
|
2020-04-02 22:16:46 +03:00
|
|
|
{
|
2020-05-26 21:23:01 +03:00
|
|
|
if (c_cli.stats.subop_stat_count[i] != prev_stats.subop_stat_count[i])
|
2020-04-14 14:37:50 +03:00
|
|
|
{
|
2020-05-26 21:23:01 +03:00
|
|
|
uint64_t avg = (c_cli.stats.subop_stat_sum[i] - prev_stats.subop_stat_sum[i])/(c_cli.stats.subop_stat_count[i] - prev_stats.subop_stat_count[i]);
|
2020-05-23 15:19:51 +03:00
|
|
|
printf("[OSD %lu] avg latency for subop %d (%s): %ld us\n", osd_num, i, osd_op_names[i], avg);
|
2020-05-26 21:23:01 +03:00
|
|
|
prev_stats.subop_stat_count[i] = c_cli.stats.subop_stat_count[i];
|
|
|
|
prev_stats.subop_stat_sum[i] = c_cli.stats.subop_stat_sum[i];
|
2020-04-14 14:37:50 +03:00
|
|
|
}
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
2020-05-11 16:12:18 +03:00
|
|
|
for (int i = 0; i < 2; i++)
|
|
|
|
{
|
|
|
|
if (recovery_stat_count[0][i] != recovery_stat_count[1][i])
|
|
|
|
{
|
|
|
|
uint64_t bw = (recovery_stat_bytes[0][i] - recovery_stat_bytes[1][i]) / print_stats_interval;
|
|
|
|
printf(
|
2020-05-23 15:19:51 +03:00
|
|
|
"[OSD %lu] %s recovery: %.1f op/s, B/W: %.2f %s\n", osd_num, recovery_stat_names[i],
|
2020-05-11 16:12:18 +03:00
|
|
|
(recovery_stat_count[0][i] - recovery_stat_count[1][i]) * 1.0 / print_stats_interval,
|
|
|
|
(bw > 1024*1024*1024 ? bw/1024.0/1024/1024 : (bw > 1024*1024 ? bw/1024.0/1024 : bw/1024.0)),
|
|
|
|
(bw > 1024*1024*1024 ? "GB/s" : (bw > 1024*1024 ? "MB/s" : "KB/s"))
|
|
|
|
);
|
|
|
|
recovery_stat_count[1][i] = recovery_stat_count[0][i];
|
|
|
|
recovery_stat_bytes[1][i] = recovery_stat_bytes[0][i];
|
|
|
|
}
|
|
|
|
}
|
2020-04-02 22:16:46 +03:00
|
|
|
if (incomplete_objects > 0)
|
|
|
|
{
|
2020-05-23 15:19:51 +03:00
|
|
|
printf("[OSD %lu] %lu object(s) incomplete\n", osd_num, incomplete_objects);
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
|
|
|
if (degraded_objects > 0)
|
|
|
|
{
|
2020-05-23 15:19:51 +03:00
|
|
|
printf("[OSD %lu] %lu object(s) degraded\n", osd_num, degraded_objects);
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
|
|
|
if (misplaced_objects > 0)
|
|
|
|
{
|
2020-05-23 15:19:51 +03:00
|
|
|
printf("[OSD %lu] %lu object(s) misplaced\n", osd_num, misplaced_objects);
|
2020-04-02 22:16:46 +03:00
|
|
|
}
|
|
|
|
}
|