From 72aa2fd819204cf3e2ca01563331b926e3e52283 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 30 Apr 2021 00:14:43 +0300 Subject: [PATCH] Make OSD and client read common configuration from /etc/vitastor/vitastor.conf --- src/blockstore.cpp | 5 --- src/blockstore.h | 3 -- src/cluster_client.cpp | 2 ++ src/fio_cluster.cpp | 1 - src/messenger.cpp | 82 +++++++++++++++++++++++++++++++++++------- src/messenger.h | 22 ++++++------ src/mock/messenger.cpp | 5 +++ src/osd.cpp | 2 +- 8 files changed, 88 insertions(+), 34 deletions(-) diff --git a/src/blockstore.cpp b/src/blockstore.cpp index bb4b4112..1a2cf629 100644 --- a/src/blockstore.cpp +++ b/src/blockstore.cpp @@ -43,11 +43,6 @@ int blockstore_t::read_bitmap(object_id oid, uint64_t target_version, void *bitm return impl->read_bitmap(oid, target_version, bitmap, result_version); } -std::unordered_map & blockstore_t::get_unstable_writes() -{ - return impl->unstable_writes; -} - std::map & blockstore_t::get_inode_space_stats() { return impl->inode_space_stats; diff --git a/src/blockstore.h b/src/blockstore.h index 07817bf8..d3a79fd0 100644 --- a/src/blockstore.h +++ b/src/blockstore.h @@ -183,9 +183,6 @@ public: // Simplified synchronous operation: get object bitmap & current version int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL); - // Unstable writes are added here (map of object_id -> version) - std::unordered_map & get_unstable_writes(); - // Get per-inode space usage statistics std::map & get_inode_space_stats(); diff --git a/src/cluster_client.cpp b/src/cluster_client.cpp index 260d65ea..61d25eb0 100644 --- a/src/cluster_client.cpp +++ b/src/cluster_client.cpp @@ -16,6 +16,8 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config) { + config = osd_messenger_t::read_config(config); + this->ringloop = ringloop; this->tfd = tfd; this->config = config; diff --git a/src/fio_cluster.cpp b/src/fio_cluster.cpp index 68855bf4..a59f147b 100644 --- a/src/fio_cluster.cpp +++ b/src/fio_cluster.cpp @@ -24,7 +24,6 @@ #include #include -#include #include "epoll_manager.h" #include "cluster_client.h" diff --git a/src/messenger.cpp b/src/messenger.cpp index 61beacd2..d978f91e 100644 --- a/src/messenger.cpp +++ b/src/messenger.cpp @@ -131,36 +131,46 @@ void osd_messenger_t::parse_config(const json11::Json & config) { #ifdef WITH_RDMA if (!config["use_rdma"].is_null()) + { + // RDMA is on by default in RDMA-enabled builds this->use_rdma = config["use_rdma"].bool_value() || config["use_rdma"].uint64_value() != 0; + } this->rdma_device = config["rdma_device"].string_value(); this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value(); if (!this->rdma_port_num) this->rdma_port_num = 1; this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value(); this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value(); + this->rdma_max_sge = config["rdma_max_sge"].uint64_value(); + if (!this->rdma_max_sge) + this->rdma_max_sge = 128; + this->rdma_max_send = config["rdma_max_send"].uint64_value(); + if (!this->rdma_max_send) + this->rdma_max_send = 32; + this->rdma_max_recv = config["rdma_max_recv"].uint64_value(); + if (!this->rdma_max_recv) + this->rdma_max_recv = 8; + this->rdma_max_msg = config["rdma_max_msg"].uint64_value(); + if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024) + this->rdma_max_msg = 1024*1024; #endif + this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value(); + if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024) + this->receive_buffer_size = 65536; this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() || config["use_sync_send_recv"].uint64_value(); this->peer_connect_interval = config["peer_connect_interval"].uint64_value(); if (!this->peer_connect_interval) - { - this->peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL; - } + this->peer_connect_interval = 5; this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value(); if (!this->peer_connect_timeout) - { - this->peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT; - } + this->peer_connect_timeout = 5; this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value(); if (!this->osd_idle_timeout) - { - this->osd_idle_timeout = DEFAULT_OSD_PING_TIMEOUT; - } + this->osd_idle_timeout = 5; this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value(); if (!this->osd_ping_timeout) - { - this->osd_ping_timeout = DEFAULT_OSD_PING_TIMEOUT; - } + this->osd_ping_timeout = 5; this->log_level = config["log_level"].uint64_value(); } @@ -507,3 +517,51 @@ bool osd_messenger_t::is_rdma_enabled() { return rdma_context != NULL; } + +json11::Json osd_messenger_t::read_config(const json11::Json & config) +{ + const char *config_path = config["config_path"].string_value() != "" + ? config["config_path"].string_value().c_str() : VITASTOR_CONFIG_PATH; + int fd = open(config_path, O_RDONLY); + if (fd < 0) + { + fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno)); + return config; + } + struct stat st; + if (fstat(fd, &st) != 0) + { + fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno)); + close(fd); + return config; + } + std::string buf; + buf.resize(st.st_size); + int done = 0; + while (done < st.st_size) + { + int r = read(fd, (void*)buf.data()+done, st.st_size-done); + if (r < 0) + { + fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno)); + close(fd); + return config; + } + done += r; + } + close(fd); + std::string json_err; + json11::Json::object file_config = json11::Json::parse(buf, json_err).object_items(); + if (json_err != "") + { + fprintf(stderr, "Invalid JSON in %s: %s\n", config_path, json_err.c_str()); + return config; + } + file_config.erase("config_path"); + file_config.erase("osd_num"); + for (auto kv: config.object_items()) + { + file_config[kv.first] = kv.second; + } + return file_config; +} diff --git a/src/messenger.h b/src/messenger.h index a98b5aa7..42d58980 100644 --- a/src/messenger.h +++ b/src/messenger.h @@ -33,10 +33,8 @@ #define PEER_RDMA 4 #define PEER_STOPPED 5 -#define DEFAULT_PEER_CONNECT_INTERVAL 5 -#define DEFAULT_PEER_CONNECT_TIMEOUT 5 -#define DEFAULT_OSD_PING_TIMEOUT 5 #define DEFAULT_BITMAP_GRANULARITY 4096 +#define VITASTOR_CONFIG_PATH "/etc/vitastor/vitastor.conf" #define MSGR_SENDP_HDR 1 #define MSGR_SENDP_FREE 2 @@ -122,12 +120,11 @@ struct osd_messenger_t protected: int keepalive_timer_id = -1; - // FIXME: make receive_buffer_size configurable - int receive_buffer_size = 64*1024; - int peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL; - int peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT; - int osd_idle_timeout = DEFAULT_OSD_PING_TIMEOUT; - int osd_ping_timeout = DEFAULT_OSD_PING_TIMEOUT; + uint32_t receive_buffer_size = 0; + int peer_connect_interval = 0; + int peer_connect_timeout = 0; + int osd_idle_timeout = 0; + int osd_ping_timeout = 0; int log_level = 0; bool use_sync_send_recv = false; @@ -136,9 +133,8 @@ protected: std::string rdma_device; uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0; msgr_rdma_context_t *rdma_context = NULL; - // FIXME: Allow to configure these options - uint64_t rdma_max_sge = 128, rdma_max_send = 32, rdma_max_recv = 8; - uint64_t rdma_max_msg = 1024*1024; + uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 8; + uint64_t rdma_max_msg = 0; #endif std::vector read_ready_clients; @@ -169,6 +165,8 @@ public: void accept_connections(int listen_fd); ~osd_messenger_t(); + static json11::Json read_config(const json11::Json & config); + #ifdef WITH_RDMA bool is_rdma_enabled(); bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg); diff --git a/src/mock/messenger.cpp b/src/mock/messenger.cpp index 0fcd3cbd..7ba9ffa4 100644 --- a/src/mock/messenger.cpp +++ b/src/mock/messenger.cpp @@ -42,3 +42,8 @@ void osd_messenger_t::read_requests() void osd_messenger_t::send_replies() { } + +json11::Json osd_messenger_t::read_config(const json11::Json & config) +{ + return config; +} diff --git a/src/osd.cpp b/src/osd.cpp index a33a4d41..f61d8072 100644 --- a/src/osd.cpp +++ b/src/osd.cpp @@ -29,9 +29,9 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop) zero_buffer = malloc_or_die(zero_buffer_size); memset(zero_buffer, 0, zero_buffer_size); - this->config = config.object_items(); this->ringloop = ringloop; + this->config = msgr.read_config(config).object_items(); if (this->config.find("log_level") == this->config.end()) this->config["log_level"] = 1; parse_config(this->config);