forked from vitalif/vitastor
Make OSD and client read common configuration from /etc/vitastor/vitastor.conf
parent
5010b0dd75
commit
72aa2fd819
|
@ -43,11 +43,6 @@ int blockstore_t::read_bitmap(object_id oid, uint64_t target_version, void *bitm
|
||||||
return impl->read_bitmap(oid, target_version, bitmap, result_version);
|
return impl->read_bitmap(oid, target_version, bitmap, result_version);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unordered_map<object_id, uint64_t> & blockstore_t::get_unstable_writes()
|
|
||||||
{
|
|
||||||
return impl->unstable_writes;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
|
std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
|
||||||
{
|
{
|
||||||
return impl->inode_space_stats;
|
return impl->inode_space_stats;
|
||||||
|
|
|
@ -183,9 +183,6 @@ public:
|
||||||
// Simplified synchronous operation: get object bitmap & current version
|
// Simplified synchronous operation: get object bitmap & current version
|
||||||
int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL);
|
int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL);
|
||||||
|
|
||||||
// Unstable writes are added here (map of object_id -> version)
|
|
||||||
std::unordered_map<object_id, uint64_t> & get_unstable_writes();
|
|
||||||
|
|
||||||
// Get per-inode space usage statistics
|
// Get per-inode space usage statistics
|
||||||
std::map<uint64_t, uint64_t> & get_inode_space_stats();
|
std::map<uint64_t, uint64_t> & get_inode_space_stats();
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,8 @@
|
||||||
|
|
||||||
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config)
|
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config)
|
||||||
{
|
{
|
||||||
|
config = osd_messenger_t::read_config(config);
|
||||||
|
|
||||||
this->ringloop = ringloop;
|
this->ringloop = ringloop;
|
||||||
this->tfd = tfd;
|
this->tfd = tfd;
|
||||||
this->config = config;
|
this->config = config;
|
||||||
|
|
|
@ -24,7 +24,6 @@
|
||||||
#include <netinet/tcp.h>
|
#include <netinet/tcp.h>
|
||||||
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <unordered_map>
|
|
||||||
|
|
||||||
#include "epoll_manager.h"
|
#include "epoll_manager.h"
|
||||||
#include "cluster_client.h"
|
#include "cluster_client.h"
|
||||||
|
|
|
@ -131,36 +131,46 @@ void osd_messenger_t::parse_config(const json11::Json & config)
|
||||||
{
|
{
|
||||||
#ifdef WITH_RDMA
|
#ifdef WITH_RDMA
|
||||||
if (!config["use_rdma"].is_null())
|
if (!config["use_rdma"].is_null())
|
||||||
|
{
|
||||||
|
// RDMA is on by default in RDMA-enabled builds
|
||||||
this->use_rdma = config["use_rdma"].bool_value() || config["use_rdma"].uint64_value() != 0;
|
this->use_rdma = config["use_rdma"].bool_value() || config["use_rdma"].uint64_value() != 0;
|
||||||
|
}
|
||||||
this->rdma_device = config["rdma_device"].string_value();
|
this->rdma_device = config["rdma_device"].string_value();
|
||||||
this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
|
this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
|
||||||
if (!this->rdma_port_num)
|
if (!this->rdma_port_num)
|
||||||
this->rdma_port_num = 1;
|
this->rdma_port_num = 1;
|
||||||
this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
|
this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
|
||||||
this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
|
this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
|
||||||
|
this->rdma_max_sge = config["rdma_max_sge"].uint64_value();
|
||||||
|
if (!this->rdma_max_sge)
|
||||||
|
this->rdma_max_sge = 128;
|
||||||
|
this->rdma_max_send = config["rdma_max_send"].uint64_value();
|
||||||
|
if (!this->rdma_max_send)
|
||||||
|
this->rdma_max_send = 32;
|
||||||
|
this->rdma_max_recv = config["rdma_max_recv"].uint64_value();
|
||||||
|
if (!this->rdma_max_recv)
|
||||||
|
this->rdma_max_recv = 8;
|
||||||
|
this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
|
||||||
|
if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
|
||||||
|
this->rdma_max_msg = 1024*1024;
|
||||||
#endif
|
#endif
|
||||||
|
this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value();
|
||||||
|
if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024)
|
||||||
|
this->receive_buffer_size = 65536;
|
||||||
this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
|
this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
|
||||||
config["use_sync_send_recv"].uint64_value();
|
config["use_sync_send_recv"].uint64_value();
|
||||||
this->peer_connect_interval = config["peer_connect_interval"].uint64_value();
|
this->peer_connect_interval = config["peer_connect_interval"].uint64_value();
|
||||||
if (!this->peer_connect_interval)
|
if (!this->peer_connect_interval)
|
||||||
{
|
this->peer_connect_interval = 5;
|
||||||
this->peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
|
||||||
}
|
|
||||||
this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value();
|
this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value();
|
||||||
if (!this->peer_connect_timeout)
|
if (!this->peer_connect_timeout)
|
||||||
{
|
this->peer_connect_timeout = 5;
|
||||||
this->peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
|
||||||
}
|
|
||||||
this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value();
|
this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value();
|
||||||
if (!this->osd_idle_timeout)
|
if (!this->osd_idle_timeout)
|
||||||
{
|
this->osd_idle_timeout = 5;
|
||||||
this->osd_idle_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
|
||||||
}
|
|
||||||
this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value();
|
this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value();
|
||||||
if (!this->osd_ping_timeout)
|
if (!this->osd_ping_timeout)
|
||||||
{
|
this->osd_ping_timeout = 5;
|
||||||
this->osd_ping_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
|
||||||
}
|
|
||||||
this->log_level = config["log_level"].uint64_value();
|
this->log_level = config["log_level"].uint64_value();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -507,3 +517,51 @@ bool osd_messenger_t::is_rdma_enabled()
|
||||||
{
|
{
|
||||||
return rdma_context != NULL;
|
return rdma_context != NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
json11::Json osd_messenger_t::read_config(const json11::Json & config)
|
||||||
|
{
|
||||||
|
const char *config_path = config["config_path"].string_value() != ""
|
||||||
|
? config["config_path"].string_value().c_str() : VITASTOR_CONFIG_PATH;
|
||||||
|
int fd = open(config_path, O_RDONLY);
|
||||||
|
if (fd < 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (fstat(fd, &st) != 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
|
||||||
|
close(fd);
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
std::string buf;
|
||||||
|
buf.resize(st.st_size);
|
||||||
|
int done = 0;
|
||||||
|
while (done < st.st_size)
|
||||||
|
{
|
||||||
|
int r = read(fd, (void*)buf.data()+done, st.st_size-done);
|
||||||
|
if (r < 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
|
||||||
|
close(fd);
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
done += r;
|
||||||
|
}
|
||||||
|
close(fd);
|
||||||
|
std::string json_err;
|
||||||
|
json11::Json::object file_config = json11::Json::parse(buf, json_err).object_items();
|
||||||
|
if (json_err != "")
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Invalid JSON in %s: %s\n", config_path, json_err.c_str());
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
file_config.erase("config_path");
|
||||||
|
file_config.erase("osd_num");
|
||||||
|
for (auto kv: config.object_items())
|
||||||
|
{
|
||||||
|
file_config[kv.first] = kv.second;
|
||||||
|
}
|
||||||
|
return file_config;
|
||||||
|
}
|
||||||
|
|
|
@ -33,10 +33,8 @@
|
||||||
#define PEER_RDMA 4
|
#define PEER_RDMA 4
|
||||||
#define PEER_STOPPED 5
|
#define PEER_STOPPED 5
|
||||||
|
|
||||||
#define DEFAULT_PEER_CONNECT_INTERVAL 5
|
|
||||||
#define DEFAULT_PEER_CONNECT_TIMEOUT 5
|
|
||||||
#define DEFAULT_OSD_PING_TIMEOUT 5
|
|
||||||
#define DEFAULT_BITMAP_GRANULARITY 4096
|
#define DEFAULT_BITMAP_GRANULARITY 4096
|
||||||
|
#define VITASTOR_CONFIG_PATH "/etc/vitastor/vitastor.conf"
|
||||||
|
|
||||||
#define MSGR_SENDP_HDR 1
|
#define MSGR_SENDP_HDR 1
|
||||||
#define MSGR_SENDP_FREE 2
|
#define MSGR_SENDP_FREE 2
|
||||||
|
@ -122,12 +120,11 @@ struct osd_messenger_t
|
||||||
protected:
|
protected:
|
||||||
int keepalive_timer_id = -1;
|
int keepalive_timer_id = -1;
|
||||||
|
|
||||||
// FIXME: make receive_buffer_size configurable
|
uint32_t receive_buffer_size = 0;
|
||||||
int receive_buffer_size = 64*1024;
|
int peer_connect_interval = 0;
|
||||||
int peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
int peer_connect_timeout = 0;
|
||||||
int peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
int osd_idle_timeout = 0;
|
||||||
int osd_idle_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
int osd_ping_timeout = 0;
|
||||||
int osd_ping_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
|
||||||
int log_level = 0;
|
int log_level = 0;
|
||||||
bool use_sync_send_recv = false;
|
bool use_sync_send_recv = false;
|
||||||
|
|
||||||
|
@ -136,9 +133,8 @@ protected:
|
||||||
std::string rdma_device;
|
std::string rdma_device;
|
||||||
uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
|
uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
|
||||||
msgr_rdma_context_t *rdma_context = NULL;
|
msgr_rdma_context_t *rdma_context = NULL;
|
||||||
// FIXME: Allow to configure these options
|
uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 8;
|
||||||
uint64_t rdma_max_sge = 128, rdma_max_send = 32, rdma_max_recv = 8;
|
uint64_t rdma_max_msg = 0;
|
||||||
uint64_t rdma_max_msg = 1024*1024;
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
std::vector<int> read_ready_clients;
|
std::vector<int> read_ready_clients;
|
||||||
|
@ -169,6 +165,8 @@ public:
|
||||||
void accept_connections(int listen_fd);
|
void accept_connections(int listen_fd);
|
||||||
~osd_messenger_t();
|
~osd_messenger_t();
|
||||||
|
|
||||||
|
static json11::Json read_config(const json11::Json & config);
|
||||||
|
|
||||||
#ifdef WITH_RDMA
|
#ifdef WITH_RDMA
|
||||||
bool is_rdma_enabled();
|
bool is_rdma_enabled();
|
||||||
bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
|
bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
|
||||||
|
|
|
@ -42,3 +42,8 @@ void osd_messenger_t::read_requests()
|
||||||
void osd_messenger_t::send_replies()
|
void osd_messenger_t::send_replies()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
json11::Json osd_messenger_t::read_config(const json11::Json & config)
|
||||||
|
{
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
|
@ -29,9 +29,9 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
|
||||||
zero_buffer = malloc_or_die(zero_buffer_size);
|
zero_buffer = malloc_or_die(zero_buffer_size);
|
||||||
memset(zero_buffer, 0, zero_buffer_size);
|
memset(zero_buffer, 0, zero_buffer_size);
|
||||||
|
|
||||||
this->config = config.object_items();
|
|
||||||
this->ringloop = ringloop;
|
this->ringloop = ringloop;
|
||||||
|
|
||||||
|
this->config = msgr.read_config(config).object_items();
|
||||||
if (this->config.find("log_level") == this->config.end())
|
if (this->config.find("log_level") == this->config.end())
|
||||||
this->config["log_level"] = 1;
|
this->config["log_level"] = 1;
|
||||||
parse_config(this->config);
|
parse_config(this->config);
|
||||||
|
|
Loading…
Reference in New Issue