From 30ac89907455e94172b20171ae1ffc9c8f2c7bb7 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 4 Jul 2023 00:23:59 +0300 Subject: [PATCH] Make QEMU driver compatible with older vitastor_client and with systems without io_uring --- src/qemu_driver.c | 37 +++++++++++++++++++++++-------- src/ringloop.cpp | 1 + src/vitastor_c.cpp | 55 ++++++++++++++++++++++++++++++++++++++-------- src/vitastor_c.h | 5 ++++- 4 files changed, 79 insertions(+), 19 deletions(-) diff --git a/src/qemu_driver.c b/src/qemu_driver.c index 32bf6ed8..1d3210ec 100644 --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -259,7 +259,7 @@ static void vitastor_bh_uring_handler(void *opaque) static void vitastor_schedule_uring_handler(VitastorClient *client) { void *opaque = client; - if (!client->bh_uring_scheduled) + if (client->uring_eventfd >= 0 && !client->bh_uring_scheduled) { client->bh_uring_scheduled = 1; #if QEMU_VERSION_MAJOR > 4 || QEMU_VERSION_MAJOR == 4 && QEMU_VERSION_MINOR >= 2 @@ -401,19 +401,38 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E client->rdma_gid_index = qdict_get_try_int(options, "rdma-gid-index", 0); client->rdma_mtu = qdict_get_try_int(options, "rdma-mtu", 0); client->ctx = bdrv_get_aio_context(bs); - client->proxy = vitastor_c_create_qemu( +#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2 + client->proxy = vitastor_c_create_qemu_uring( vitastor_aio_set_fd_handler, client, client->config_path, client->etcd_host, client->etcd_prefix, client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0 ); - client->uring_eventfd = vitastor_c_uring_register_eventfd(client->proxy); - if (client->uring_eventfd < 0) +#else + client->proxy = vitastor_c_create_uring( + client->config_path, client->etcd_host, client->etcd_prefix, + client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0 + ); +#endif + if (!client->proxy) { - fprintf(stderr, "failed to create eventfd: %s\n", strerror(errno)); - error_setg(errp, "failed to create eventfd"); - vitastor_close(bs); - return -1; + fprintf(stderr, "vitastor: failed to create io_uring: %s - I/O will be slower\n", strerror(errno)); + client->uring_eventfd = -1; + client->proxy = vitastor_c_create_qemu( + vitastor_aio_set_fd_handler, client, client->config_path, client->etcd_host, client->etcd_prefix, + client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0 + ); + } + else + { + client->uring_eventfd = vitastor_c_uring_register_eventfd(client->proxy); + if (client->uring_eventfd < 0) + { + fprintf(stderr, "vitastor: failed to create io_uring eventfd: %s\n", strerror(errno)); + error_setg(errp, "failed to create io_uring eventfd"); + vitastor_close(bs); + return -1; + } + universal_aio_set_fd_handler(client->ctx, client->uring_eventfd, vitastor_uring_handler, NULL, client); } - universal_aio_set_fd_handler(client->ctx, client->uring_eventfd, vitastor_uring_handler, NULL, client); image = client->image = g_strdup(qdict_get_try_str(options, "image")); client->readonly = (flags & BDRV_O_RDWR) ? 1 : 0; // Get image metadata (size and readonly flag) or just wait until the client is ready diff --git a/src/ringloop.cpp b/src/ringloop.cpp index 92cfd901..fb06dd1b 100644 --- a/src/ringloop.cpp +++ b/src/ringloop.cpp @@ -2,6 +2,7 @@ // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) #include +#include #include diff --git a/src/vitastor_c.cpp b/src/vitastor_c.cpp index 19349df9..0836190c 100644 --- a/src/vitastor_c.cpp +++ b/src/vitastor_c.cpp @@ -72,14 +72,8 @@ static void vitastor_c_write_handler(void *opaque) data->callback(data->fd, EPOLLOUT); } -vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, - const char *config_path, const char *etcd_host, const char *etcd_prefix, - int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) +static vitastor_c *vitastor_c_create_qemu_common(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context) { - json11::Json cfg_json = vitastor_c_common_config( - config_path, etcd_host, etcd_prefix, use_rdma, - rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level - ); vitastor_c *self = new vitastor_c; self->aio_set_fd_handler = aio_set_fd_handler; self->aio_ctx = aio_context; @@ -97,7 +91,41 @@ vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *a self->aio_set_fd_handler(self->aio_ctx, fd, false, NULL, NULL, NULL, NULL); } }); - self->ringloop = new ring_loop_t(512); + return self; +} + +vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, + const char *config_path, const char *etcd_host, const char *etcd_prefix, + int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) +{ + json11::Json cfg_json = vitastor_c_common_config( + config_path, etcd_host, etcd_prefix, use_rdma, + rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level + ); + auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context); + self->cli = new cluster_client_t(NULL, self->tfd, cfg_json); + return self; +} + +vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, + const char *config_path, const char *etcd_host, const char *etcd_prefix, + int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) +{ + ring_loop_t *ringloop = NULL; + try + { + ringloop = new ring_loop_t(512); + } + catch (std::exception & e) + { + return NULL; + } + json11::Json cfg_json = vitastor_c_common_config( + config_path, etcd_host, etcd_prefix, use_rdma, + rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level + ); + auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context); + self->ringloop = ringloop; self->cli = new cluster_client_t(self->ringloop, self->tfd, cfg_json); return self; } @@ -105,12 +133,21 @@ vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *a vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix, int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) { + ring_loop_t *ringloop = NULL; + try + { + ringloop = new ring_loop_t(512); + } + catch (std::exception & e) + { + return NULL; + } json11::Json cfg_json = vitastor_c_common_config( config_path, etcd_host, etcd_prefix, use_rdma, rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level ); vitastor_c *self = new vitastor_c; - self->ringloop = new ring_loop_t(512); + self->ringloop = ringloop; self->epmgr = new epoll_manager_t(self->ringloop); self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json); return self; diff --git a/src/vitastor_c.h b/src/vitastor_c.h index 57f7f28d..0eccdca1 100644 --- a/src/vitastor_c.h +++ b/src/vitastor_c.h @@ -7,7 +7,7 @@ #define VITASTOR_QEMU_PROXY_H // C API wrapper version -#define VITASTOR_C_API_VERSION 1 +#define VITASTOR_C_API_VERSION 2 #ifndef POOL_ID_BITS #define POOL_ID_BITS 16 @@ -34,6 +34,9 @@ typedef void QEMUSetFDHandler(void *ctx, int fd, int is_external, IOHandler *fd_ vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, const char *config_path, const char *etcd_host, const char *etcd_prefix, int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); +vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, + const char *config_path, const char *etcd_host, const char *etcd_prefix, + int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix, int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len);