Add postponed send loop to QEMU driver

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "0.9.3")
+set(VERSION "0.9.2")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.9.3
+VERSION ?= v0.9.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.9.3
+          image: vitalif/vitastor-csi:v0.9.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.9.3
+          image: vitalif/vitastor-csi:v0.9.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.9.3"
+    vitastorCSIDriverVersion = "0.9.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.9.3-1) unstable; urgency=medium
+vitastor (0.9.2-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.9.3-1) unstable; urgency=medium
+vitastor (0.9.2-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.9.3; \
+    cp -r /root/vitastor vitastor-0.9.2; \
-    cd vitastor-0.9.3; \
+    cd vitastor-0.9.2; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.3.orig.tar.xz vitastor-0.9.3; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.2.orig.tar.xz vitastor-0.9.2; \
-    cd vitastor-0.9.3; \
+    cd vitastor-0.9.2; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.9.3'
+VERSION = '0.9.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.9.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.3$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.9.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.2$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.3.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.2.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.3
+Version:        0.9.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.3.el7.tar.gz
+Source0:        vitastor-0.9.2.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.3.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.2.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.3
+Version:        0.9.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.3.el8.tar.gz
+Source0:        vitastor-0.9.2.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.3.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.2.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.3
+Version:        0.9.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.3.el9.tar.gz
+Source0:        vitastor-0.9.2.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.9.3")
+add_definitions(-DVERSION="0.9.2")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/messenger.h

@@ -132,6 +132,7 @@ protected:
     uint64_t rdma_max_msg = 0;
 #endif
 
+    bool has_send_loop = false;
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
@@ -159,6 +160,7 @@ public:
     std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
     void read_requests();
     void send_replies();
+    bool can_send();
     void accept_connections(int listen_fd);
     ~osd_messenger_t();
 

src/msgr_send.cpp

@@ -111,22 +111,28 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         return;
     }
 #endif
-    if (!ringloop)
+    if (!ringloop && !has_send_loop)
     {
-        // FIXME: It's worse because it doesn't allow batching
+        // "Send loop" should be used, like in QEMU driver, or performance will suffer
+        // due to a large number of sendmsg() syscalls where they can be avoided.
+        // "Send loop" increases 4k T1Q128 randread from ~55k iops to ~90k on a 1 OSD.
+        // The difference is smaller when there are more OSDs though...
         while (cl->outbox.size())
         {
             try_send(cl);
         }
     }
-    else if (cl->write_msg.msg_iovlen > 0 || !try_send(cl))
+    else if (cl->write_msg.msg_iovlen > 0 || has_send_loop || !try_send(cl))
     {
         if (cl->write_state == 0)
         {
             cl->write_state = CL_WRITE_READY;
             write_ready_clients.push_back(cur_op->peer_fd);
         }
-        ringloop->wakeup();
+        if (ringloop)
+        {
+            ringloop->wakeup();
+        }
     }
 }
 
@@ -218,6 +224,12 @@ void osd_messenger_t::send_replies()
     write_ready_clients.clear();
 }
 
+bool osd_messenger_t::can_send()
+{
+    has_send_loop = true;
+    return write_ready_clients.size() > 0;
+}
+
 void osd_messenger_t::handle_send(int result, osd_client_t *cl)
 {
     cl->write_msg.msg_iovlen = 0;

src/qemu_driver.c

@@ -77,6 +77,7 @@ typedef struct VitastorClient
     AioContext *ctx;
     VitastorFdData **fds;
     int fd_count, fd_alloc;
+    int bh_send_all_scheduled;
 
     uint64_t last_bitmap_inode, last_bitmap_offset, last_bitmap_len;
     uint32_t last_bitmap_granularity;
@@ -106,6 +107,14 @@ typedef struct VitastorRPC
 #endif
 } VitastorRPC;
 
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
+typedef struct VitastorBH
+{
+    VitastorClient *cli;
+    QEMUBH *bh;
+} VitastorBH;
+#endif
+
 static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task);
 static void vitastor_co_generic_cb(void *opaque, long retval);
 static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version);
@@ -223,6 +232,49 @@ out:
     return;
 }
 
+#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
+static void vitastor_bh_send_all(void *opaque)
+{
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
+    VitastorBH *vbh = opaque;
+    VitastorClient *client = vbh->cli;
+#else
+    VitastorClient *client = opaque;
+#endif
+    qemu_mutex_lock(&client->mutex);
+    client->bh_send_all_scheduled = 0;
+    vitastor_c_qemu_send_all(client->proxy);
+    qemu_mutex_unlock(&client->mutex);
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
+    qemu_bh_delete(vbh->bh);
+    free(vbh);
+#endif
+}
+#endif
+
+static void vitastor_schedule_send_all(VitastorClient *client)
+{
+#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
+    if (!client->bh_send_all_scheduled)
+    {
+        client->bh_send_all_scheduled = 1;
+#if QEMU_VERSION_MAJOR > 4 || QEMU_VERSION_MAJOR == 4 && QEMU_VERSION_MINOR >= 2
+        replay_bh_schedule_oneshot_event(client->ctx, vitastor_bh_send_all, client);
+#elif QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 8
+        aio_bh_schedule_oneshot(client->ctx, vitastor_bh_send_all, client);
+#elif QEMU_VERSION_MAJOR >= 2
+        VitastorBH *vbh = (VitastorBH*)malloc(sizeof(VitastorBH));
+        vbh->cli = client;
+        vbh->bh = aio_bh_new(bdrv_get_aio_context(task->bs), vitastor_bh_send_all, vbh);
+        qemu_bh_schedule(vbh->bh);
+#else
+        client->bh_send_all_scheduled = 0;
+        vitastor_c_qemu_send_all(client->proxy);
+#endif
+    }
+#endif
+}
+
 static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 {
     BlockDriverState *bs = task->bs;
@@ -231,6 +283,7 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
+    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task->complete)
@@ -244,6 +297,7 @@ static void vitastor_aio_fd_read(void *fddv)
     VitastorFdData *fdd = (VitastorFdData*)fddv;
     qemu_mutex_lock(&fdd->cli->mutex);
     fdd->fd_read(fdd->opaque);
+    vitastor_schedule_send_all(fdd->cli);
     qemu_mutex_unlock(&fdd->cli->mutex);
 }
 
@@ -252,6 +306,7 @@ static void vitastor_aio_fd_write(void *fddv)
     VitastorFdData *fdd = (VitastorFdData*)fddv;
     qemu_mutex_lock(&fdd->cli->mutex);
     fdd->fd_write(fdd->opaque);
+    vitastor_schedule_send_all(fdd->cli);
     qemu_mutex_unlock(&fdd->cli->mutex);
 }
 
@@ -593,6 +648,7 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
+    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -626,6 +682,7 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
+    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -704,6 +761,7 @@ static int coroutine_fn vitastor_co_block_status(
         task.bitmap = client->last_bitmap = NULL;
         qemu_mutex_lock(&client->mutex);
         vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
+        vitastor_schedule_send_all(client);
         qemu_mutex_unlock(&client->mutex);
         while (!task.complete)
         {
@@ -790,6 +848,7 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
+    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.9.3
+Version: 0.9.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/vitastor_c.cpp

@@ -166,6 +166,14 @@ void vitastor_c_uring_wait_events(vitastor_c *client)
     client->ringloop->wait();
 }
 
+void vitastor_c_qemu_send_all(vitastor_c *client)
+{
+    while (client->cli->msgr.can_send())
+    {
+        client->cli->msgr.send_replies();
+    }
+}
+
 void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
     struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque)
 {

src/vitastor_c.h

@@ -7,7 +7,7 @@
 #define VITASTOR_QEMU_PROXY_H
 
 // C API wrapper version
-#define VITASTOR_C_API_VERSION 1
+#define VITASTOR_C_API_VERSION 2
 
 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16
@@ -42,6 +42,7 @@ int vitastor_c_is_ready(vitastor_c *client);
 void vitastor_c_uring_wait_ready(vitastor_c *client);
 void vitastor_c_uring_handle_events(vitastor_c *client);
 void vitastor_c_uring_wait_events(vitastor_c *client);
+void vitastor_c_qemu_send_all(vitastor_c *client);
 void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
     struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,