WIP another experiment - "smart" iothreads

Move RDMA handling
Change bool wr to event mask in epoll_manager
2024-06-27 01:15:09 +03:00 · 2024-06-24 10:43:12 +03:00 · 2024-06-24 00:54:44 +03:00
19 changed files with 235 additions and 2503 deletions
--- a/patches/cinder-vitastor.py
+++ b/patches/cinder-vitastor.py
@@ -707,10 +707,10 @@ class VitastorDriver(driver.CloneableImageVD,
                    return ({}, True)
        return ({}, False)

-    def copy_image_to_encrypted_volume(self, context, volume, image_service, image_id, disable_sparse=False):
-        self.copy_image_to_volume(context, volume, image_service, image_id, encrypted = True, disable_sparse=False)
+    def copy_image_to_encrypted_volume(self, context, volume, image_service, image_id):
+        self.copy_image_to_volume(context, volume, image_service, image_id, encrypted = True)

-    def copy_image_to_volume(self, context, volume, image_service, image_id, encrypted = False, disable_sparse=False):
+    def copy_image_to_volume(self, context, volume, image_service, image_id, encrypted = False):
        tmp_dir = volume_utils.image_conversion_dir()
        with tempfile.NamedTemporaryFile(dir = tmp_dir) as tmp:
            image_utils.fetch_to_raw(
--- a/patches/libvirt-10.0-vitastor.diff
+++ b/patches/libvirt-10.0-vitastor.diff
@@ -1,670 +0,0 @@
-From 571bde71268dcca6446454bb1e895e21bcc7b2a0 Mon Sep 17 00:00:00 2001
-From: ace <ace@0xace.cc>
-Date: Sat, 18 May 2024 19:45:49 +0300
-Subject: [PATCH] Add Vitastor support
-
---
- include/libvirt/libvirt-storage.h             |   1 +
- src/conf/domain_conf.c                        |   4 +-
- src/conf/domain_validate.c                    |  10 +-
- src/conf/schemas/domaincommon.rng             |  30 +++++
- src/conf/storage_conf.c                       |  20 ++-
- src/conf/storage_conf.h                       |   2 +
- src/conf/storage_source_conf.c                |   2 +
- src/conf/storage_source_conf.h                |   1 +
- src/conf/virstorageobj.c                      |   3 +
- src/libvirt-storage.c                         |   1 +
- src/libxl/libxl_conf.c                        |   1 +
- src/libxl/xen_xl.c                            |   1 +
- src/qemu/qemu_block.c                         |  45 +++++++
- src/qemu/qemu_domain.c                        |   4 +-
- src/qemu/qemu_snapshot.c                      |   2 +
- src/storage/storage_driver.c                  |   1 +
- .../storage_source_backingstore.c             | 123 ++++++++++++++++++
- src/test/test_driver.c                        |   1 +
- .../storagepoolcapsschemadata/poolcaps-fs.xml |   7 +
- .../poolcaps-full.xml                         |   7 +
- tests/storagepoolxml2argvtest.c               |   1 +
- tools/virsh-pool.c                            |   3 +
- 22 files changed, 265 insertions(+), 5 deletions(-)
-
-diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
-index aaad4a3da1..5f5daa8341 100644
--- a/include/libvirt/libvirt-storage.h
-+++ b/include/libvirt/libvirt-storage.h
-@@ -326,6 +326,7 @@ typedef enum {
-     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
-+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
- } virConnectListAllStoragePoolsFlags;
- 
- int                     virConnectListAllStoragePools(virConnectPtr conn,
-diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
-index 52a5796ad2..089697b2a3 100644
--- a/src/conf/domain_conf.c
-+++ b/src/conf/domain_conf.c
-@@ -7191,7 +7191,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
-     src->configFile = virXPathString("string(./config/@file)", ctxt);
- 
-     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
-        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
-         src->query = virXMLPropString(node, "query");
- 
-     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
-@@ -30657,6 +30658,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
- 
-     case VIR_STORAGE_POOL_MPATH:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_SHEEPDOG:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_LAST:
-diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
-index faa7659f07..01b907d60d 100644
--- a/src/conf/domain_validate.c
-+++ b/src/conf/domain_validate.c
-@@ -495,6 +495,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-@@ -541,7 +542,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         }
-     }
- 
-    /* internal snapshots and config files are currently supported only with rbd: */
-+    /* internal snapshots are currently supported only with rbd: */
-     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
-         if (src->snapshot) {
-@@ -549,10 +550,15 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-                            _("<snapshot> element is currently supported only with 'rbd' disks"));
-             return -1;
-         }
-+    }
- 
-+    /* config files are currently supported only with rbd and vitastor: */
-+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
-         if (src->configFile) {
-             virReportError(VIR_ERR_XML_ERROR, "%s",
-                           _("<config> element is currently supported only with 'rbd' disks"));
-+                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
-             return -1;
-         }
-     }
-diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
-index df44cd9857..4bb72fc697 100644
--- a/src/conf/schemas/domaincommon.rng
-+++ b/src/conf/schemas/domaincommon.rng
-@@ -1997,6 +1997,35 @@
-     </element>
-   </define>
- 
-+  <define name="diskSourceNetworkProtocolVitastor">
-+    <element name="source">
-+      <interleave>
-+        <attribute name="protocol">
-+          <value>vitastor</value>
-+        </attribute>
-+        <ref name="diskSourceCommon"/>
-+        <optional>
-+          <attribute name="name"/>
-+        </optional>
-+        <optional>
-+          <attribute name="query"/>
-+        </optional>
-+        <zeroOrMore>
-+          <ref name="diskSourceNetworkHost"/>
-+        </zeroOrMore>
-+        <optional>
-+          <element name="config">
-+            <attribute name="file">
-+              <ref name="absFilePath"/>
-+            </attribute>
-+            <empty/>
-+          </element>
-+        </optional>
-+        <empty/>
-+      </interleave>
-+    </element>
-+  </define>
-+
-   <define name="diskSourceNetworkProtocolISCSI">
-     <element name="source">
-       <attribute name="protocol">
-@@ -2347,6 +2376,7 @@
-       <ref name="diskSourceNetworkProtocolSimple"/>
-       <ref name="diskSourceNetworkProtocolVxHS"/>
-       <ref name="diskSourceNetworkProtocolNFS"/>
-+      <ref name="diskSourceNetworkProtocolVitastor"/>
-     </choice>
-   </define>
- 
-diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
-index 68842004b7..1d69a788b6 100644
--- a/src/conf/storage_conf.c
-+++ b/src/conf/storage_conf.c
-@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
-               "logical", "disk", "iscsi",
-               "iscsi-direct", "scsi", "mpath",
-               "rbd", "sheepdog", "gluster",
-              "zfs", "vstorage",
-+              "zfs", "vstorage", "vitastor",
- );
- 
- VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
-@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
-           .formatToString = virStorageFileFormatTypeToString,
-       }
-     },
-+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
-+     .poolOptions = {
-+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
-+                   VIR_STORAGE_POOL_SOURCE_NAME),
-+      },
-+      .volOptions = {
-+          .defaultFormat = VIR_STORAGE_FILE_RAW,
-+          .formatFromString = virStorageVolumeFormatFromString,
-+          .formatToString = virStorageFileFormatTypeToString,
-+      }
-+    },
-     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
-      .poolOptions = {
-          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
-                        _("element 'name' is mandatory for RBD pool"));
-         return -1;
-     }
-+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
-+        virReportError(VIR_ERR_XML_ERROR, "%s",
-+                       _("element 'name' is mandatory for Vitastor pool"));
-+        return -1;
-+    }
- 
-     if (options->formatFromString) {
-         g_autofree char *format = NULL;
-@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
-     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
-      * files, so they don't have a target */
-     if (def->type != VIR_STORAGE_POOL_RBD &&
-+        def->type != VIR_STORAGE_POOL_VITASTOR &&
-         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
-         def->type != VIR_STORAGE_POOL_GLUSTER &&
-         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
-diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
-index fc67957cfe..720c07ef74 100644
--- a/src/conf/storage_conf.h
-+++ b/src/conf/storage_conf.h
-@@ -103,6 +103,7 @@ typedef enum {
-     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
-     VIR_STORAGE_POOL_ZFS,      /* ZFS */
-     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
-+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
- 
-     VIR_STORAGE_POOL_LAST,
- } virStoragePoolType;
-@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
-+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
-diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
-index 959ec5ed40..e751dd4d6a 100644
--- a/src/conf/storage_source_conf.c
-+++ b/src/conf/storage_source_conf.c
-@@ -88,6 +88,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
-               "ssh",
-               "vxhs",
-               "nfs",
-+              "vitastor",
- );
- 
- 
-@@ -1301,6 +1302,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-             return 24007;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             /* we don't provide a default for RBD */
-             return 0;
-diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
-index 05b4bda16c..b5ed143c39 100644
--- a/src/conf/storage_source_conf.h
-+++ b/src/conf/storage_source_conf.h
-@@ -129,6 +129,7 @@ typedef enum {
-     VIR_STORAGE_NET_PROTOCOL_SSH,
-     VIR_STORAGE_NET_PROTOCOL_VXHS,
-     VIR_STORAGE_NET_PROTOCOL_NFS,
-+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
- 
-     VIR_STORAGE_NET_PROTOCOL_LAST
- } virStorageNetProtocol;
-diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
-index 59fa5da372..4739167f5f 100644
--- a/src/conf/virstorageobj.c
-+++ b/src/conf/virstorageobj.c
-@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
-             return 1;
-         break;
- 
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_RBD:
-     case VIR_STORAGE_POOL_LAST:
-@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
-                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
-                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
-+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
-+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
-                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
-diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
-index db7660aac4..561df34709 100644
--- a/src/libvirt-storage.c
-+++ b/src/libvirt-storage.c
-@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
-  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
-  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
-+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
-  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
-  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
-diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
-index 62e1be6672..71a1d42896 100644
--- a/src/libxl/libxl_conf.c
-+++ b/src/libxl/libxl_conf.c
-@@ -979,6 +979,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
-index f175359307..8efcf4c329 100644
--- a/src/libxl/xen_xl.c
-+++ b/src/libxl/xen_xl.c
-@@ -1456,6 +1456,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
-index c9f5cbbf29..dbbac36836 100644
--- a/src/qemu/qemu_block.c
-+++ b/src/qemu/qemu_block.c
-@@ -758,6 +758,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
- }
- 
- 
-+static virJSONValue *
-+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
-+{
-+    virJSONValue *ret = NULL;
-+    virStorageNetHostDef *host;
-+    size_t i;
-+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-+    g_autofree char *etcd = NULL;
-+
-+    for (i = 0; i < src->nhosts; i++) {
-+        host = src->hosts + i;
-+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
-+            return NULL;
-+        }
-+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
-+    }
-+    if (src->nhosts > 0) {
-+        etcd = virBufferContentAndReset(&buf);
-+    }
-+
-+    if (virJSONValueObjectAdd(&ret,
-+                              "S:etcd-host", etcd,
-+                              "S:etcd-prefix", src->query,
-+                              "S:config-path", src->configFile,
-+                              "s:image", src->path,
-+                              NULL) < 0)
-+        return NULL;
-+
-+    return ret;
-+}
-+
-+
- static virJSONValue *
- qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src)
- {
-@@ -1140,6 +1172,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
-                 return NULL;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return NULL;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             driver = "sheepdog";
-             if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src)))
-@@ -2020,6 +2058,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
- 
-             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             case VIR_STORAGE_NET_PROTOCOL_RBD:
-+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-             case VIR_STORAGE_NET_PROTOCOL_VXHS:
-             case VIR_STORAGE_NET_PROTOCOL_NFS:
-             case VIR_STORAGE_NET_PROTOCOL_SSH:
-@@ -2400,6 +2439,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
-                 return -1;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return -1;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             driver = "sheepdog";
-             if (!(location = qemuBlockStorageSourceGetSheepdogProps(src)))
-diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
-index 341c543280..61b248fa2c 100644
--- a/src/qemu/qemu_domain.c
-+++ b/src/qemu/qemu_domain.c
-@@ -5207,7 +5207,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
-     if (src->query &&
-         (actualType != VIR_STORAGE_TYPE_NETWORK ||
-          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
-          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
-         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                        _("query is supported only with HTTP(S) protocols"));
-         return -1;
-@@ -10387,6 +10388,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
-         break;
- 
-     case VIR_STORAGE_NET_PROTOCOL_RBD:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
-index 0cac0c4146..4955ebd8d4 100644
--- a/src/qemu/qemu_snapshot.c
-+++ b/src/qemu/qemu_snapshot.c
-@@ -423,6 +423,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-@@ -648,6 +649,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
-index 314fe930e0..fb615a8b4e 100644
--- a/src/storage/storage_driver.c
-+++ b/src/storage/storage_driver.c
-@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
- 
-         case VIR_STORAGE_POOL_GLUSTER:
-         case VIR_STORAGE_POOL_RBD:
-+        case VIR_STORAGE_POOL_VITASTOR:
-         case VIR_STORAGE_POOL_SHEEPDOG:
-         case VIR_STORAGE_POOL_ZFS:
-         case VIR_STORAGE_POOL_LAST:
-diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
-index 80681924ea..8a3ade9ec0 100644
--- a/src/storage_file/storage_source_backingstore.c
-+++ b/src/storage_file/storage_source_backingstore.c
-@@ -287,6 +287,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
- }
- 
- 
-+static int
-+virStorageSourceParseVitastorColonString(const char *colonstr,
-+                                         virStorageSource *src)
-+{
-+    char *p, *e, *next;
-+    g_autofree char *options = NULL;
-+
-+    /* optionally skip the "vitastor:" prefix if provided */
-+    if (STRPREFIX(colonstr, "vitastor:"))
-+        colonstr += strlen("vitastor:");
-+
-+    options = g_strdup(colonstr);
-+
-+    p = options;
-+    while (*p) {
-+        /* find : delimiter or end of string */
-+        for (e = p; *e && *e != ':'; ++e) {
-+            if (*e == '\\') {
-+                e++;
-+                if (*e == '\0')
-+                    break;
-+            }
-+        }
-+        if (*e == '\0') {
-+            next = e;    /* last kv pair */
-+        } else {
-+            next = e + 1;
-+            *e = '\0';
-+        }
-+
-+        if (STRPREFIX(p, "image=")) {
-+            src->path = g_strdup(p + strlen("image="));
-+        } else if (STRPREFIX(p, "etcd-prefix=")) {
-+            src->query = g_strdup(p + strlen("etcd-prefix="));
-+        } else if (STRPREFIX(p, "config-path=")) {
-+            src->configFile = g_strdup(p + strlen("config-path="));
-+        } else if (STRPREFIX(p, "etcd-host=")) {
-+            char *h, *sep;
-+
-+            h = p + strlen("etcd-host=");
-+            while (h < e) {
-+                for (sep = h; sep < e; ++sep) {
-+                    if (*sep == '\\' && (sep[1] == ',' ||
-+                                         sep[1] == ';' ||
-+                                         sep[1] == ' ')) {
-+                        *sep = '\0';
-+                        sep += 2;
-+                        break;
-+                    }
-+                }
-+
-+                if (virStorageSourceRBDAddHost(src, h) < 0)
-+                    return -1;
-+
-+                h = sep;
-+            }
-+        }
-+
-+        p = next;
-+    }
-+
-+    if (!src->path) {
-+        return -1;
-+    }
-+
-+    return 0;
-+}
-+
-+
- static int
- virStorageSourceParseNBDColonString(const char *nbdstr,
-                                     virStorageSource *src)
-@@ -399,6 +468,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
-             return -1;
-         break;
- 
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
-+            return -1;
-+        break;
-+
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-@@ -975,6 +1049,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
-     return 0;
- }
- 
-+static int
-+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
-+                                         virJSONValue *json,
-+                                         const char *jsonstr G_GNUC_UNUSED,
-+                                         int opaque G_GNUC_UNUSED)
-+{
-+    const char *filename;
-+    const char *image = virJSONValueObjectGetString(json, "image");
-+    const char *conf = virJSONValueObjectGetString(json, "config-path");
-+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
-+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
-+    size_t nservers;
-+    size_t i;
-+
-+    src->type = VIR_STORAGE_TYPE_NETWORK;
-+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
-+
-+    /* legacy syntax passed via 'filename' option */
-+    if ((filename = virJSONValueObjectGetString(json, "filename")))
-+        return virStorageSourceParseVitastorColonString(filename, src);
-+
-+    if (!image) {
-+        virReportError(VIR_ERR_INVALID_ARG, "%s",
-+                       _("missing image name in Vitastor backing volume "
-+                         "JSON specification"));
-+        return -1;
-+    }
-+
-+    src->path = g_strdup(image);
-+    src->configFile = g_strdup(conf);
-+    src->query = g_strdup(etcd_prefix);
-+
-+    if (servers) {
-+        nservers = virJSONValueArraySize(servers);
-+
-+        src->hosts = g_new0(virStorageNetHostDef, nservers);
-+        src->nhosts = nservers;
-+
-+        for (i = 0; i < nservers; i++) {
-+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
-+                                                                  virJSONValueArrayGet(servers, i)) < 0)
-+                return -1;
-+        }
-+    }
-+
-+    return 0;
-+}
-+
- static int
- virStorageSourceParseBackingJSONRaw(virStorageSource *src,
-                                     virJSONValue *json,
-@@ -1152,6 +1274,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
-     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
-     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
-     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
-+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
-     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
-     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
-     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
-diff --git a/src/test/test_driver.c b/src/test/test_driver.c
-index ed545848af..dbfdbe8476 100644
--- a/src/test/test_driver.c
-+++ b/src/test/test_driver.c
-@@ -7336,6 +7336,7 @@ testStorageVolumeTypeForPool(int pooltype)
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-         return VIR_STORAGE_VOL_NETWORK;
-     case VIR_STORAGE_POOL_LOGICAL:
-     case VIR_STORAGE_POOL_DISK:
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-index eee75af746..8bd0a57bdd 100644
--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='no'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-index 805950a937..852df0de16 100644
--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='yes'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
-index e8e40d695e..db55fe5f3a 100644
--- a/tests/storagepoolxml2argvtest.c
-+++ b/tests/storagepoolxml2argvtest.c
-@@ -65,6 +65,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_ZFS:
-     case VIR_STORAGE_POOL_VSTORAGE:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_LAST:
-     default:
-         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
-diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
-index 36f00cf643..5f5bd3464e 100644
--- a/tools/virsh-pool.c
-+++ b/tools/virsh-pool.c
-@@ -1223,6 +1223,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
-             case VIR_STORAGE_POOL_VSTORAGE:
-                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
-                 break;
-+            case VIR_STORAGE_POOL_VITASTOR:
-+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
-+                break;
-             case VIR_STORAGE_POOL_LAST:
-                 break;
-             }
-- 
-2.43.0
-
--- a/patches/libvirt-10.4-vitastor.diff
+++ b/patches/libvirt-10.4-vitastor.diff
@@ -1,643 +0,0 @@
-commit 1f7e90e36b2afca0312392979b96d31951a8d66b
-Author: Vitaliy Filippov <vitalif@yourcmc.ru>
-Date:   Thu Jun 27 01:34:54 2024 +0300
-
-    Add Vitastor support
-
-diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
-index aaad4a3da1..5f5daa8341 100644
--- a/include/libvirt/libvirt-storage.h
-+++ b/include/libvirt/libvirt-storage.h
-@@ -326,6 +326,7 @@ typedef enum {
-     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
-+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
- } virConnectListAllStoragePoolsFlags;
- 
- int                     virConnectListAllStoragePools(virConnectPtr conn,
-diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
-index fde594f811..66537db3e3 100644
--- a/src/conf/domain_conf.c
-+++ b/src/conf/domain_conf.c
-@@ -7220,7 +7220,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
-     src->configFile = virXPathString("string(./config/@file)", ctxt);
- 
-     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
-        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
-         src->query = virXMLPropString(node, "query");
- 
-     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
-@@ -30734,6 +30735,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
- 
-     case VIR_STORAGE_POOL_MPATH:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_SHEEPDOG:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_LAST:
-diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
-index 395e036e8f..8a0190f85b 100644
--- a/src/conf/domain_validate.c
-+++ b/src/conf/domain_validate.c
-@@ -495,6 +495,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-@@ -541,7 +542,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         }
-     }
- 
-    /* internal snapshots and config files are currently supported only with rbd: */
-+    /* internal snapshots are currently supported only with rbd: */
-     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
-         if (src->snapshot) {
-@@ -549,10 +550,15 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-                            _("<snapshot> element is currently supported only with 'rbd' disks"));
-             return -1;
-         }
-+    }
- 
-+    /* config files are currently supported only with rbd and vitastor: */
-+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
-         if (src->configFile) {
-             virReportError(VIR_ERR_XML_ERROR, "%s",
-                           _("<config> element is currently supported only with 'rbd' disks"));
-+                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
-             return -1;
-         }
-     }
-diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
-index a46a824f88..4c5b720643 100644
--- a/src/conf/schemas/domaincommon.rng
-+++ b/src/conf/schemas/domaincommon.rng
-@@ -1997,6 +1997,35 @@
-     </element>
-   </define>
- 
-+  <define name="diskSourceNetworkProtocolVitastor">
-+    <element name="source">
-+      <interleave>
-+        <attribute name="protocol">
-+          <value>vitastor</value>
-+        </attribute>
-+        <ref name="diskSourceCommon"/>
-+        <optional>
-+          <attribute name="name"/>
-+        </optional>
-+        <optional>
-+          <attribute name="query"/>
-+        </optional>
-+        <zeroOrMore>
-+          <ref name="diskSourceNetworkHost"/>
-+        </zeroOrMore>
-+        <optional>
-+          <element name="config">
-+            <attribute name="file">
-+              <ref name="absFilePath"/>
-+            </attribute>
-+            <empty/>
-+          </element>
-+        </optional>
-+        <empty/>
-+      </interleave>
-+    </element>
-+  </define>
-+
-   <define name="diskSourceNetworkProtocolISCSI">
-     <element name="source">
-       <attribute name="protocol">
-@@ -2347,6 +2376,7 @@
-       <ref name="diskSourceNetworkProtocolSimple"/>
-       <ref name="diskSourceNetworkProtocolVxHS"/>
-       <ref name="diskSourceNetworkProtocolNFS"/>
-+      <ref name="diskSourceNetworkProtocolVitastor"/>
-     </choice>
-   </define>
- 
-diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
-index 68842004b7..1d69a788b6 100644
--- a/src/conf/storage_conf.c
-+++ b/src/conf/storage_conf.c
-@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
-               "logical", "disk", "iscsi",
-               "iscsi-direct", "scsi", "mpath",
-               "rbd", "sheepdog", "gluster",
-              "zfs", "vstorage",
-+              "zfs", "vstorage", "vitastor",
- );
- 
- VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
-@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
-           .formatToString = virStorageFileFormatTypeToString,
-       }
-     },
-+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
-+     .poolOptions = {
-+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
-+                   VIR_STORAGE_POOL_SOURCE_NAME),
-+      },
-+      .volOptions = {
-+          .defaultFormat = VIR_STORAGE_FILE_RAW,
-+          .formatFromString = virStorageVolumeFormatFromString,
-+          .formatToString = virStorageFileFormatTypeToString,
-+      }
-+    },
-     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
-      .poolOptions = {
-          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
-                        _("element 'name' is mandatory for RBD pool"));
-         return -1;
-     }
-+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
-+        virReportError(VIR_ERR_XML_ERROR, "%s",
-+                       _("element 'name' is mandatory for Vitastor pool"));
-+        return -1;
-+    }
- 
-     if (options->formatFromString) {
-         g_autofree char *format = NULL;
-@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
-     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
-      * files, so they don't have a target */
-     if (def->type != VIR_STORAGE_POOL_RBD &&
-+        def->type != VIR_STORAGE_POOL_VITASTOR &&
-         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
-         def->type != VIR_STORAGE_POOL_GLUSTER &&
-         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
-diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
-index fc67957cfe..720c07ef74 100644
--- a/src/conf/storage_conf.h
-+++ b/src/conf/storage_conf.h
-@@ -103,6 +103,7 @@ typedef enum {
-     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
-     VIR_STORAGE_POOL_ZFS,      /* ZFS */
-     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
-+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
- 
-     VIR_STORAGE_POOL_LAST,
- } virStoragePoolType;
-@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
-+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
-diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
-index 959ec5ed40..e751dd4d6a 100644
--- a/src/conf/storage_source_conf.c
-+++ b/src/conf/storage_source_conf.c
-@@ -88,6 +88,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
-               "ssh",
-               "vxhs",
-               "nfs",
-+              "vitastor",
- );
- 
- 
-@@ -1301,6 +1302,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-             return 24007;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             /* we don't provide a default for RBD */
-             return 0;
-diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
-index 05b4bda16c..b5ed143c39 100644
--- a/src/conf/storage_source_conf.h
-+++ b/src/conf/storage_source_conf.h
-@@ -129,6 +129,7 @@ typedef enum {
-     VIR_STORAGE_NET_PROTOCOL_SSH,
-     VIR_STORAGE_NET_PROTOCOL_VXHS,
-     VIR_STORAGE_NET_PROTOCOL_NFS,
-+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
- 
-     VIR_STORAGE_NET_PROTOCOL_LAST
- } virStorageNetProtocol;
-diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
-index 59fa5da372..4739167f5f 100644
--- a/src/conf/virstorageobj.c
-+++ b/src/conf/virstorageobj.c
-@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
-             return 1;
-         break;
- 
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_RBD:
-     case VIR_STORAGE_POOL_LAST:
-@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
-                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
-                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
-+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
-+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
-                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
-diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
-index db7660aac4..561df34709 100644
--- a/src/libvirt-storage.c
-+++ b/src/libvirt-storage.c
-@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
-  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
-  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
-+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
-  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
-  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
-diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
-index 62e1be6672..71a1d42896 100644
--- a/src/libxl/libxl_conf.c
-+++ b/src/libxl/libxl_conf.c
-@@ -979,6 +979,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
-index 53f6871efc..c34b8cee1a 100644
--- a/src/libxl/xen_xl.c
-+++ b/src/libxl/xen_xl.c
-@@ -1456,6 +1456,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
-index 738b72d7ea..5dd082fc89 100644
--- a/src/qemu/qemu_block.c
-+++ b/src/qemu/qemu_block.c
-@@ -758,6 +758,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
- }
- 
- 
-+static virJSONValue *
-+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
-+{
-+    virJSONValue *ret = NULL;
-+    virStorageNetHostDef *host;
-+    size_t i;
-+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-+    g_autofree char *etcd = NULL;
-+
-+    for (i = 0; i < src->nhosts; i++) {
-+        host = src->hosts + i;
-+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
-+            return NULL;
-+        }
-+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
-+    }
-+    if (src->nhosts > 0) {
-+        etcd = virBufferContentAndReset(&buf);
-+    }
-+
-+    if (virJSONValueObjectAdd(&ret,
-+                              "S:etcd-host", etcd,
-+                              "S:etcd-prefix", src->query,
-+                              "S:config-path", src->configFile,
-+                              "s:image", src->path,
-+                              NULL) < 0)
-+        return NULL;
-+
-+    return ret;
-+}
-+
-+
- static virJSONValue *
- qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src)
- {
-@@ -1140,6 +1172,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
-                 return NULL;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return NULL;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             driver = "sheepdog";
-             if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src)))
-@@ -2020,6 +2058,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
- 
-             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             case VIR_STORAGE_NET_PROTOCOL_RBD:
-+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-             case VIR_STORAGE_NET_PROTOCOL_VXHS:
-             case VIR_STORAGE_NET_PROTOCOL_NFS:
-             case VIR_STORAGE_NET_PROTOCOL_SSH:
-@@ -2400,6 +2439,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
-                 return -1;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return -1;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             driver = "sheepdog";
-             if (!(location = qemuBlockStorageSourceGetSheepdogProps(src)))
-diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
-index bda62f2e5c..84b4e5f2b8 100644
--- a/src/qemu/qemu_domain.c
-+++ b/src/qemu/qemu_domain.c
-@@ -5260,7 +5260,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
-     if (src->query &&
-         (actualType != VIR_STORAGE_TYPE_NETWORK ||
-          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
-          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
-         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                        _("query is supported only with HTTP(S) protocols"));
-         return -1;
-@@ -10514,6 +10515,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
-         break;
- 
-     case VIR_STORAGE_NET_PROTOCOL_RBD:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
-index f5260c4a22..2f9d8406fe 100644
--- a/src/qemu/qemu_snapshot.c
-+++ b/src/qemu/qemu_snapshot.c
-@@ -423,6 +423,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-@@ -648,6 +649,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
-index 86c03762d2..630c6eff1a 100644
--- a/src/storage/storage_driver.c
-+++ b/src/storage/storage_driver.c
-@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
- 
-         case VIR_STORAGE_POOL_GLUSTER:
-         case VIR_STORAGE_POOL_RBD:
-+        case VIR_STORAGE_POOL_VITASTOR:
-         case VIR_STORAGE_POOL_SHEEPDOG:
-         case VIR_STORAGE_POOL_ZFS:
-         case VIR_STORAGE_POOL_LAST:
-diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
-index 80681924ea..8a3ade9ec0 100644
--- a/src/storage_file/storage_source_backingstore.c
-+++ b/src/storage_file/storage_source_backingstore.c
-@@ -287,6 +287,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
- }
- 
- 
-+static int
-+virStorageSourceParseVitastorColonString(const char *colonstr,
-+                                         virStorageSource *src)
-+{
-+    char *p, *e, *next;
-+    g_autofree char *options = NULL;
-+
-+    /* optionally skip the "vitastor:" prefix if provided */
-+    if (STRPREFIX(colonstr, "vitastor:"))
-+        colonstr += strlen("vitastor:");
-+
-+    options = g_strdup(colonstr);
-+
-+    p = options;
-+    while (*p) {
-+        /* find : delimiter or end of string */
-+        for (e = p; *e && *e != ':'; ++e) {
-+            if (*e == '\\') {
-+                e++;
-+                if (*e == '\0')
-+                    break;
-+            }
-+        }
-+        if (*e == '\0') {
-+            next = e;    /* last kv pair */
-+        } else {
-+            next = e + 1;
-+            *e = '\0';
-+        }
-+
-+        if (STRPREFIX(p, "image=")) {
-+            src->path = g_strdup(p + strlen("image="));
-+        } else if (STRPREFIX(p, "etcd-prefix=")) {
-+            src->query = g_strdup(p + strlen("etcd-prefix="));
-+        } else if (STRPREFIX(p, "config-path=")) {
-+            src->configFile = g_strdup(p + strlen("config-path="));
-+        } else if (STRPREFIX(p, "etcd-host=")) {
-+            char *h, *sep;
-+
-+            h = p + strlen("etcd-host=");
-+            while (h < e) {
-+                for (sep = h; sep < e; ++sep) {
-+                    if (*sep == '\\' && (sep[1] == ',' ||
-+                                         sep[1] == ';' ||
-+                                         sep[1] == ' ')) {
-+                        *sep = '\0';
-+                        sep += 2;
-+                        break;
-+                    }
-+                }
-+
-+                if (virStorageSourceRBDAddHost(src, h) < 0)
-+                    return -1;
-+
-+                h = sep;
-+            }
-+        }
-+
-+        p = next;
-+    }
-+
-+    if (!src->path) {
-+        return -1;
-+    }
-+
-+    return 0;
-+}
-+
-+
- static int
- virStorageSourceParseNBDColonString(const char *nbdstr,
-                                     virStorageSource *src)
-@@ -399,6 +468,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
-             return -1;
-         break;
- 
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
-+            return -1;
-+        break;
-+
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-@@ -975,6 +1049,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
-     return 0;
- }
- 
-+static int
-+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
-+                                         virJSONValue *json,
-+                                         const char *jsonstr G_GNUC_UNUSED,
-+                                         int opaque G_GNUC_UNUSED)
-+{
-+    const char *filename;
-+    const char *image = virJSONValueObjectGetString(json, "image");
-+    const char *conf = virJSONValueObjectGetString(json, "config-path");
-+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
-+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
-+    size_t nservers;
-+    size_t i;
-+
-+    src->type = VIR_STORAGE_TYPE_NETWORK;
-+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
-+
-+    /* legacy syntax passed via 'filename' option */
-+    if ((filename = virJSONValueObjectGetString(json, "filename")))
-+        return virStorageSourceParseVitastorColonString(filename, src);
-+
-+    if (!image) {
-+        virReportError(VIR_ERR_INVALID_ARG, "%s",
-+                       _("missing image name in Vitastor backing volume "
-+                         "JSON specification"));
-+        return -1;
-+    }
-+
-+    src->path = g_strdup(image);
-+    src->configFile = g_strdup(conf);
-+    src->query = g_strdup(etcd_prefix);
-+
-+    if (servers) {
-+        nservers = virJSONValueArraySize(servers);
-+
-+        src->hosts = g_new0(virStorageNetHostDef, nservers);
-+        src->nhosts = nservers;
-+
-+        for (i = 0; i < nservers; i++) {
-+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
-+                                                                  virJSONValueArrayGet(servers, i)) < 0)
-+                return -1;
-+        }
-+    }
-+
-+    return 0;
-+}
-+
- static int
- virStorageSourceParseBackingJSONRaw(virStorageSource *src,
-                                     virJSONValue *json,
-@@ -1152,6 +1274,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
-     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
-     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
-     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
-+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
-     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
-     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
-     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
-diff --git a/src/test/test_driver.c b/src/test/test_driver.c
-index d2d1bc43e3..31a92e4a01 100644
--- a/src/test/test_driver.c
-+++ b/src/test/test_driver.c
-@@ -7339,6 +7339,7 @@ testStorageVolumeTypeForPool(int pooltype)
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-         return VIR_STORAGE_VOL_NETWORK;
-     case VIR_STORAGE_POOL_LOGICAL:
-     case VIR_STORAGE_POOL_DISK:
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-index eee75af746..8bd0a57bdd 100644
--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='no'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-index 805950a937..852df0de16 100644
--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='yes'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
-index e8e40d695e..db55fe5f3a 100644
--- a/tests/storagepoolxml2argvtest.c
-+++ b/tests/storagepoolxml2argvtest.c
-@@ -65,6 +65,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_ZFS:
-     case VIR_STORAGE_POOL_VSTORAGE:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_LAST:
-     default:
-         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
-diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
-index f9aad8ded0..64704b4288 100644
--- a/tools/virsh-pool.c
-+++ b/tools/virsh-pool.c
-@@ -1187,6 +1187,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
-             case VIR_STORAGE_POOL_VSTORAGE:
-                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
-                 break;
-+            case VIR_STORAGE_POOL_VITASTOR:
-+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
-+                break;
-             case VIR_STORAGE_POOL_LAST:
-                 break;
-             }
--- a/patches/nova-28.diff
+++ b/patches/nova-28.diff
@@ -1,288 +0,0 @@
-diff --git a/nova/virt/image/model.py b/nova/virt/image/model.py
-index 971f7e9c07..ec3fca72cb 100644
--- a/nova/virt/image/model.py
-+++ b/nova/virt/image/model.py
-@@ -129,3 +129,22 @@ class RBDImage(Image):
-         self.user = user
-         self.password = password
-         self.servers = servers
-+
-+
-+class VitastorImage(Image):
-+    """Class for images in a remote Vitastor cluster"""
-+
-+    def __init__(self, name, etcd_address = None, etcd_prefix = None, config_path = None):
-+        """Create a new Vitastor image object
-+
-+        :param name: name of the image
-+        :param etcd_address: etcd URL(s) (optional)
-+        :param etcd_prefix: etcd prefix (optional)
-+        :param config_path: path to the configuration (optional)
-+        """
-+        super(VitastorImage, self).__init__(FORMAT_RAW)
-+
-+        self.name = name
-+        self.etcd_address = etcd_address
-+        self.etcd_prefix = etcd_prefix
-+        self.config_path = config_path
-diff --git a/nova/virt/images.py b/nova/virt/images.py
-index 5358f3766a..ebe3d6effb 100644
--- a/nova/virt/images.py
-+++ b/nova/virt/images.py
-@@ -41,7 +41,7 @@ IMAGE_API = glance.API()
- 
- def qemu_img_info(path, format=None):
-     """Return an object containing the parsed output from qemu-img info."""
-    if not os.path.exists(path) and not path.startswith('rbd:'):
-+    if not os.path.exists(path) and not path.startswith('rbd:') and not path.startswith('vitastor:'):
-         raise exception.DiskNotFound(location=path)
- 
-     info = nova.privsep.qemu.unprivileged_qemu_img_info(path, format=format)
-@@ -50,7 +50,7 @@ def qemu_img_info(path, format=None):
- 
- def privileged_qemu_img_info(path, format=None, output_format='json'):
-     """Return an object containing the parsed output from qemu-img info."""
-    if not os.path.exists(path) and not path.startswith('rbd:'):
-+    if not os.path.exists(path) and not path.startswith('rbd:') and not path.startswith('vitastor:'):
-         raise exception.DiskNotFound(location=path)
- 
-     info = nova.privsep.qemu.privileged_qemu_img_info(path, format=format)
-diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py
-index f9475776b3..a2e18aab67 100644
--- a/nova/virt/libvirt/config.py
-+++ b/nova/virt/libvirt/config.py
-@@ -1060,6 +1060,8 @@ class LibvirtConfigGuestDisk(LibvirtConfigGuestDevice):
-         self.driver_iommu = False
-         self.source_path = None
-         self.source_protocol = None
-+        self.source_query = None
-+        self.source_config = None
-         self.source_name = None
-         self.source_hosts = []
-         self.source_ports = []
-@@ -1189,6 +1191,10 @@ class LibvirtConfigGuestDisk(LibvirtConfigGuestDevice):
-             source = etree.Element("source", protocol=self.source_protocol)
-             if self.source_name is not None:
-                 source.set('name', self.source_name)
-+            if self.source_query is not None:
-+                source.set('query', self.source_query)
-+            if self.source_config is not None:
-+                source.append(etree.Element('config', file=self.source_config))
-             hosts_info = zip(self.source_hosts, self.source_ports)
-             for name, port in hosts_info:
-                 host = etree.Element('host', name=name)
-diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
-index 391231c527..f38faa1608 100644
--- a/nova/virt/libvirt/driver.py
-+++ b/nova/virt/libvirt/driver.py
-@@ -179,6 +179,7 @@ VOLUME_DRIVERS = {
-     'local': 'nova.virt.libvirt.volume.volume.LibvirtVolumeDriver',
-     'fake': 'nova.virt.libvirt.volume.volume.LibvirtFakeVolumeDriver',
-     'rbd': 'nova.virt.libvirt.volume.net.LibvirtNetVolumeDriver',
-+    'vitastor': 'nova.virt.libvirt.volume.vitastor.LibvirtVitastorVolumeDriver',
-     'nfs': 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver',
-     'smbfs': 'nova.virt.libvirt.volume.smbfs.LibvirtSMBFSVolumeDriver',
-     'fibre_channel': 'nova.virt.libvirt.volume.fibrechannel.LibvirtFibreChannelVolumeDriver',  # noqa:E501
-@@ -385,10 +386,10 @@ class LibvirtDriver(driver.ComputeDriver):
-         # This prevents the risk of one test setting a capability
-         # which bleeds over into other tests.
- 
-        # LVM and RBD require raw images. If we are not configured to
-+        # LVM, RBD, Vitastor require raw images. If we are not configured to
-         # force convert images into raw format, then we _require_ raw
-         # images only.
-        raw_only = ('rbd', 'lvm')
-+        raw_only = ('rbd', 'lvm', 'vitastor')
-         requires_raw_image = (CONF.libvirt.images_type in raw_only and
-                               not CONF.force_raw_images)
-         requires_ploop_image = CONF.libvirt.virt_type == 'parallels'
-@@ -775,12 +776,12 @@ class LibvirtDriver(driver.ComputeDriver):
-         # Some imagebackends are only able to import raw disk images,
-         # and will fail if given any other format. See the bug
-         # https://bugs.launchpad.net/nova/+bug/1816686 for more details.
-        if CONF.libvirt.images_type in ('rbd',):
-+        if CONF.libvirt.images_type in ('rbd', 'vitastor'):
-             if not CONF.force_raw_images:
-                 msg = _("'[DEFAULT]/force_raw_images = False' is not "
-                        "allowed with '[libvirt]/images_type = rbd'. "
-+                        "allowed with '[libvirt]/images_type = rbd' or 'vitastor'. "
-                         "Please check the two configs and if you really "
-                        "do want to use rbd as images_type, set "
-+                        "do want to use rbd or vitastor as images_type, set "
-                         "force_raw_images to True.")
-                 raise exception.InvalidConfiguration(msg)
- 
-@@ -2603,6 +2604,16 @@ class LibvirtDriver(driver.ComputeDriver):
-                     if connection_info['data'].get('auth_enabled'):
-                         username = connection_info['data']['auth_username']
-                         path = f"rbd:{volume_name}:id={username}"
-+                elif connection_info['driver_volume_type'] == 'vitastor':
-+                    volume_name = connection_info['data']['name']
-+                    path = 'vitastor:image='+volume_name.replace(':', '\\:')
-+                    for k in [ 'config_path', 'etcd_address', 'etcd_prefix' ]:
-+                        if k in connection_info['data']:
-+                            kk = k
-+                            if kk == 'etcd_address':
-+                                # FIXME use etcd_address in qemu driver
-+                                kk = 'etcd_host'
-+                            path += ":"+kk.replace('_', '-')+"="+connection_info['data'][k].replace(':', '\\:')
-                 else:
-                     path = 'unknown'
-                     raise exception.DiskNotFound(location='unknown')
-@@ -2827,8 +2838,8 @@ class LibvirtDriver(driver.ComputeDriver):
- 
-         image_format = CONF.libvirt.snapshot_image_format or source_type
- 
-        # NOTE(bfilippov): save lvm and rbd as raw
-        if image_format == 'lvm' or image_format == 'rbd':
-+        # NOTE(bfilippov): save lvm and rbd and vitastor as raw
-+        if image_format == 'lvm' or image_format == 'rbd' or image_format == 'vitastor':
-             image_format = 'raw'
- 
-         metadata = self._create_snapshot_metadata(instance.image_meta,
-@@ -2899,7 +2910,7 @@ class LibvirtDriver(driver.ComputeDriver):
-                               expected_state=task_states.IMAGE_UPLOADING)
- 
-             # TODO(nic): possibly abstract this out to the root_disk
-            if source_type == 'rbd' and live_snapshot:
-+            if (source_type == 'rbd' or source_type == 'vitastor') and live_snapshot:
-                 # Standard snapshot uses qemu-img convert from RBD which is
-                 # not safe to run with live_snapshot.
-                 live_snapshot = False
-@@ -4099,7 +4110,7 @@ class LibvirtDriver(driver.ComputeDriver):
-         # cleanup rescue volume
-         lvm.remove_volumes([lvmdisk for lvmdisk in self._lvm_disks(instance)
-                                 if lvmdisk.endswith('.rescue')])
-        if CONF.libvirt.images_type == 'rbd':
-+        if CONF.libvirt.images_type == 'rbd' or CONF.libvirt.images_type == 'vitastor':
-             filter_fn = lambda disk: (disk.startswith(instance.uuid) and
-                                       disk.endswith('.rescue'))
-             rbd_utils.RBDDriver().cleanup_volumes(filter_fn)
-@@ -4356,6 +4367,8 @@ class LibvirtDriver(driver.ComputeDriver):
-         # TODO(mikal): there is a bug here if images_type has
-         # changed since creation of the instance, but I am pretty
-         # sure that this bug already exists.
-+        if CONF.libvirt.images_type == 'vitastor':
-+            return 'vitastor'
-         return 'rbd' if CONF.libvirt.images_type == 'rbd' else 'raw'
- 
-     @staticmethod
-@@ -4764,10 +4777,10 @@ class LibvirtDriver(driver.ComputeDriver):
-                 finally:
-                     # NOTE(mikal): if the config drive was imported into RBD,
-                     # then we no longer need the local copy
-                    if CONF.libvirt.images_type == 'rbd':
-+                    if CONF.libvirt.images_type == 'rbd' or CONF.libvirt.images_type == 'vitastor':
-                         LOG.info('Deleting local config drive %(path)s '
-                                 'because it was imported into RBD.',
-                                 {'path': config_disk_local_path},
-+                                 'because it was imported into %(type).',
-+                                 {'path': config_disk_local_path, 'type': CONF.libvirt.images_type},
-                                  instance=instance)
-                         os.unlink(config_disk_local_path)
- 
-diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py
-index da2a6e8b8a..52c02e72f1 100644
--- a/nova/virt/libvirt/utils.py
-+++ b/nova/virt/libvirt/utils.py
-@@ -340,6 +340,10 @@ def find_disk(guest: libvirt_guest.Guest) -> ty.Tuple[str, ty.Optional[str]]:
-             disk_path = disk.source_name
-             if disk_path:
-                 disk_path = 'rbd:' + disk_path
-+        elif not disk_path and disk.source_protocol == 'vitastor':
-+            disk_path = disk.source_name
-+            if disk_path:
-+                disk_path = 'vitastor:' + disk_path
- 
-     if not disk_path:
-         raise RuntimeError(_("Can't retrieve root device path "
-@@ -354,6 +358,8 @@ def get_disk_type_from_path(path: str) -> ty.Optional[str]:
-         return 'lvm'
-     elif path.startswith('rbd:'):
-         return 'rbd'
-+    elif path.startswith('vitastor:'):
-+        return 'vitastor'
-     elif (os.path.isdir(path) and
-           os.path.exists(os.path.join(path, "DiskDescriptor.xml"))):
-         return 'ploop'
-diff --git a/nova/virt/libvirt/volume/vitastor.py b/nova/virt/libvirt/volume/vitastor.py
-new file mode 100644
-index 0000000000..0256df62c1
--- /dev/null
-+++ b/nova/virt/libvirt/volume/vitastor.py
-@@ -0,0 +1,75 @@
-+# Copyright (c) 2021+, Vitaliy Filippov <vitalif@yourcmc.ru>
-+#
-+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-+#    not use this file except in compliance with the License. You may obtain
-+#    a copy of the License at
-+#
-+#         http://www.apache.org/licenses/LICENSE-2.0
-+#
-+#    Unless required by applicable law or agreed to in writing, software
-+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-+#    License for the specific language governing permissions and limitations
-+#    under the License.
-+
-+from os_brick import exception as os_brick_exception
-+from os_brick import initiator
-+from os_brick.initiator import connector
-+from oslo_log import log as logging
-+
-+import nova.conf
-+from nova import utils
-+from nova.virt.libvirt.volume import volume as libvirt_volume
-+
-+
-+CONF = nova.conf.CONF
-+LOG = logging.getLogger(__name__)
-+
-+
-+class LibvirtVitastorVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver):
-+    """Driver to attach Vitastor volumes to libvirt."""
-+    def __init__(self, host):
-+        super(LibvirtVitastorVolumeDriver, self).__init__(host, is_block_dev=False)
-+
-+    def connect_volume(self, connection_info, instance):
-+        pass
-+
-+    def disconnect_volume(self, connection_info, instance, force=False):
-+        pass
-+
-+    def get_config(self, connection_info, disk_info):
-+        """Returns xml for libvirt."""
-+        conf = super(LibvirtVitastorVolumeDriver, self).get_config(connection_info, disk_info)
-+        conf.source_type = 'network'
-+        conf.source_protocol = 'vitastor'
-+        conf.source_name = connection_info['data'].get('name')
-+        conf.source_query = connection_info['data'].get('etcd_prefix') or None
-+        conf.source_config = connection_info['data'].get('config_path') or None
-+        conf.source_hosts = []
-+        conf.source_ports = []
-+        addresses = connection_info['data'].get('etcd_address', '')
-+        if addresses:
-+            if not isinstance(addresses, list):
-+                addresses = addresses.split(',')
-+            for addr in addresses:
-+                if addr.startswith('https://'):
-+                    raise NotImplementedError('Vitastor block driver does not support SSL for etcd communication yet')
-+                if addr.startswith('http://'):
-+                    addr = addr[7:]
-+                addr = addr.rstrip('/')
-+                if addr.endswith('/v3'):
-+                    addr = addr[0:-3]
-+                p = addr.find('/')
-+                if p > 0:
-+                    raise NotImplementedError('libvirt does not support custom URL paths for Vitastor etcd yet. Use /etc/vitastor/vitastor.conf')
-+                p = addr.find(':')
-+                port = '2379'
-+                if p > 0:
-+                    port = addr[p+1:]
-+                    addr = addr[0:p]
-+                conf.source_hosts.append(addr)
-+                conf.source_ports.append(port)
-+        return conf
-+
-+    def extend_volume(self, connection_info, instance, requested_size):
-+        return requested_size
--- a/patches/qemu-8.2-vitastor.patch
+++ b/patches/qemu-8.2-vitastor.patch
@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 59ff6d380c..abde3715c2 100644
--- a/block/meson.build
-+++ b/block/meson.build
-@@ -109,6 +109,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 6c77d9687d..390683ee71 100644
--- a/meson.build
-+++ b/meson.build
-@@ -1295,6 +1295,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'))
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -2157,6 +2177,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
- config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
-@@ -4356,6 +4377,7 @@ summary_info += {'fdt support':       fdt_opt == 'disabled' ? false : fdt_opt}
- summary_info += {'libcap-ng support': libcap_ng}
- summary_info += {'bpf support':       libbpf}
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index c9baeda639..85e1df5a56 100644
--- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index ca390c5700..8f11ae9fa5 100644
--- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3201,7 +3201,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
-            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4255,6 +4255,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4713,6 +4735,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5148,6 +5171,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5370,6 +5404,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index 76781f17f4..ac5fe3aa08 100755
--- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -30,7 +30,7 @@
- --with-suffix="qemu-kvm" \
- --firmwarepath=/usr/share/qemu-firmware \
- --target-list="x86_64-softmmu" \
---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -176,6 +176,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 680fa3f581..dab422bf04 100644
--- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -168,6 +168,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
-@@ -445,6 +446,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-relocatable) printf "%s" -Drelocatable=true ;;
--- a/patches/qemu-9.0-vitastor.patch
+++ b/patches/qemu-9.0-vitastor.patch
@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index e1f03fd773..db0cfb2321 100644
--- a/block/meson.build
-+++ b/block/meson.build
-@@ -114,6 +114,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 91a0aa64c6..e8bc710578 100644
--- a/meson.build
-+++ b/meson.build
-@@ -1452,6 +1452,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'))
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -2250,6 +2270,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
- config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
-@@ -4443,6 +4464,7 @@ summary_info += {'fdt support':       fdt_opt == 'disabled' ? false : fdt_opt}
- summary_info += {'libcap-ng support': libcap_ng}
- summary_info += {'bpf support':       libbpf}
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index 0a99a059ec..16dc440118 100644
--- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index 746d1694c2..fb7aa4423b 100644
--- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3203,7 +3203,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
-            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4285,6 +4285,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4741,6 +4763,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5180,6 +5203,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5402,6 +5436,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index 76781f17f4..ac5fe3aa08 100755
--- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -30,7 +30,7 @@
- --with-suffix="qemu-kvm" \
- --firmwarepath=/usr/share/qemu-firmware \
- --target-list="x86_64-softmmu" \
---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -176,6 +176,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 680fa3f581..dab422bf04 100644
--- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -168,6 +168,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
-@@ -445,6 +446,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-relocatable) printf "%s" -Drelocatable=true ;;
--- a/src/blockstore/blockstore_flush.cpp
+++ b/src/blockstore/blockstore_flush.cpp
@@ -366,7 +366,6 @@ resume_0:
        !flusher->flush_queue.size() || !flusher->dequeuing)
    {
 stop_flusher:
-        flusher->dequeuing = false;
        if (flusher->trim_wanted > 0 && try_trim)
        {
            // Attempt forced trim
@@ -374,6 +373,7 @@ stop_flusher:
            flusher->active_flushers++;
            goto trim_journal;
        }
+        flusher->dequeuing = false;
        wait_state = 0;
        return true;
    }
--- a/src/client/http_client.cpp
+++ b/src/client/http_client.cpp
@@ -314,7 +314,7 @@ void http_co_t::start_connection()
        stackout();
        return;
    }
-    tfd->set_fd_handler(peer_fd, EPOLLIN|EPOLLOUT, [this](int peer_fd, int epoll_events)
+    tfd->set_fd_handler(peer_fd, EPOLLOUT, [this](int peer_fd, int epoll_events)
    {
        this->epoll_events |= epoll_events;
        handle_events();
--- a/src/client/messenger.cpp
+++ b/src/client/messenger.cpp
@@ -18,8 +18,6 @@
 #include <sys/poll.h>
 #include <sys/eventfd.h>

-static uint64_t one = 1;
-
 msgr_iothread_t::msgr_iothread_t()
 {
    ring = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
@@ -29,7 +27,7 @@ msgr_iothread_t::msgr_iothread_t()
    {
        throw std::runtime_error(std::string("failed to create eventfd: ")+strerror(errno));
    }
-    epmgr->tfd->set_fd_handler(submit_eventfd, EPOLLIN, [this](int fd, int epoll_events)
+    epmgr->tfd->set_fd_handler(submit_eventfd, false, [this](int fd, int epoll_events)
    {
        // Reset eventfd counter
        uint64_t ctr = 0;
@@ -38,15 +36,9 @@ msgr_iothread_t::msgr_iothread_t()
        {
            fprintf(stderr, "Error resetting eventfd: %s\n", strerror(errno));
        }
-        ring->wakeup();
+        // Submit pending events
+        submit_pending_writes();
    });
-    consumer.loop = [this]()
-    {
-        read_requests();
-        send_replies();
-        ring->submit();
-    };
-    ring->register_consumer(&consumer);
    thread = new std::thread(&msgr_iothread_t::run, this);
 }

@@ -67,67 +59,22 @@ void msgr_iothread_t::stop()
        return;
    }
    stopped = true;
-    write(submit_eventfd, &one, sizeof(one));
+    if (callback_stopped)
+    {
+        *callback_stopped = true;
+        callback_stopped = NULL;
+    }
+    close(submit_eventfd);
    mu.unlock();
    thread->join();
-    ring->unregister_consumer(&consumer);
-    close(submit_eventfd);
 }

-void msgr_iothread_t::add_client(osd_client_t *cl)
-{
-    mu.lock();
-    if (stopped)
-    {
-        mu.unlock();
-        return;
-    }
-    assert(!clients[cl->peer_fd]);
-    clients[cl->peer_fd] = cl;
-    epmgr->tfd->set_fd_handler(cl->peer_fd, EPOLLIN, [this](int peer_fd, int epoll_events)
-    {
-        // FIXME: Slight copypaste (see handle_peer_epoll)
-        if (epoll_events & EPOLLIN)
-        {
-            auto cl_it = clients.find(peer_fd);
-            if (cl_it != clients.end())
-            {
-                auto cl = cl_it->second;
-                cl->mu.lock();
-                cl->read_ready++;
-                if (cl->read_ready == 1)
-                {
-                    read_ready_clients.push_back(peer_fd);
-                    ring->wakeup();
-                }
-                cl->mu.unlock();
-            }
-        }
-    });
-    mu.unlock();
-}
+static uint64_t one = 1;

-void msgr_iothread_t::remove_client(osd_client_t *cl)
+void msgr_iothread_t::wakeup(osd_client_t *cl, ring_loop_t *outer_ring)
 {
-    mu.lock();
-    if (stopped)
-    {
-        mu.unlock();
-        return;
-    }
-    auto cl_it = clients.find(cl->peer_fd);
-    if (cl_it != clients.end() && cl_it->second == cl)
-    {
-        clients.erase(cl->peer_fd);
-        epmgr->tfd->set_fd_handler(cl->peer_fd, 0, NULL);
-    }
-    mu.unlock();
-}
-
-void msgr_iothread_t::wakeup_out(int peer_fd, ring_loop_t *outer_ring)
-{
-    write_ready_mu.lock();
-    if (!write_ready_clients.size())
+    std::unique_lock<std::mutex> lock(mu);
+    if (!pending_clients.size())
    {
        io_uring_sqe* sqe = outer_ring->get_sqe();
        if (!sqe)
@@ -141,63 +88,52 @@ void msgr_iothread_t::wakeup_out(int peer_fd, ring_loop_t *outer_ring)
            my_uring_prep_write(sqe, submit_eventfd, &one, sizeof(one), 0);
        }
    }
-    write_ready_clients.push_back(peer_fd);
-    write_ready_mu.unlock();
+    add_pending(cl);
 }

-void msgr_iothread_t::read_requests()
+void msgr_iothread_t::add_pending(osd_client_t *cl)
 {
-    // FIXME: Slight copypaste (see messenger_t::read_requests)
-    auto to_recv = std::move(read_ready_clients);
-    for (int i = 0; i < to_recv.size(); i++)
+    auto it = std::lower_bound(pending_clients.begin(), pending_clients.end(), cl);
+    if (it == pending_clients.end() || *it != cl)
    {
-        int peer_fd = to_recv[i];
-        auto cl_it = clients.find(peer_fd);
-        if (cl_it == clients.end())
-        {
-            continue;
-        }
-        osd_client_t *cl = cl_it->second;
-        cl->mu.lock();
-        auto ok = cl->try_recv(ring, false);
-        cl->mu.unlock();
-        if (!ok)
-        {
-            read_ready_clients.insert(read_ready_clients.end(), to_recv.begin()+i, to_recv.end());
-            break;
-        }
+        pending_clients.insert(it, cl);
    }
 }

-void msgr_iothread_t::send_replies()
+void msgr_iothread_t::submit_pending_writes()
 {
-    if (stopped)
+    mu.lock();
+    std::vector<osd_client_t*> clients = std::move(pending_clients);
+    mu.unlock();
+    for (int i = 0; i < clients.size(); i++)
    {
-        return;
-    }
-    write_ready_mu.lock();
-    auto to_send = std::move(write_ready_clients);
-    write_ready_mu.unlock();
-    for (int i = 0; i < to_send.size(); i++)
-    {
-        auto cl_it = clients.find(to_send[i]);
-        if (cl_it == clients.end())
+        auto cl = clients[i];
+        std::unique_lock<std::mutex> lock(cl->mu);
+        if (cl->peer_state == PEER_STOPPED)
        {
+            cl->refs--;
+            if (cl->refs <= 0)
+                delete cl;
            continue;
        }
-        auto cl = cl_it->second;
-        cl->mu.lock();
-        auto ok = cl->try_send(ring, false/*, lock*/);
-        cl->mu.unlock();
-        if (!ok)
+        if (!cl->try_send(ring, false/*, lock*/))
        {
            // ring is full (rare but what if...)
-            write_ready_mu.lock();
-            write_ready_clients.insert(write_ready_clients.end(), to_send.begin()+i, to_send.end());
-            write_ready_mu.unlock();
-            break;
+            ring->submit();
+            mu.lock();
+            for (; i < clients.size(); i++)
+            {
+                add_pending(clients[i]);
+            }
+            mu.unlock();
+            return;
+        }
+        else
+        {
+            cl->refs--;
        }
    }
+    ring->submit();
 }

 void msgr_iothread_t::run()
@@ -206,12 +142,9 @@ void msgr_iothread_t::run()
    {
        mu.lock();
        if (stopped)
-        {
-            mu.unlock();
            return;
-        }
-        ring->loop();
        mu.unlock();
+        ring->loop();
        ring->wait();
    }
 }
@@ -251,36 +184,11 @@ void osd_messenger_t::init()
            auto iot = new msgr_iothread_t();
            iothreads.push_back(iot);
        }
-        immediates_eventfd = eventfd(0, EFD_CLOEXEC|EFD_NONBLOCK);
-        if (immediates_eventfd < 0)
+        completion_eventfd = eventfd(0, EFD_CLOEXEC|EFD_NONBLOCK);
+        if (completion_eventfd < 0)
        {
-            throw std::runtime_error(std::string("failed to create set_immediate eventfd: ")+strerror(errno));
+            throw std::runtime_error(std::string("failed to create completion eventfd: ")+strerror(errno));
        }
-        tfd->set_fd_handler(immediates_eventfd, EPOLLIN, [this](int peer_fd, int epoll_events)
-        {
-            // Reset eventfd counter
-            uint64_t ctr = 0;
-            int r = read(immediates_eventfd, &ctr, 8);
-            if (r < 0 && errno != EAGAIN && errno != EINTR)
-            {
-                fprintf(stderr, "Error resetting eventfd: %s\n", strerror(errno));
-            }
-            while (true)
-            {
-                immediates_mu.lock();
-                auto to_run = std::move(immediates);
-                immediates_mu.unlock();
-                if (!to_run.size())
-                {
-                    break;
-                }
-                for (auto & cb: to_run)
-                {
-                    cb();
-                }
-            }
-            ringloop->wakeup();
-        });
    }
    keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
    {
@@ -359,11 +267,10 @@ void osd_messenger_t::init()

 osd_messenger_t::~osd_messenger_t()
 {
-    if (immediates_eventfd >= 0)
+    if (completion_eventfd >= 0)
    {
-        tfd->set_fd_handler(immediates_eventfd, 0, NULL);
-        close(immediates_eventfd);
-        immediates_eventfd = -1;
+        close(completion_eventfd);
+        completion_eventfd = -1;
    }
    if (keepalive_timer_id >= 0)
    {
@@ -418,10 +325,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
        this->rdma_max_msg = 129*1024;
    this->rdma_odp = config["rdma_odp"].bool_value();
 #endif
-    if (!osd_num)
-        this->iothread_count = config["client_iothread_count"].is_null() ? 4 : (uint32_t)config["client_iothread_count"].uint64_value();
-    else
-        this->iothread_count = (uint32_t)config["osd_iothread_count"].uint64_value();
    this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value();
    if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024)
        this->receive_buffer_size = 65536;
@@ -512,7 +415,6 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
    {
        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
    }
-    clients[peer_fd]->msgr = this;
    clients[peer_fd]->peer_addr = addr;
    clients[peer_fd]->peer_port = peer_port;
    clients[peer_fd]->peer_fd = peer_fd;
@@ -521,7 +423,7 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
    clients[peer_fd]->osd_num = peer_osd;
    clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
    clients[peer_fd]->receive_buffer_size = receive_buffer_size;
-    tfd->set_fd_handler(peer_fd, EPOLLIN|EPOLLOUT, [this](int peer_fd, int epoll_events)
+    tfd->set_fd_handler(peer_fd, EPOLLOUT, [this](int peer_fd, int epoll_events)
    {
        // Either OUT (connected) or HUP
        handle_connect_epoll(peer_fd);
@@ -562,11 +464,7 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
    int one = 1;
    setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
    cl->peer_state = PEER_CONNECTED;
-    if (iothreads.size())
-    {
-        iothreads[peer_fd % iothreads.size()]->add_client(cl);
-    }
-    tfd->set_fd_handler(peer_fd, iothreads.size() ? 0 : EPOLLIN, [this](int peer_fd, int epoll_events)
+    tfd->set_fd_handler(peer_fd, EPOLLIN, [this](int peer_fd, int epoll_events)
    {
        handle_peer_epoll(peer_fd, epoll_events);
    });
@@ -590,6 +488,14 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
    {
        // Mark client as ready (i.e. some data is available)
        auto cl = clients[peer_fd];
+#ifdef WITH_RDMA
+        if (cl->peer_state == PEER_RDMA)
+        {
+            // FIXME: Do something better than just forgetting the FD
+            // FIXME: Ignore pings during RDMA state transition
+            return;
+        }
+#endif
        cl->read_ready++;
        if (cl->read_ready == 1)
        {
@@ -750,7 +656,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                    fprintf(stderr, "Connected to OSD %ju using RDMA\n", cl->osd_num);
                }
                cl->peer_state = PEER_RDMA;
-                tfd->set_fd_handler(cl->peer_fd, 0, [this](int peer_fd, int epoll_events)
+                tfd->set_fd_handler(cl->peer_fd, EPOLLIN, [this](int peer_fd, int epoll_events)
                {
                    // Do not miss the disconnection!
                    if (epoll_events & EPOLLRDHUP)
@@ -785,7 +691,6 @@ void osd_messenger_t::accept_connections(int listen_fd)
        int one = 1;
        setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
        clients[peer_fd] = new osd_client_t();
-        clients[peer_fd]->msgr = this;
        clients[peer_fd]->peer_addr = addr;
        clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
        clients[peer_fd]->peer_fd = peer_fd;
@@ -793,11 +698,7 @@ void osd_messenger_t::accept_connections(int listen_fd)
        clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
        clients[peer_fd]->receive_buffer_size = receive_buffer_size;
        // Add FD to epoll
-        if (iothreads.size())
-        {
-            iothreads[peer_fd % iothreads.size()]->add_client(clients[peer_fd]);
-        }
-        tfd->set_fd_handler(peer_fd, iothreads.size() ? 0 : EPOLLIN, [this](int peer_fd, int epoll_events)
+        tfd->set_fd_handler(peer_fd, EPOLLIN, [this](int peer_fd, int epoll_events)
        {
            handle_peer_epoll(peer_fd, epoll_events);
        });
--- a/src/client/messenger.h
+++ b/src/client/messenger.h
@@ -46,12 +46,9 @@ struct msgr_rdma_connection_t;
 struct msgr_rdma_context_t;
 #endif

-struct osd_messenger_t;
-
 struct osd_client_t
 {
    std::mutex mu;
-    osd_messenger_t *msgr = NULL;

    int refs = 0;

@@ -98,10 +95,10 @@ struct osd_client_t
    ~osd_client_t();

    bool try_send(ring_loop_t *ringloop, bool use_sync_send_recv);
-    int handle_send(int result);
+    void handle_send(int result);

    bool try_recv(ring_loop_t *ringloop, bool use_sync_send_recv);
-    int handle_read(int result);
+    bool handle_read(int result);
    bool handle_read_buffer(void *curbuf, int remain);
    bool handle_finished_read();
    void handle_op_hdr();
@@ -142,35 +139,35 @@ class msgr_iothread_t
 protected:
    ring_loop_t *ring = NULL;
    epoll_manager_t *epmgr = NULL;
-    ring_consumer_t consumer;
    int submit_eventfd = -1;
    bool stopped = false;
+    bool pending = false;
+    bool *callback_stopped = NULL;
    std::mutex mu;
-    std::map<int, osd_client_t*> clients;
-    std::vector<int> read_ready_clients;
-    std::mutex write_ready_mu;
-    std::vector<int> write_ready_clients;
+    std::vector<osd_client_t*> pending_clients;
    std::thread *thread = NULL;

    void run();
-
-    void read_requests();
-
-    void send_replies();
-
 public:

-    void handle_client_read(osd_client_t *cl, int res);
-    void handle_client_send(osd_client_t *cl, int res);
-
    msgr_iothread_t();
    ~msgr_iothread_t();

-    void add_client(osd_client_t *cl);
+    epoll_manager_t *get_epmgr()
+    {
+        return epmgr;
+    }

-    void remove_client(osd_client_t *cl);
+    int get_eventfd()
+    {
+        return submit_eventfd;
+    }

-    void wakeup_out(int peer_fd, ring_loop_t *outer_ring);
+    void wakeup(osd_client_t *cl, ring_loop_t *outer_ring);
+
+    void add_pending(osd_client_t *cl);
+
+    void submit_pending_writes();

    void stop();
 };
@@ -188,7 +185,7 @@ protected:
    int osd_ping_timeout = 0;
    int log_level = 0;
    bool use_sync_send_recv = false;
-    int iothread_count = 0;
+    int iothread_count = 4;

 #ifdef WITH_RDMA
    bool use_rdma = true;
@@ -203,10 +200,9 @@ protected:
    std::vector<msgr_iothread_t*> iothreads;
    std::vector<int> read_ready_clients;
    std::vector<int> write_ready_clients;
-    int immediates_eventfd = -1;
-    std::mutex immediates_mu;
-    // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::vector<std::function<void()>> immediates;
+    int completion_eventfd = -1;
+    std::mutex completions_mu;
+    std::vector<osd_op_t*> completions;

 public:
    timerfd_manager_t *tfd;
@@ -224,13 +220,10 @@ public:
    void parse_config(const json11::Json & config);
    void connect_peer(uint64_t osd_num, json11::Json peer_state);
    void stop_client(int peer_fd, bool force = false, bool force_delete = false);
-    void stop_client_from_iothread(osd_client_t *cl);
    void outbox_push(osd_op_t *cur_op);
    std::function<void(osd_op_t*)> exec_op;
    std::function<void(osd_num_t)> repeer_pgs;
    std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
-    void handle_client_read(osd_client_t *cl, int res);
-    void handle_client_send(osd_client_t *cl, int res);
    void read_requests();
    void send_replies();
    void accept_connections(int listen_fd);
@@ -250,9 +243,6 @@ public:
    void inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len);
    void measure_exec(osd_op_t *cur_op);

-    void set_immediate(std::function<void()> cb);
-    void set_immediate_or_run(std::function<void()> cb);
-
 protected:
    void try_connect_peer(uint64_t osd_num);
    void try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port);
@@ -263,7 +253,8 @@ protected:
    void cancel_osd_ops(osd_client_t *cl);
    void cancel_op(osd_op_t *op);

-    void handle_immediates();
+    void add_completion(osd_op_t *op, ring_loop_t *ringloop = NULL);
+    void handle_completions();

 #ifdef WITH_RDMA
    void try_send_rdma(osd_client_t *cl);
@@ -272,6 +263,4 @@ protected:
    bool try_recv_rdma(osd_client_t *cl);
    void handle_rdma_events();
 #endif
-
-    friend struct osd_client_t;
 };
--- a/src/client/msgr_rdma.cpp
+++ b/src/client/msgr_rdma.cpp
@@ -363,6 +363,8 @@ bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64
                auto cl = clients.at(peer_fd);
                cl->rdma_conn = rdma_conn;
                cl->peer_state = PEER_RDMA_CONNECTING;
+                // Add the initial receive request
+                try_recv_rdma(cl);
                return true;
            }
        }
@@ -666,5 +668,5 @@ void osd_messenger_t::handle_rdma_events()
            }
        }
    } while (event_count > 0);
-    handle_immediates();
+    handle_received_ops();
 }
--- a/src/client/msgr_receive.cpp
+++ b/src/client/msgr_receive.cpp
@@ -1,7 +1,6 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)

-#include <unistd.h>
 #include "messenger.h"

 void osd_messenger_t::read_requests()
@@ -17,10 +16,6 @@ void osd_messenger_t::read_requests()
        }
    }
    read_ready_clients.clear();
-    if (!iothreads.size())
-    {
-        handle_immediates();
-    }
 }

 bool osd_client_t::try_recv(ring_loop_t *ringloop, bool use_sync_send_recv)
@@ -54,14 +49,7 @@ bool osd_client_t::try_recv(ring_loop_t *ringloop, bool use_sync_send_recv)
            return false;
        }
        ring_data_t* data = ((ring_data_t*)sqe->user_data);
-        if (msgr->iothreads.size())
-        {
-            data->callback = [this](ring_data_t *data) { msgr->iothreads[peer_fd % msgr->iothreads.size()]->handle_client_read(this, data->res); };
-        }
-        else
-        {
-            data->callback = [this](ring_data_t *data) { msgr->handle_client_read(this, data->res); };
-        }
+        data->callback = [this](ring_data_t *data) { handle_read(data->res); };
        my_uring_prep_recvmsg(sqe, peer_fd, &cl->read_msg, 0);
    }
    else
@@ -71,58 +59,24 @@ bool osd_client_t::try_recv(ring_loop_t *ringloop, bool use_sync_send_recv)
        {
            result = -errno;
        }
-        msgr->handle_client_read(this, result);
+        handle_read(result);
    }
    return true;
 }

-void osd_messenger_t::handle_client_read(osd_client_t *cl, int res)
-{
-    res = cl->handle_read(res);
-    if (res == -ENOENT)
-    {
-        if (!cl->refs)
-            delete cl;
-    }
-    else if (res == -EIO)
-    {
-        stop_client(cl->peer_fd);
-    }
-    else if (res == -EAGAIN)
-    {
-        read_ready_clients.push_back(cl->peer_fd);
-    }
-}
-
-void msgr_iothread_t::handle_client_read(osd_client_t *cl, int res)
-{
-    cl->mu.lock();
-    res = cl->handle_read(res);
-    if (res == -ENOENT)
-    {
-        if (!cl->refs)
-            cl->msgr->set_immediate([cl]() { delete cl; });
-    }
-    cl->mu.unlock();
-    if (res == -EIO)
-    {
-        cl->msgr->stop_client_from_iothread(cl);
-    }
-    else if (res == -EAGAIN)
-    {
-        read_ready_clients.push_back(cl->peer_fd);
-        ring->wakeup();
-    }
-}
-
-int osd_client_t::handle_read(int result)
+bool osd_client_t::handle_read(int result)
 {
    auto cl = this;
+    bool ret = false;
    cl->read_msg.msg_iovlen = 0;
    cl->refs--;
    if (cl->peer_state == PEER_STOPPED)
    {
-        return -ENOENT;
+        if (cl->refs <= 0)
+        {
+            delete cl;
+        }
+        return false;
    }
    if (result <= 0 && result != -EAGAIN && result != -EINTR)
    {
@@ -131,14 +85,27 @@ int osd_client_t::handle_read(int result)
        {
            fprintf(stderr, "Client %d socket read error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
        }
-        return -EIO;
+        close(cl->peer_fd);
+        return false;
+    }
+    if (result == -EAGAIN || result == -EINTR || result < cl->read_iov.iov_len)
+    {
+        cl->read_ready--;
+        if (cl->read_ready > 0)
+            read_ready_clients.push_back(cl->peer_fd);
+    }
+    else
+    {
+        read_ready_clients.push_back(cl->peer_fd);
    }
-    int expected = cl->read_iov.iov_len;
    if (result > 0)
    {
        if (cl->read_iov.iov_base == cl->in_buf)
        {
-            handle_read_buffer(cl->in_buf, result);
+            if (!handle_read_buffer(cl->in_buf, result))
+            {
+                goto fin;
+            }
        }
        else
        {
@@ -147,20 +114,20 @@ int osd_client_t::handle_read(int result)
            cl->recv_list.eat(result);
            if (cl->recv_list.done >= cl->recv_list.count)
            {
-                handle_finished_read();
+                if (!handle_finished_read())
+                {
+                    goto fin;
+                }
            }
        }
+        if (result >= cl->read_iov.iov_len)
+        {
+            ret = true;
+        }
    }
-    if (result == -EAGAIN || result == -EINTR || result < expected)
-    {
-        cl->read_ready--;
-        assert(cl->read_ready >= 0);
-    }
-    if (cl->read_ready > 0)
-    {
-        return -EAGAIN;
-    }
-    return 0;
+fin:
+    msgr->handle_completions();
+    return ret;
 }

 bool osd_client_t::handle_read_buffer(void *curbuf, int remain)
@@ -223,7 +190,7 @@ bool osd_client_t::handle_finished_read()
        else
        {
            fprintf(stderr, "Received garbage: magic=%jx id=%ju opcode=%jx from %d\n", cl->read_op->req.hdr.magic, cl->read_op->req.hdr.id, cl->read_op->req.hdr.opcode, cl->peer_fd);
-            msgr->stop_client_from_iothread(cl);
+            close(cl->peer_fd);
            return false;
        }
    }
@@ -231,14 +198,14 @@ bool osd_client_t::handle_finished_read()
    {
        // Operation is ready
        cl->received_ops.push_back(cl->read_op);
-        msgr->set_immediate([msgr = this->msgr, op = cl->read_op, cl]() { msgr->exec_op(op); });
+        msgr->add_completion(cl->read_op);
        cl->read_op = NULL;
        cl->read_state = 0;
    }
    else if (cl->read_state == CL_READ_REPLY_DATA)
    {
        // Reply is ready
-        handle_reply_ready(cl->read_op);
+        msgr->add_completion(cl->read_op);
        cl->read_op = NULL;
        cl->read_state = 0;
    }
@@ -328,7 +295,7 @@ void osd_client_t::handle_op_hdr()
    {
        // Operation is ready
        cl->received_ops.push_back(cur_op);
-        msgr->set_immediate([msgr = this->msgr, cur_op, cl]() { msgr->exec_op(cur_op); });
+        msgr->add_completion(cur_op);
        cl->read_op = NULL;
        cl->read_state = 0;
    }
@@ -342,7 +309,7 @@ bool osd_client_t::handle_reply_hdr()
    {
        // Command out of sync. Drop connection
        fprintf(stderr, "Client %d command out of sync: id %ju\n", cl->peer_fd, cl->read_op->req.hdr.id);
-        msgr->stop_client_from_iothread(cl);
+        close(cl->peer_fd);
        return false;
    }
    osd_op_t *op = req_it->second;
@@ -359,7 +326,7 @@ bool osd_client_t::handle_reply_hdr()
            fprintf(stderr, "Client %d read reply of different length: expected %u+%u, got %jd+%u\n",
                cl->peer_fd, expected_size, op->bitmap_len, op->reply.hdr.retval, bmp_len);
            cl->sent_ops[op->req.hdr.id] = op;
-            msgr->stop_client_from_iothread(cl);
+            close(cl->peer_fd);
            return false;
        }
        if (bmp_len > 0)
@@ -427,7 +394,7 @@ bool osd_client_t::handle_reply_hdr()
    {
 reuse:
        // It's fine to reuse cl->read_op for the next reply
-        handle_reply_ready(op);
+        msgr->add_completion(op);
        cl->recv_list.push_back(cl->read_op->req.buf, OSD_PACKET_SIZE);
        cl->read_remaining = OSD_PACKET_SIZE;
        cl->read_state = CL_READ_HDR;
@@ -435,92 +402,65 @@ reuse:
    return true;
 }

-void osd_client_t::handle_reply_ready(osd_op_t *op)
-{
-    msgr->set_immediate([msgr = this->msgr, op, cl = this]()
-    {
-        // Measure subop latency
-        auto & stats = msgr->stats;
-        timespec tv_end;
-        clock_gettime(CLOCK_REALTIME, &tv_end);
-        stats.subop_stat_count[op->req.hdr.opcode]++;
-        if (!stats.subop_stat_count[op->req.hdr.opcode])
-        {
-            stats.subop_stat_count[op->req.hdr.opcode]++;
-            stats.subop_stat_sum[op->req.hdr.opcode] = 0;
-        }
-        stats.subop_stat_sum[op->req.hdr.opcode] += (
-            (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
-            (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
-        );
-        // Copy lambda to be unaffected by `delete op`
-        std::function<void(osd_op_t*)>(op->callback)(op);
-    });
-}
-
 static uint64_t one = 1;

-void osd_messenger_t::set_immediate(std::function<void()> cb/*, ring_loop_t *ringloop*/)
+void osd_messenger_t::add_completion(osd_op_t *op, ring_loop_t *ringloop)
 {
-    if (!iothreads.size())
-    {
-        immediates.push_back(cb);
-        return;
-    }
-    immediates_mu.lock();
-    bool wakeup_main_thread = !immediates.size();
-    immediates.push_back(cb);
-    immediates_mu.unlock();
+    completions_mu.lock();
+    bool wakeup_main_thread = !completions.size();
+    completions.push_back(op);
+    completions_mu.unlock();
    if (wakeup_main_thread)
    {
-//        io_uring_sqe* sqe = ringloop ? ringloop->get_sqe() : NULL;
-//        if (!sqe)
-//        {
-            write(immediates_eventfd, &one, sizeof(one));
-// FIXME: Can't use ringloop here, oops
-//        }
-//        else
-//        {
-//            ring_data_t* data = ((ring_data_t*)sqe->user_data);
-//            data->callback = [](ring_data_t*){};
-//            my_uring_prep_write(sqe, immediates_eventfd, &one, sizeof(one), 0);
-//        }
+        io_uring_sqe* sqe = ringloop ? ringloop->get_sqe() : NULL;
+        if (!sqe)
+        {
+            write(completion_eventfd, &one, sizeof(one));
+        }
+        else
+        {
+            ring_data_t* data = ((ring_data_t*)sqe->user_data);
+            data->callback = [](ring_data_t*){};
+            my_uring_prep_write(sqe, completion_eventfd, &one, sizeof(one), 0);
+        }
    }
 }

-void osd_messenger_t::set_immediate_or_run(std::function<void()> cb/*, ring_loop_t *ringloop*/)
+void osd_messenger_t::handle_completions()
 {
-    if (!iothreads.size())
+    // Reset eventfd counter
+    uint64_t ctr = 0;
+    int r = read(completion_eventfd, &ctr, 8);
+    if (r < 0 && errno != EAGAIN && errno != EINTR)
    {
-        cb();
-        return;
+        fprintf(stderr, "Error resetting eventfd: %s\n", strerror(errno));
    }
-    immediates_mu.lock();
-    bool wakeup_main_thread = !immediates.size();
-    immediates.push_back(cb);
-    immediates_mu.unlock();
-    if (wakeup_main_thread)
+    completions_mu.lock();
+    auto ops = std::move(completions);
+    completions_mu.unlock();
+    for (auto op: completions)
    {
-//        io_uring_sqe* sqe = ringloop ? ringloop->get_sqe() : NULL;
-//        if (!sqe)
-//        {
-            write(immediates_eventfd, &one, sizeof(one));
-// FIXME: Can't use ringloop here, oops
-//        }
-//        else
-//        {
-//            ring_data_t* data = ((ring_data_t*)sqe->user_data);
-//            data->callback = [](ring_data_t*){};
-//            my_uring_prep_write(sqe, immediates_eventfd, &one, sizeof(one), 0);
-//        }
-    }
-}
-
-void osd_messenger_t::handle_immediates()
-{
-    auto to_run = std::move(immediates);
-    for (auto & cb: to_run)
-    {
-        cb();
+        if (op->op_type == OSD_OP_IN)
+        {
+            exec_op(op);
+        }
+        else
+        {
+            // Measure subop latency
+            timespec tv_end;
+            clock_gettime(CLOCK_REALTIME, &tv_end);
+            stats.subop_stat_count[op->req.hdr.opcode]++;
+            if (!stats.subop_stat_count[op->req.hdr.opcode])
+            {
+                stats.subop_stat_count[op->req.hdr.opcode]++;
+                stats.subop_stat_sum[op->req.hdr.opcode] = 0;
+            }
+            stats.subop_stat_sum[op->req.hdr.opcode] += (
+                (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
+                (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
+            );
+            // Copy lambda to be unaffected by `delete op`
+            std::function<void(osd_op_t*)>(op->callback)(op);
+        }
    }
 }
--- a/src/client/msgr_send.cpp
+++ b/src/client/msgr_send.cpp
@@ -19,33 +19,27 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
    {
        measure_exec(cur_op);
    }
+    std::unique_lock<std::mutex> lock;
    if (iothreads.size())
    {
-        cl->mu.lock();
+        lock = std::unique_lock<std::mutex>(cl->mu);
    }
-    if (cur_op->op_type == OSD_OP_IN)
+    // Check that operation actually belongs to this client
+    bool found = false;
+    for (auto it = cl->received_ops.begin(); it != cl->received_ops.end(); it++)
    {
-        // Check that operation actually belongs to this client
-        bool found = false;
-        for (auto it = cl->received_ops.begin(); it != cl->received_ops.end(); it++)
+        if (*it == cur_op)
        {
-            if (*it == cur_op)
-            {
-                found = true;
-                cl->received_ops.erase(it, it+1);
-                break;
-            }
-        }
-        if (!found)
-        {
-            delete cur_op;
-            if (iothreads.size())
-            {
-                cl->mu.unlock();
-            }
-            return;
+            found = true;
+            cl->received_ops.erase(it, it+1);
+            break;
        }
    }
+    if (!found)
+    {
+        delete cur_op;
+        return;
+    }
    auto & to_send_list = cl->write_msg.msg_iovlen ? cl->next_send_list : cl->send_list;
    auto & to_outbox = cl->write_msg.msg_iovlen ? cl->next_outbox : cl->outbox;
    if (cur_op->op_type == OSD_OP_IN)
@@ -118,26 +112,11 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
 #ifdef WITH_RDMA
    if (cl->peer_state == PEER_RDMA)
    {
-        if (iothreads.size())
-        {
-            cl->mu.unlock();
-        }
        try_send_rdma(cl);
        return;
    }
 #endif
-    if (iothreads.size())
-    {
-        int should_wakeup = !cl->write_msg.msg_iovlen && !cl->write_state;
-        cl->write_state = CL_WRITE_READY;
-        cl->mu.unlock();
-        if (should_wakeup)
-        {
-            auto iot = iothreads[cl->peer_fd % iothreads.size()];
-            iot->wakeup_out(cl->peer_fd, ringloop);
-        }
-    }
-    else if (!ringloop)
+    if (!ringloop)
    {
        // FIXME: It's worse because it doesn't allow batching
        while (cl->outbox.size())
@@ -145,6 +124,12 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
            cl->try_send(NULL, true);
        }
    }
+    else if (iothreads.size())
+    {
+        auto iot = iothreads[cl->peer_fd % iothreads.size()];
+        cl->refs++;
+        iot->wakeup(cl, ringloop);
+    }
    else
    {
        if ((cl->write_msg.msg_iovlen > 0 || !cl->try_send(ringloop, use_sync_send_recv)) && (cl->write_state == 0))
@@ -224,14 +209,7 @@ bool osd_client_t::try_send(ring_loop_t *ringloop, bool use_sync_send_recv)
        cl->write_msg.msg_iovlen = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
        cl->refs++;
        ring_data_t* data = ((ring_data_t*)sqe->user_data);
-        if (msgr->iothreads.size())
-        {
-            data->callback = [this](ring_data_t *data) { msgr->iothreads[this->peer_fd % msgr->iothreads.size()]->handle_client_send(this, data->res); };
-        }
-        else
-        {
-            data->callback = [this](ring_data_t *data) { msgr->handle_client_send(this, data->res); };
-        }
+        data->callback = [this](ring_data_t *data) { handle_send(data->res); };
        my_uring_prep_sendmsg(sqe, peer_fd, &cl->write_msg, 0);
    }
    else
@@ -244,52 +222,11 @@ bool osd_client_t::try_send(ring_loop_t *ringloop, bool use_sync_send_recv)
        {
            result = -errno;
        }
-        msgr->handle_client_send(this, result);
+        handle_send(result);
    }
    return true;
 }

-void osd_messenger_t::handle_client_send(osd_client_t *cl, int res)
-{
-    res = cl->handle_send(res);
-    if (res == -ENOENT)
-    {
-        if (!cl->refs)
-            delete cl;
-    }
-    else if (res == -EIO)
-    {
-        stop_client(cl->peer_fd);
-    }
-    else if (res == -EAGAIN)
-    {
-        write_ready_clients.push_back(cl->peer_fd);
-    }
-}
-
-void msgr_iothread_t::handle_client_send(osd_client_t *cl, int res)
-{
-    cl->mu.lock();
-    res = cl->handle_send(res);
-    if (res == -ENOENT)
-    {
-        if (!cl->refs)
-            cl->msgr->set_immediate([cl]() { delete cl; });
-    }
-    cl->mu.unlock();
-    if (res == -EIO)
-    {
-        cl->msgr->stop_client_from_iothread(cl);
-    }
-    else if (res == -EAGAIN)
-    {
-        write_ready_mu.lock();
-        write_ready_clients.push_back(cl->peer_fd);
-        write_ready_mu.unlock();
-        ring->wakeup();
-    }
-}
-
 void osd_messenger_t::send_replies()
 {
    if (iothreads.size())
@@ -314,20 +251,25 @@ void osd_messenger_t::send_replies()
    write_ready_clients.clear();
 }

-int osd_client_t::handle_send(int result)
+void osd_client_t::handle_send(int result)
 {
    auto cl = this;
    cl->write_msg.msg_iovlen = 0;
    cl->refs--;
    if (cl->peer_state == PEER_STOPPED)
    {
-        return -ENOENT;
+        if (cl->refs <= 0)
+        {
+            delete cl;
+        }
+        return;
    }
    if (result < 0 && result != -EAGAIN && result != -EINTR)
    {
        // this is a client socket, so don't panic. just disconnect it
        fprintf(stderr, "Client %d socket write error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
-        return -EIO;
+        close(cl->peer_fd);
+        return;
    }
    if (result >= 0)
    {
@@ -340,7 +282,7 @@ int osd_client_t::handle_send(int result)
                if (cl->outbox[done].flags & MSGR_SENDP_FREE)
                {
                    // Reply fully sent
-                    msgr->set_immediate_or_run([op = cl->outbox[done].op] { delete op; });
+                    delete cl->outbox[done].op;
                }
                result -= iov.iov_len;
                done++;
@@ -368,37 +310,16 @@ int osd_client_t::handle_send(int result)
 #ifdef WITH_RDMA
        if (cl->rdma_conn && !cl->outbox.size() && cl->peer_state == PEER_RDMA_CONNECTING)
        {
-            // FIXME: Do something better than just forgetting the FD
-            // FIXME: Ignore pings during RDMA state transition
-            cl->peer_state = PEER_RDMA;
-            msgr->set_immediate_or_run([cl = this, msgr = this->msgr, peer_fd = this->peer_fd]()
+            if (log_level > 0)
            {
-                auto cl_it = msgr->clients.find(peer_fd);
-                if (cl_it == msgr->clients.end() || cl_it->second != cl)
-                {
-                    return;
-                }
-                if (msgr->log_level > 0)
-                {
-                    fprintf(stderr, "Successfully connected with client %d using RDMA\n", peer_fd);
-                }
-                msgr->tfd->set_fd_handler(peer_fd, 0, [msgr](int peer_fd, int epoll_events)
-                {
-                    // Do not miss the disconnection!
-                    if (epoll_events & EPOLLRDHUP)
-                    {
-                        msgr->handle_peer_epoll(peer_fd, epoll_events);
-                    }
-                });
-                // Add the initial receive request
-                msgr->try_recv_rdma(cl);
-            });
+                fprintf(stderr, "Successfully connected with client %d using RDMA\n", cl->peer_fd);
+            }
+            cl->peer_state = PEER_RDMA;
        }
 #endif
    }
    if (cl->write_state != 0)
    {
-        return -EAGAIN;
+        write_ready_clients.push_back(cl->peer_fd);
    }
-    return 0;
 }
--- a/src/client/msgr_stop.cpp
+++ b/src/client/msgr_stop.cpp
@@ -11,7 +11,6 @@

 void osd_messenger_t::cancel_osd_ops(osd_client_t *cl)
 {
-    cl->mu.lock();
    std::vector<osd_op_t*> cancel_ops;
    cancel_ops.resize(cl->sent_ops.size());
    int i = 0;
@@ -21,7 +20,6 @@ void osd_messenger_t::cancel_osd_ops(osd_client_t *cl)
    }
    cl->sent_ops.clear();
    cl->outbox.clear();
-    cl->mu.unlock();
    for (auto op: cancel_ops)
    {
        cancel_op(op);
@@ -55,10 +53,8 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
        return;
    }
    osd_client_t *cl = it->second;
-    cl->mu.lock();
    if (cl->peer_state == PEER_CONNECTING && !force || cl->peer_state == PEER_STOPPED)
    {
-        cl->mu.unlock();
        return;
    }
    if (log_level > 0)
@@ -75,7 +71,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
    // First set state to STOPPED so another stop_client() call doesn't try to free it again
    cl->refs++;
    cl->peer_state = PEER_STOPPED;
-    cl->mu.unlock();
    if (cl->osd_num)
    {
        // ...and forget OSD peer
@@ -84,10 +79,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
 #ifndef __MOCK__
    // Then remove FD from the eventloop so we don't accidentally read something
    tfd->set_fd_handler(peer_fd, 0, NULL);
-    if (iothreads.size())
-    {
-        iothreads[peer_fd % iothreads.size()]->remove_client(cl);
-    }
    if (cl->connect_timeout_id >= 0)
    {
        tfd->clear_timer(cl->connect_timeout_id);
@@ -117,24 +108,17 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
        repeer_pgs(cl->osd_num);
    }
    // Then cancel all operations
-    cl->mu.lock();
    if (cl->read_op)
    {
-        auto op = cl->read_op;
-        cl->read_op = NULL;
-        cl->mu.unlock();
-        if (!op->callback)
+        if (!cl->read_op->callback)
        {
-            delete op;
+            delete cl->read_op;
        }
        else
        {
-            cancel_op(op);
+            cancel_op(cl->read_op);
        }
-    }
-    else
-    {
-        cl->mu.unlock();
+        cl->read_op = NULL;
    }
    if (cl->osd_num)
    {
@@ -147,32 +131,11 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
    {
        clients.erase(it);
    }
-    cl->mu.lock();
    cl->refs--;
    if (cl->refs <= 0 || force_delete)
    {
-        cl->mu.unlock();
        delete cl;
    }
-    else
-        cl->mu.unlock();
-}
-
-void osd_messenger_t::stop_client_from_iothread(osd_client_t *cl)
-{
-    if (!iothreads.size())
-    {
-        stop_client(cl->peer_fd);
-        return;
-    }
-    set_immediate([this, cl, peer_fd = cl->peer_fd]()
-    {
-        auto cl_it = clients.find(peer_fd);
-        if (cl_it != clients.end() && cl_it->second == cl)
-        {
-            stop_client(peer_fd);
-        }
-    });
 }

 osd_client_t::~osd_client_t()
--- a/src/cmd/cli_status.cpp
+++ b/src/cmd/cli_status.cpp
@@ -133,7 +133,7 @@ resume_2:
        }
        int osd_count = 0, osd_up = 0;
        uint64_t total_raw = 0, free_raw = 0, free_down_raw = 0, down_raw = 0;
-        parent->iterate_kvs_1(osd_stats, "/osd/stats/", [&](uint64_t stat_osd_num, json11::Json value)
+        parent->iterate_kvs_1(osd_stats, "/osd/stats", [&](uint64_t stat_osd_num, json11::Json value)
        {
            osd_count++;
            auto osd_size = value["size"].uint64_value();
--- a/src/nfs/nfs_proxy.cpp
+++ b/src/nfs/nfs_proxy.cpp
@@ -466,7 +466,7 @@ void nfs_proxy_t::do_accept(int listen_fd)
        {
            cli->proc_table.insert(fn);
        }
-        epmgr->tfd->set_fd_handler(nfs_fd, EPOLLIN|EPOLLOUT, [cli](int nfs_fd, int epoll_events)
+        epmgr->tfd->set_fd_handler(nfs_fd, EPOLLOUT, [cli](int nfs_fd, int epoll_events)
        {
            // Handle incoming event
            if (epoll_events & EPOLLRDHUP)
--- a/src/osd/osd.cpp
+++ b/src/osd/osd.cpp
@@ -141,14 +141,6 @@ void osd_t::parse_config(bool init)
    config = msgr.merge_configs(cli_config, file_config, etcd_global_config, etcd_osd_config);
    if (config.find("log_level") == this->config.end())
        config["log_level"] = 1;
-    if (init)
-    {
-        // OSD number
-        osd_num = config["osd_num"].uint64_value();
-        if (!osd_num)
-            throw std::runtime_error("osd_num is required in the configuration");
-        msgr.osd_num = osd_num;
-    }
    if (bs)
    {
        auto bs_cfg = json_to_bs(config);
@@ -158,6 +150,11 @@ void osd_t::parse_config(bool init)
    msgr.parse_config(config);
    if (init)
    {
+        // OSD number
+        osd_num = config["osd_num"].uint64_value();
+        if (!osd_num)
+            throw std::runtime_error("osd_num is required in the configuration");
+        msgr.osd_num = osd_num;
        // Vital Blockstore parameters
        bs_block_size = config["block_size"].uint64_value();
        if (!bs_block_size)
--- a/src/test/stub_uring_osd.cpp
+++ b/src/test/stub_uring_osd.cpp
@@ -43,7 +43,7 @@ int main(int narg, char *args[])
    // Accept new connections
    int listen_fd = create_and_bind_socket("0.0.0.0", 11203, 128, NULL);
    fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
-    epmgr->set_fd_handler(listen_fd, EPOLLIN, [listen_fd, msgr](int fd, int events)
+    epmgr->set_fd_handler(listen_fd, false, [listen_fd, msgr](int fd, int events)
    {
        msgr->accept_connections(listen_fd);
    });
--- a/src/util/epoll_manager.cpp
+++ b/src/util/epoll_manager.cpp
@@ -21,7 +21,7 @@ epoll_manager_t::epoll_manager_t(ring_loop_t *ringloop)
        throw std::runtime_error(std::string("epoll_create: ") + strerror(errno));
    }

-    tfd = new timerfd_manager_t([this](int fd, int events, std::function<void(int, int)> handler) { set_fd_handler(fd, events, handler); });
+    tfd = new timerfd_manager_t([this](int fd, bool wr, std::function<void(int, int)> handler) { set_fd_handler(fd, wr, handler); });

    if (ringloop)
    {
Author	SHA1	Message	Date
Vitaliy Filippov	1d6f1203f8	WIP another experiment - "smart" iothreads All checks were successful Test / test_splitbrain (push) Has been skipped Details Test / test_rebalance_verify (push) Has been skipped Details Test / test_rebalance_verify_imm (push) Has been skipped Details Test / test_rebalance_verify_ec (push) Has been skipped Details Test / test_rebalance_verify_ec_imm (push) Has been skipped Details Test / test_root_node (push) Has been skipped Details Test / test_switch_primary (push) Has been skipped Details Test / test_write (push) Has been skipped Details Test / test_write_xor (push) Has been skipped Details Test / test_write_no_same (push) Has been skipped Details Test / test_heal_pg_size_2 (push) Has been skipped Details Test / test_heal_ec (push) Has been skipped Details Test / test_heal_csum_32k_dmj (push) Has been skipped Details Test / test_heal_csum_32k_dj (push) Has been skipped Details Test / test_heal_csum_32k (push) Has been skipped Details Test / test_heal_csum_4k_dmj (push) Has been skipped Details Test / test_heal_csum_4k_dj (push) Has been skipped Details Test / test_heal_csum_4k (push) Has been skipped Details Test / test_osd_tags (push) Has been skipped Details Test / test_enospc (push) Has been skipped Details Test / test_enospc_xor (push) Has been skipped Details Test / test_enospc_imm (push) Has been skipped Details Test / test_enospc_imm_xor (push) Has been skipped Details Test / test_scrub (push) Has been skipped Details Test / test_scrub_zero_osd_2 (push) Has been skipped Details Test / test_scrub_xor (push) Has been skipped Details Test / test_scrub_pg_size_3 (push) Has been skipped Details Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Has been skipped Details Test / test_scrub_ec (push) Has been skipped Details Test / test_nfs (push) Has been skipped Details	2024-06-27 01:15:09 +03:00
Vitaliy Filippov	d4dc3f4a9d	Move RDMA handling	2024-06-24 10:43:12 +03:00
Vitaliy Filippov	0c0439b33b	Change bool wr to event mask in epoll_manager	2024-06-24 00:54:44 +03:00