trace writes

.gitea/workflows/test.yml

@@ -144,24 +144,6 @@ jobs:
           echo ""
         done
 
-  test_change_pg_count_online:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_change_pg_count_online.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_change_pg_size:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.2.3")
+set(VITASTOR_VERSION "2.2.2")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.2.3
+VITASTOR_VERSION ?= v2.2.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.2.3
+          image: vitalif/vitastor-csi:v2.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.2.3
+          image: vitalif/vitastor-csi:v2.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.2.3"
+    vitastorCSIDriverVersion = "2.2.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-vitastor-bookworm.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
-docker build --build-arg DISTRO=debian --build-arg REL=bookworm -t vitastor-buildenv:bookworm -f vitastor-buildenv.Dockerfile .
+cat < vitastor.Dockerfile > ../Dockerfile
-docker run -i --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh
+cd ..
+mkdir -p packages
+sudo podman build --build-arg DISTRO=debian --build-arg REL=bookworm -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-bullseye.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
-docker build --build-arg DISTRO=debian --build-arg REL=bullseye -t vitastor-buildenv:bullseye -f vitastor-buildenv.Dockerfile .
+cat < vitastor.Dockerfile > ../Dockerfile
-docker run -i --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh
+cd ..
+mkdir -p packages
+sudo podman build --build-arg DISTRO=debian --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-buster.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
-docker build --build-arg DISTRO=debian --build-arg REL=buster -t vitastor-buildenv:buster -f vitastor-buildenv.Dockerfile .
+cat < vitastor.Dockerfile > ../Dockerfile
-docker run -i --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh
+cd ..
+mkdir -p packages
+sudo podman build --build-arg DISTRO=debian --build-arg REL=buster -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-ubuntu-jammy.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
-# Ubuntu 22.04 Jammy Jellyfish
 
-docker build --build-arg DISTRO=ubuntu --build-arg REL=jammy -t vitastor-buildenv:jammy -f vitastor-buildenv.Dockerfile .
+cat < vitastor.Dockerfile > ../Dockerfile
-docker run -i --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh
+cd ..
+mkdir -p packages
+sudo podman build --build-arg DISTRO=ubuntu --build-arg REL=jammy -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-ubuntu-noble.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-# 24.04 Noble Numbat
-
-docker build --build-arg DISTRO=ubuntu --build-arg REL=noble -t vitastor-buildenv:noble -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.2.3-1) unstable; urgency=medium
+vitastor (2.2.2-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor-build.sh

@@ -1,59 +0,0 @@
-#!/bin/bash
-# To be ran inside buildenv docker
-
-set -e -x
-
-[ -e /usr/lib/x86_64-linux-gnu/pkgconfig/libisal.pc ] || cp /root/vitastor/debian/libisal.pc /usr/lib/x86_64-linux-gnu/pkgconfig
-
-mkdir -p /root/fio-build/
-cd /root/fio-build/
-rm -rf /root/fio-build/*
-dpkg-source -x /root/fio*.dsc
-
-FULLVER=`head -n1 /root/vitastor/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'`
-VER=${FULLVER%%-*}
-rm -rf /root/vitastor-$VER
-mkdir /root/vitastor-$VER
-cd /root/vitastor
-cp -a $(ls | grep -v packages) /root/vitastor-$VER
-
-rm -rf /root/vitastor/packages/vitastor-$REL
-mkdir -p /root/vitastor/packages/vitastor-$REL
-mv /root/vitastor-$VER /root/vitastor/packages/vitastor-$REL/
-
-cd /root/vitastor/packages/vitastor-$REL/vitastor-$VER
-
-rm -rf fio
-ln -s /root/fio-build/fio-*/ ./fio
-FIO=`head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'`
-ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h
-sh copy-fio-includes.sh
-rm fio
-mkdir -p a b debian/patches
-mv fio-copy b/fio
-diff -NaurpbB a b > debian/patches/fio-headers.patch || true
-echo fio-headers.patch >> debian/patches/series
-rm -rf a b
-
-echo "dep:fio=$FIO" > debian/fio_version
-
-cd /root/vitastor/packages/vitastor-$REL/vitastor-$VER
-mkdir mon/node_modules
-cd mon/node_modules
-curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
-curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx
-
-cd /root/vitastor/packages/vitastor-$REL
-tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
-cd vitastor-$VER
-DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$FULLVER""$REL" "Rebuild for $REL"
-DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa
-rm -rf /root/vitastor/packages/vitastor-$REL/vitastor-*/
-
-# Why does ubuntu rename debug packages to *.ddeb?
-if ls *.ddeb >/dev/null; then
-    perl -i -pe 's/\.ddeb/.deb/' *.buildinfo *.changes
-    for i in *.ddeb; do
-        mv $i ${i%%.ddeb}.deb
-    done
-fi

debian/vitastor-buildenv.Dockerfile

@@ -1,31 +0,0 @@
-# Build environment for building Vitastor packages for Debian inside a container
-# cd ..
-# docker build --build-arg DISTRO=debian --build-arg REL=bullseye -f debian/vitastor.Dockerfile -t vitastor-buildenv:bullseye .
-# docker run --rm -e REL=bullseye -v ./:/root/vitastor /root/vitastor/debian/vitastor-build.sh
-
-ARG DISTRO=debian
-ARG REL=
-FROM $DISTRO:$REL
-ARG DISTRO=debian
-ARG REL=
-
-WORKDIR /root
-
-RUN set -e -x; \
-    if [ "$REL" = "buster" ]; then \
-        perl -i -pe 's/deb.debian.org/archive.debian.org/' /etc/apt/sources.list; \
-        apt-get update; \
-        apt-get -y install wget; \
-        wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg; \
-        echo "deb https://vitastor.io/debian $REL main" >> /etc/apt/sources.list; \
-    fi; \
-    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
-    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/*.sources || true; \
-    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
-    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
-
-RUN apt-get update && \
-    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake \
-        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
-    apt-get -y build-dep fio && \
-    apt-get --download-only source fio

debian/vitastor.Dockerfile

@@ -0,0 +1,65 @@
+# Build Vitastor packages for Debian inside a container
+# cd ..; podman build --build-arg DISTRO=debian --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
+
+ARG DISTRO=debian
+ARG REL=
+FROM $DISTRO:$REL
+ARG DISTRO=debian
+ARG REL=
+
+WORKDIR /root
+
+RUN set -e -x; \
+    if [ "$REL" = "buster" ]; then \
+        apt-get update; \
+        apt-get -y install wget; \
+        wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg; \
+        echo "deb https://vitastor.io/debian $REL main" >> /etc/apt/sources.list; \
+    fi; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake \
+        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
+    apt-get -y build-dep fio && \
+    apt-get --download-only source fio
+
+ADD . /root/vitastor
+RUN set -e -x; \
+    [ -e /usr/lib/x86_64-linux-gnu/pkgconfig/libisal.pc ] || cp /root/vitastor/debian/libisal.pc /usr/lib/x86_64-linux-gnu/pkgconfig; \
+    mkdir -p /root/fio-build/; \
+    cd /root/fio-build/; \
+    rm -rf /root/fio-build/*; \
+    dpkg-source -x /root/fio*.dsc; \
+    mkdir -p /root/packages/vitastor-$REL; \
+    rm -rf /root/packages/vitastor-$REL/*; \
+    cd /root/packages/vitastor-$REL; \
+    FULLVER=$(head -n1 /root/vitastor/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    VER=${FULLVER%%-*}; \
+    cp -r /root/vitastor vitastor-$VER; \
+    cd vitastor-$VER; \
+    ln -s /root/fio-build/fio-*/ ./fio; \
+    FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
+    sh copy-fio-includes.sh; \
+    rm fio; \
+    mkdir -p a b debian/patches; \
+    mv fio-copy b/fio; \
+    diff -NaurpbB a b > debian/patches/fio-headers.patch || true; \
+    echo fio-headers.patch >> debian/patches/series; \
+    rm -rf a b; \
+    echo "dep:fio=$FIO" > debian/fio_version; \
+    cd /root/packages/vitastor-$REL/vitastor-$VER; \
+    mkdir mon/node_modules; \
+    cd mon/node_modules; \
+    curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx; \
+    curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx; \
+    cd /root/packages/vitastor-$REL; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER; \
+    cd vitastor-$VER; \
+    DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$FULLVER""$REL" "Rebuild for $REL"; \
+    DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
+    rm -rf /root/packages/vitastor-$REL/vitastor-*/

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.2.3
+VITASTOR_VERSION ?= v2.2.2
 
 all: build push
 

docker/etc/apt/preferences

@@ -1,3 +0,0 @@
-Package: *
-Pin: release n=bookworm-backports
-Pin-Priority: 500

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.2.3
+VITASTOR_VERSION=v2.2.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/network.en.md

@@ -74,7 +74,7 @@ Consider `use_rdmacm` for such networks.
 ## use_rdmacm
 
 - Type: boolean
-- Default: false
+- Default: true
 
 Use an alternative implementation of RDMA through RDMA-CM (Connection
 Manager). Works with all RDMA networks: Infiniband, iWARP and

docs/config/network.ru.md

@@ -74,7 +74,7 @@ RDMA-устройства, но они не имеют соединения с
 ## use_rdmacm
 
 - Тип: булево (да/нет)
-- Значение по умолчанию: false
+- Значение по умолчанию: true
 
 Использовать альтернативную реализацию RDMA на основе RDMA-CM (Connection
 Manager). Работает со всеми типами RDMA-сетей: Infiniband, iWARP и

docs/config/src/network.yml

@@ -51,7 +51,7 @@
     Рассмотрите включение `use_rdmacm` для таких сетей.
 - name: use_rdmacm
   type: bool
-  default: false
+  default: true
   info: |
     Use an alternative implementation of RDMA through RDMA-CM (Connection
     Manager). Works with all RDMA networks: Infiniband, iWARP and

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.2.3`
+   `docker pull vitastor:v2.2.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.2.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.2.3`
+   `docker pull vitastor:v2.2.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.2.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/packages.en.md

@@ -15,7 +15,6 @@
   - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
   - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
-  - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
   - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
     stable version from 0.9.x branch instead of 1.x
 - Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`

docs/installation/packages.ru.md

@@ -15,7 +15,6 @@
   - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
   - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
-  - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
   - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
     установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`

mon/mon.js

@@ -96,7 +96,6 @@ class Mon
                     }
                     else
                     {
-                        res.setHeader('Content-Type', 'text/plain; version=0.0.4; charset=utf-8');
                         res.write(export_prometheus_metrics(this.state));
                     }
                 }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.2.3",
+  "version": "2.2.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.2.3",
+  "version": "2.2.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/VitastorPlugin.pm

@@ -410,8 +410,8 @@ sub volume_size_info
     my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/';
     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
     my $info = _process_list($scfg, $storeid, run_cli($scfg, [ 'ls', $prefix.$name ]))->[0];
-    # (size, format, used, parent, ctime)
+    #return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size;
-    return wantarray ? ($info->{size}, $info->{format}, $info->{size}, $info->{parent}, 0) : $info->{size};
+    return $info->{size};
 }
 
 sub volume_resize

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.2.3'
+VITASTOR_VERSION = '2.2.2'
 
 LOG = logging.getLogger(__name__)
 

patches/libvirt-11.5-vitastor.diff

@@ -1,637 +0,0 @@
-diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
-index aaad4a3da1..5f5daa8341 100644
---- a/include/libvirt/libvirt-storage.h
-+++ b/include/libvirt/libvirt-storage.h
-@@ -326,6 +326,7 @@ typedef enum {
-     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
-+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
- } virConnectListAllStoragePoolsFlags;
- 
- int                     virConnectListAllStoragePools(virConnectPtr conn,
-diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
-index 1e24e41a48..ce359a4cf8 100644
---- a/src/conf/domain_conf.c
-+++ b/src/conf/domain_conf.c
-@@ -7435,7 +7435,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
-     src->configFile = virXPathString("string(./config/@file)", ctxt);
- 
-     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
--        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
-         src->query = virXMLPropString(node, "query");
- 
-     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
-@@ -31871,6 +31872,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
- 
-     case VIR_STORAGE_POOL_MPATH:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_SHEEPDOG:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_LAST:
-diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
-index b28af7fa56..d1aae6e43e 100644
---- a/src/conf/domain_validate.c
-+++ b/src/conf/domain_validate.c
-@@ -504,6 +504,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-@@ -576,7 +577,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         }
-     }
- 
--    /* internal snapshots and config files are currently supported only with rbd: */
-+    /* internal snapshots are currently supported only with rbd: */
-     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
-         if (src->snapshot) {
-@@ -584,10 +585,14 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-                            _("<snapshot> element is currently supported only with 'rbd' disks"));
-             return -1;
-         }
--
-+    }
-+    /* config files are currently supported only with rbd and vitastor: */
-+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
-         if (src->configFile) {
-             virReportError(VIR_ERR_XML_ERROR, "%s",
--                           _("<config> element is currently supported only with 'rbd' disks"));
-+                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
-             return -1;
-         }
-     }
-diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
-index 183dd5db5e..dcc0d1a778 100644
---- a/src/conf/schemas/domaincommon.rng
-+++ b/src/conf/schemas/domaincommon.rng
-@@ -2066,6 +2066,35 @@
-     </element>
-   </define>
- 
-+  <define name="diskSourceNetworkProtocolVitastor">
-+    <element name="source">
-+      <interleave>
-+        <attribute name="protocol">
-+          <value>vitastor</value>
-+        </attribute>
-+        <ref name="diskSourceCommon"/>
-+        <optional>
-+          <attribute name="name"/>
-+        </optional>
-+        <optional>
-+          <attribute name="query"/>
-+        </optional>
-+        <zeroOrMore>
-+          <ref name="diskSourceNetworkHost"/>
-+        </zeroOrMore>
-+        <optional>
-+          <element name="config">
-+            <attribute name="file">
-+              <ref name="absFilePath"/>
-+            </attribute>
-+            <empty/>
-+          </element>
-+        </optional>
-+        <empty/>
-+      </interleave>
-+    </element>
-+  </define>
-+
-   <define name="diskSourceNetworkProtocolISCSI">
-     <element name="source">
-       <attribute name="protocol">
-@@ -2416,6 +2445,7 @@
-       <ref name="diskSourceNetworkProtocolSimple"/>
-       <ref name="diskSourceNetworkProtocolVxHS"/>
-       <ref name="diskSourceNetworkProtocolNFS"/>
-+      <ref name="diskSourceNetworkProtocolVitastor"/>
-     </choice>
-   </define>
- 
-diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
-index 1dc9365bf2..a8a736be81 100644
---- a/src/conf/storage_conf.c
-+++ b/src/conf/storage_conf.c
-@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
-               "logical", "disk", "iscsi",
-               "iscsi-direct", "scsi", "mpath",
-               "rbd", "sheepdog", "gluster",
--              "zfs", "vstorage",
-+              "zfs", "vstorage", "vitastor",
- );
- 
- VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
-@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
-           .formatToString = virStorageFileFormatTypeToString,
-       }
-     },
-+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
-+     .poolOptions = {
-+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
-+                   VIR_STORAGE_POOL_SOURCE_NAME),
-+      },
-+      .volOptions = {
-+          .defaultFormat = VIR_STORAGE_FILE_RAW,
-+          .formatFromString = virStorageVolumeFormatFromString,
-+          .formatToString = virStorageFileFormatTypeToString,
-+      }
-+    },
-     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
-      .poolOptions = {
-          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
-                        _("element 'name' is mandatory for RBD pool"));
-         return -1;
-     }
-+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
-+        virReportError(VIR_ERR_XML_ERROR, "%s",
-+                       _("element 'name' is mandatory for Vitastor pool"));
-+        return -1;
-+    }
- 
-     if (options->formatFromString) {
-         g_autofree char *format = NULL;
-@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
-     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
-      * files, so they don't have a target */
-     if (def->type != VIR_STORAGE_POOL_RBD &&
-+        def->type != VIR_STORAGE_POOL_VITASTOR &&
-         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
-         def->type != VIR_STORAGE_POOL_GLUSTER &&
-         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
-diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
-index fc67957cfe..720c07ef74 100644
---- a/src/conf/storage_conf.h
-+++ b/src/conf/storage_conf.h
-@@ -103,6 +103,7 @@ typedef enum {
-     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
-     VIR_STORAGE_POOL_ZFS,      /* ZFS */
-     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
-+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
- 
-     VIR_STORAGE_POOL_LAST,
- } virStoragePoolType;
-@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
-+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
-diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
-index 8a063be244..dd9c7f11a2 100644
---- a/src/conf/storage_source_conf.c
-+++ b/src/conf/storage_source_conf.c
-@@ -89,6 +89,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
-               "ssh",
-               "vxhs",
-               "nfs",
-+              "vitastor",
- );
- 
- 
-@@ -1314,6 +1315,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-             return 24007;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             /* we don't provide a default for RBD */
-             return 0;
-diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
-index ebddf28cd6..873a2be65c 100644
---- a/src/conf/storage_source_conf.h
-+++ b/src/conf/storage_source_conf.h
-@@ -130,6 +130,7 @@ typedef enum {
-     VIR_STORAGE_NET_PROTOCOL_SSH,
-     VIR_STORAGE_NET_PROTOCOL_VXHS,
-     VIR_STORAGE_NET_PROTOCOL_NFS,
-+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
- 
-     VIR_STORAGE_NET_PROTOCOL_LAST
- } virStorageNetProtocol;
-diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
-index 59fa5da372..4739167f5f 100644
---- a/src/conf/virstorageobj.c
-+++ b/src/conf/virstorageobj.c
-@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
-             return 1;
-         break;
- 
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_RBD:
-     case VIR_STORAGE_POOL_LAST:
-@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
-                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
-                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
-+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
-+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
-                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
-diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
-index db7660aac4..561df34709 100644
---- a/src/libvirt-storage.c
-+++ b/src/libvirt-storage.c
-@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
-  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
-  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
-+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
-  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
-  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
-diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
-index bdd30dd65a..5353e00b4a 100644
---- a/src/libxl/libxl_conf.c
-+++ b/src/libxl/libxl_conf.c
-@@ -1081,6 +1081,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
-index ec8de30c01..61eab9606d 100644
---- a/src/libxl/xen_xl.c
-+++ b/src/libxl/xen_xl.c
-@@ -1461,6 +1461,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
-index 32568d4ae6..e625fa0720 100644
---- a/src/qemu/qemu_block.c
-+++ b/src/qemu/qemu_block.c
-@@ -731,6 +731,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
- }
- 
- 
-+static virJSONValue *
-+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
-+{
-+    virJSONValue *ret = NULL;
-+    virStorageNetHostDef *host;
-+    size_t i;
-+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-+    g_autofree char *etcd = NULL;
-+
-+    for (i = 0; i < src->nhosts; i++) {
-+        host = src->hosts + i;
-+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
-+            return NULL;
-+        }
-+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
-+    }
-+    if (src->nhosts > 0) {
-+        etcd = virBufferContentAndReset(&buf);
-+    }
-+
-+    if (virJSONValueObjectAdd(&ret,
-+                              "S:etcd-host", etcd,
-+                              "S:etcd-prefix", src->query,
-+                              "S:config-path", src->configFile,
-+                              "s:image", src->path,
-+                              NULL) < 0)
-+        return NULL;
-+
-+    return ret;
-+}
-+
-+
- static virJSONValue *
- qemuBlockStorageSourceGetSshProps(virStorageSource *src)
- {
-@@ -1082,6 +1114,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
-                 return NULL;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return NULL;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SSH:
-             driver = "ssh";
-             if (!(fileprops = qemuBlockStorageSourceGetSshProps(src)))
-@@ -1985,6 +2023,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
- 
-             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             case VIR_STORAGE_NET_PROTOCOL_RBD:
-+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-             case VIR_STORAGE_NET_PROTOCOL_VXHS:
-             case VIR_STORAGE_NET_PROTOCOL_NFS:
-             case VIR_STORAGE_NET_PROTOCOL_SSH:
-@@ -2365,6 +2404,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
-                 return -1;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return -1;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SSH:
-             if (srcPriv->nbdkitProcess) {
-                 /* disk creation not yet supported with nbdkit, and even if it
-diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
-index 0d2548d8d4..91121d6e1f 100644
---- a/src/qemu/qemu_domain.c
-+++ b/src/qemu/qemu_domain.c
-@@ -4526,7 +4526,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
-     if (src->query &&
-         (actualType != VIR_STORAGE_TYPE_NETWORK ||
-          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
--          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
-         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                        _("query is supported only with HTTP(S) protocols"));
-         return -1;
-@@ -8954,6 +8955,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
-         break;
- 
-     case VIR_STORAGE_NET_PROTOCOL_RBD:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
-index 8128154749..afb339b9b0 100644
---- a/src/qemu/qemu_snapshot.c
-+++ b/src/qemu/qemu_snapshot.c
-@@ -662,6 +662,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-@@ -887,6 +888,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
-index e19e032427..59f91f4710 100644
---- a/src/storage/storage_driver.c
-+++ b/src/storage/storage_driver.c
-@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
- 
-         case VIR_STORAGE_POOL_GLUSTER:
-         case VIR_STORAGE_POOL_RBD:
-+        case VIR_STORAGE_POOL_VITASTOR:
-         case VIR_STORAGE_POOL_SHEEPDOG:
-         case VIR_STORAGE_POOL_ZFS:
-         case VIR_STORAGE_POOL_LAST:
-diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
-index 80681924ea..8a3ade9ec0 100644
---- a/src/storage_file/storage_source_backingstore.c
-+++ b/src/storage_file/storage_source_backingstore.c
-@@ -287,6 +287,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
- }
- 
- 
-+static int
-+virStorageSourceParseVitastorColonString(const char *colonstr,
-+                                         virStorageSource *src)
-+{
-+    char *p, *e, *next;
-+    g_autofree char *options = NULL;
-+
-+    /* optionally skip the "vitastor:" prefix if provided */
-+    if (STRPREFIX(colonstr, "vitastor:"))
-+        colonstr += strlen("vitastor:");
-+
-+    options = g_strdup(colonstr);
-+
-+    p = options;
-+    while (*p) {
-+        /* find : delimiter or end of string */
-+        for (e = p; *e && *e != ':'; ++e) {
-+            if (*e == '\\') {
-+                e++;
-+                if (*e == '\0')
-+                    break;
-+            }
-+        }
-+        if (*e == '\0') {
-+            next = e;    /* last kv pair */
-+        } else {
-+            next = e + 1;
-+            *e = '\0';
-+        }
-+
-+        if (STRPREFIX(p, "image=")) {
-+            src->path = g_strdup(p + strlen("image="));
-+        } else if (STRPREFIX(p, "etcd-prefix=")) {
-+            src->query = g_strdup(p + strlen("etcd-prefix="));
-+        } else if (STRPREFIX(p, "config-path=")) {
-+            src->configFile = g_strdup(p + strlen("config-path="));
-+        } else if (STRPREFIX(p, "etcd-host=")) {
-+            char *h, *sep;
-+
-+            h = p + strlen("etcd-host=");
-+            while (h < e) {
-+                for (sep = h; sep < e; ++sep) {
-+                    if (*sep == '\\' && (sep[1] == ',' ||
-+                                         sep[1] == ';' ||
-+                                         sep[1] == ' ')) {
-+                        *sep = '\0';
-+                        sep += 2;
-+                        break;
-+                    }
-+                }
-+
-+                if (virStorageSourceRBDAddHost(src, h) < 0)
-+                    return -1;
-+
-+                h = sep;
-+            }
-+        }
-+
-+        p = next;
-+    }
-+
-+    if (!src->path) {
-+        return -1;
-+    }
-+
-+    return 0;
-+}
-+
-+
- static int
- virStorageSourceParseNBDColonString(const char *nbdstr,
-                                     virStorageSource *src)
-@@ -399,6 +468,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
-             return -1;
-         break;
- 
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
-+            return -1;
-+        break;
-+
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-@@ -975,6 +1049,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
-     return 0;
- }
- 
-+static int
-+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
-+                                         virJSONValue *json,
-+                                         const char *jsonstr G_GNUC_UNUSED,
-+                                         int opaque G_GNUC_UNUSED)
-+{
-+    const char *filename;
-+    const char *image = virJSONValueObjectGetString(json, "image");
-+    const char *conf = virJSONValueObjectGetString(json, "config-path");
-+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
-+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
-+    size_t nservers;
-+    size_t i;
-+
-+    src->type = VIR_STORAGE_TYPE_NETWORK;
-+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
-+
-+    /* legacy syntax passed via 'filename' option */
-+    if ((filename = virJSONValueObjectGetString(json, "filename")))
-+        return virStorageSourceParseVitastorColonString(filename, src);
-+
-+    if (!image) {
-+        virReportError(VIR_ERR_INVALID_ARG, "%s",
-+                       _("missing image name in Vitastor backing volume "
-+                         "JSON specification"));
-+        return -1;
-+    }
-+
-+    src->path = g_strdup(image);
-+    src->configFile = g_strdup(conf);
-+    src->query = g_strdup(etcd_prefix);
-+
-+    if (servers) {
-+        nservers = virJSONValueArraySize(servers);
-+
-+        src->hosts = g_new0(virStorageNetHostDef, nservers);
-+        src->nhosts = nservers;
-+
-+        for (i = 0; i < nservers; i++) {
-+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
-+                                                                  virJSONValueArrayGet(servers, i)) < 0)
-+                return -1;
-+        }
-+    }
-+
-+    return 0;
-+}
-+
- static int
- virStorageSourceParseBackingJSONRaw(virStorageSource *src,
-                                     virJSONValue *json,
-@@ -1152,6 +1274,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
-     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
-     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
-     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
-+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
-     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
-     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
-     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
-diff --git a/src/test/test_driver.c b/src/test/test_driver.c
-index 25335d9002..cf54069fbe 100644
---- a/src/test/test_driver.c
-+++ b/src/test/test_driver.c
-@@ -7340,6 +7340,7 @@ testStorageVolumeTypeForPool(int pooltype)
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-         return VIR_STORAGE_VOL_NETWORK;
-     case VIR_STORAGE_POOL_LOGICAL:
-     case VIR_STORAGE_POOL_DISK:
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-index eee75af746..8bd0a57bdd 100644
---- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='no'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-index 805950a937..852df0de16 100644
---- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='yes'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
-index d5c2531ab8..b19308ac38 100644
---- a/tests/storagepoolxml2argvtest.c
-+++ b/tests/storagepoolxml2argvtest.c
-@@ -57,6 +57,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_ZFS:
-     case VIR_STORAGE_POOL_VSTORAGE:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_LAST:
-     default:
-         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
-diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
-index 2010ef1356..072e2ff9e8 100644
---- a/tools/virsh-pool.c
-+++ b/tools/virsh-pool.c
-@@ -1187,6 +1187,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
-             case VIR_STORAGE_POOL_VSTORAGE:
-                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
-                 break;
-+            case VIR_STORAGE_POOL_VITASTOR:
-+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
-+                break;
-             case VIR_STORAGE_POOL_LAST:
-                 break;
-             }

rpm/build-tarball.sh

@@ -7,24 +7,22 @@ set -e
 VITASTOR=$(dirname $0)
 VITASTOR=$(realpath "$VITASTOR/..")
 
-REL=$(rpm --eval '%dist')
+EL=$(rpm --eval '%dist')
-REL=${REL##.}
+if [ "$EL" = ".el8" ]; then
-if [ "$REL" = "el8" ]; then
     # CentOS 8
     . /opt/rh/gcc-toolset-9/enable
-elif [ "$REL" = "el7" ]; then
+elif [ "$EL" = ".el7" ]; then
     # CentOS 7
     . /opt/rh/devtoolset-9/enable
 fi
 cd ~/rpmbuild/SPECS
 rpmbuild -bp fio.spec
 cd $VITASTOR
-VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
+VER=$(grep ^Version: rpm/vitastor-el7.spec | awk '{print $2}')
-rm -rf fio
 ln -s ~/rpmbuild/BUILD/fio*/ fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
-perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-$REL.spec
+perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform "s#^#vitastor-$VER/#" --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-$VER.$REL.tar.gz $(ls | grep -v packages)
+tar --transform "s#^#vitastor-$VER/#" --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-$VER$(rpm --eval '%dist').tar.gz *

rpm/vitastor-build.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -e -x
-REL=$(rpm --eval '%dist')
-REL=${REL##.}
-cd /root/vitastor/rpm
-./build-tarball.sh
-VER=$(grep ^Version: vitastor-$REL.spec | awk '{print $2}')
-cp /root/vitastor-$VER.$REL.tar.gz ~/rpmbuild/SOURCES
-cp vitastor-$REL.spec ~/rpmbuild/SPECS/vitastor.spec
-cd ~/rpmbuild/SPECS/
-rpmbuild -ba vitastor.spec
-mkdir -p /root/vitastor/packages/vitastor-$REL
-rm -rf /root/vitastor/packages/vitastor-$REL/*
-cp ~/rpmbuild/RPMS/*/*vitastor* /root/vitastor/packages/vitastor-$REL/
-cp ~/rpmbuild/SRPMS/vitastor* /root/vitastor/packages/vitastor-$REL/

rpm/vitastor-el7.Dockerfile

@@ -1,8 +1,5 @@
 # Build packages for CentOS 7 inside a container
-# cd ..
+# cd ..; podman build -t vitastor-el7 -v `pwd`/packages:/root/packages -f rpm/vitastor-el7.Dockerfile .
-# docker build -t vitastor-buildenv:el7 -f rpm/vitastor-el7.Dockerfile .
-# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el7 /root/vitastor/rpm/vitastor-build.sh
-
 # localedef -i ru_RU -f UTF-8 ru_RU.UTF-8
 
 FROM centos:7
@@ -10,9 +7,7 @@ FROM centos:7
 WORKDIR /root
 
 RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
-RUN sed -i 's/^mirrorlist=/#mirrorlist=/; s!#baseurl=http://mirror.centos.org/centos/\$releasever!baseurl=http://vault.centos.org/7.9.2009!' /etc/yum.repos.d/*.repo
 RUN yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils rpm-build
-RUN perl -i -pe 's!mirrorlist=!#mirrorlist=!s; s!#\s*baseurl=http://mirror.centos.org!baseurl=http://vault.centos.org!' /etc/yum.repos.d/CentOS-SCLo-scl*.repo
 RUN yum -y install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm
 RUN yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gcc make cmake gperftools-devel \
     fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel
@@ -35,3 +30,18 @@ RUN set -e; \
     cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el7/
 
 RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
+
+ADD . /root/vitastor
+
+RUN set -e; \
+    cd /root/vitastor/rpm; \
+    sh build-tarball.sh; \
+    VER=$(grep ^Version: vitastor-el7.spec | awk '{print $2}'); \
+    cp /root/vitastor-$VER.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
+    cd ~/rpmbuild/SPECS/; \
+    rpmbuild -ba vitastor.spec; \
+    mkdir -p /root/packages/vitastor-el7; \
+    rm -rf /root/packages/vitastor-el7/*; \
+    cp ~/rpmbuild/RPMS/*/*vitastor* /root/packages/vitastor-el7/; \
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el7/

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.3
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.3.el7.tar.gz
+Source0:        vitastor-2.2.2.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -82,7 +82,7 @@ Vitastor library headers for development.
 Summary:        Vitastor - fio drivers
 Group:          Development/Libraries
 Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.7-2.el7
+Requires:       fio = 3.7-1.el7
 
 
 %description -n vitastor-fio

rpm/vitastor-el8.Dockerfile

@@ -1,7 +1,5 @@
 # Build packages for CentOS 8 inside a container
-# cd ..
+# cd ..; podman build -t vitastor-el8 -v `pwd`/packages:/root/packages -f rpm/vitastor-el8.Dockerfile .
-# docker build -t vitastor-buildenv:el8 -f rpm/vitastor-el8.Dockerfile .
-# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el8 /root/vitastor/rpm/vitastor-build.sh
 
 FROM centos:8
 
@@ -31,3 +29,18 @@ RUN set -e; \
     cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el8/
 
 RUN rpm -i `ls /root/packages/liburing-el8/liburing-*.x86_64.rpm | grep -v debug`
+
+ADD . /root/vitastor
+
+RUN set -e; \
+    cd /root/vitastor/rpm; \
+    sh build-tarball.sh; \
+    VER=$(grep ^Version: vitastor-el8.spec | awk '{print $2}'); \
+    cp /root/vitastor-$VER.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
+    cd ~/rpmbuild/SPECS/; \
+    rpmbuild -ba vitastor.spec; \
+    mkdir -p /root/packages/vitastor-el8; \
+    rm -rf /root/packages/vitastor-el8/*; \
+    cp ~/rpmbuild/RPMS/*/*vitastor* /root/packages/vitastor-el8/; \
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el8/

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.3
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.3.el8.tar.gz
+Source0:        vitastor-2.2.2.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -80,7 +80,7 @@ Vitastor library headers for development.
 Summary:        Vitastor - fio drivers
 Group:          Development/Libraries
 Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.19-3.el8
+Requires:       fio = 3.7-3.el8
 
 
 %description -n vitastor-fio

rpm/vitastor-el9.Dockerfile

@@ -1,7 +1,5 @@
 # Build packages for AlmaLinux 9 inside a container
-# cd ..
+# cd ..; podman build -t vitastor-el9 -v `pwd`/packages:/root/packages -f rpm/vitastor-el9.Dockerfile .
-# docker build -t vitastor-buildenv:el9 -f rpm/vitastor-el9.Dockerfile .
-# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el9 /root/vitastor/rpm/vitastor-build.sh
 
 FROM almalinux:9
 
@@ -14,3 +12,18 @@ RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec
+
+ADD . /root/vitastor
+
+RUN set -e; \
+    cd /root/vitastor/rpm; \
+    sh build-tarball.sh; \
+    VER=$(grep ^Version: vitastor-el9.spec | awk '{print $2}'); \
+    cp /root/vitastor-$VER.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
+    cd ~/rpmbuild/SPECS/; \
+    rpmbuild -ba vitastor.spec; \
+    mkdir -p /root/packages/vitastor-el9; \
+    rm -rf /root/packages/vitastor-el9/*; \
+    cp ~/rpmbuild/RPMS/*/*vitastor* /root/packages/vitastor-el9/; \
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el9/

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.3
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.3.el9.tar.gz
+Source0:        vitastor-2.2.2.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.2.3")
+add_definitions(-DVITASTOR_VERSION="2.2.2")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -42,7 +42,8 @@
 #define BS_OP_DELETE 6
 #define BS_OP_LIST 7
 #define BS_OP_ROLLBACK 8
-#define BS_OP_MAX 8
+#define BS_OP_SYNC_STAB_ALL 9
+#define BS_OP_MAX 9
 
 #define BS_OP_PRIVATE_DATA_SIZE 256
 
@@ -112,6 +113,14 @@ Input:
 Output:
 - retval = 0 or negative error number (-ENOENT if no such version for stabilize)
 
+## BS_OP_SYNC_STAB_ALL
+
+ONLY FOR TESTS! Sync and mark all unstable object versions as stable, at once.
+
+Input: Nothing except opcode
+Output:
+- retval = 0 or negative error number (-EINVAL)
+
 ## BS_OP_LIST
 
 Get a list of all objects in this Blockstore.
@@ -138,7 +147,7 @@ Output:
 struct blockstore_op_t
 {
     // operation
-    uint64_t opcode = 0;
+    uint64_t opcode;
     // finish callback
     std::function<void (blockstore_op_t*)> callback;
     union __attribute__((__packed__))
@@ -162,9 +171,9 @@ struct blockstore_op_t
             uint32_t list_stable_limit;
         };
     };
-    void *buf = NULL;
+    void *buf;
-    void *bitmap = NULL;
+    void *bitmap;
-    int retval = 0;
+    int retval;
 
     uint8_t private_data[BS_OP_PRIVATE_DATA_SIZE];
 };

src/blockstore/blockstore_impl.cpp

@@ -343,6 +343,44 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
         ringloop->set_immediate([op]() { std::function<void (blockstore_op_t*)>(op->callback)(op); });
         return;
     }
+    if (op->opcode == BS_OP_SYNC_STAB_ALL)
+    {
+        std::function<void(blockstore_op_t*)> *old_callback = new std::function<void(blockstore_op_t*)>(op->callback);
+        op->opcode = BS_OP_SYNC;
+        op->callback = [this, old_callback](blockstore_op_t *op)
+        {
+            if (op->retval >= 0 && unstable_writes.size() > 0)
+            {
+                op->opcode = BS_OP_STABLE;
+                op->len = unstable_writes.size();
+                obj_ver_id *vers = new obj_ver_id[op->len];
+                op->buf = vers;
+                int i = 0;
+                for (auto it = unstable_writes.begin(); it != unstable_writes.end(); it++, i++)
+                {
+                    vers[i] = {
+                        .oid = it->first,
+                        .version = it->second,
+                    };
+                }
+                unstable_writes.clear();
+                op->callback = [old_callback](blockstore_op_t *op)
+                {
+                    obj_ver_id *vers = (obj_ver_id*)op->buf;
+                    delete[] vers;
+                    op->buf = NULL;
+                    (*old_callback)(op);
+                    delete old_callback;
+                };
+                this->enqueue_op(op);
+            }
+            else
+            {
+                (*old_callback)(op);
+                delete old_callback;
+            }
+        };
+    }
     if ((op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE) && !enqueue_write(op))
     {
         ringloop->set_immediate([op]() { std::function<void (blockstore_op_t*)>(op->callback)(op); });

src/blockstore/fio_engine.cpp

@@ -5,14 +5,14 @@
 //
 // Initialize storage for tests:
 //
-// dd if=/dev/zero of=test_data.bin bs=1M count=1024
+// dd if=/dev/zero of=test_data.bin bs=1024 count=1048576
+// dd if=/dev/zero of=test_meta.bin bs=1024 count=256
+// dd if=/dev/zero of=test_journal.bin bs=1024 count=4096
 //
 // Random write:
 //
-// [LD_PRELOAD=libasan.so.8] \
+// fio -thread -ioengine=./libfio_blockstore.so -name=test -bs=4k -direct=1 -fsync=16 -iodepth=16 -rw=randwrite \
-// fio -name=test -thread -ioengine=../build/src/blockstore/libfio_vitastor_blk.so \
+//     -bs_config='{"data_device":"./test_data.bin"}' -size=1000M
-//     -bs=4k -direct=1 -rw=randwrite -iodepth=16 -size=900M -loops=10 \
-//     -bs_config='{"data_device":"./test_data.bin","meta_offset":0,"journal_offset":16777216,"data_offset":33554432,"disable_data_fsync":true,"immediate_commit":"all","journal_no_same_sector_overwrites":true}'
 //
 // Linear write:
 //
@@ -38,14 +38,12 @@ struct bs_data
     std::vector<io_u*> completed;
     int op_n = 0, inflight = 0;
     bool last_sync = false;
-    bool trace = false;
 };
 
 struct bs_options
 {
     int __pad;
     char *json_config = NULL;
-    int trace = 0;
 };
 
 static struct fio_option options[] = {
@@ -58,16 +56,6 @@ static struct fio_option options[] = {
         .category = FIO_OPT_C_ENGINE,
         .group  = FIO_OPT_G_FILENAME,
     },
-    {
-        .name   = "bs_trace",
-        .lname  = "trace",
-        .type   = FIO_OPT_BOOL,
-        .off1   = offsetof(struct bs_options, trace),
-        .help   = "Trace operations",
-        .def    = "0",
-        .category = FIO_OPT_C_ENGINE,
-        .group  = FIO_OPT_G_FILENAME,
-    },
     {
         .name = NULL,
     },
@@ -75,7 +63,6 @@ static struct fio_option options[] = {
 
 static int bs_setup(struct thread_data *td)
 {
-    bs_options *o = (bs_options*)td->eo;
     bs_data *bsd;
     //fio_file *f;
     //int r;
@@ -96,8 +83,6 @@ static int bs_setup(struct thread_data *td)
         td->o.open_files++;
     }
 
-    bsd->trace = o->trace ? true : false;
-
     //f = td->files[0];
     //f->real_file_size = size;
     return 0;
@@ -191,19 +176,20 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->version = UINT64_MAX; // last unstable
         op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+        op->callback = [io](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
             bs_data *bsd = (bs_data*)io->engine_data;
             bsd->inflight--;
             bsd->completed.push_back(io);
-            if (bsd->trace)
+#ifdef BLOCKSTORE_DEBUG
-                printf("--- OP_READ %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+            printf("--- OP_READ %llx n=%d retval=%d\n", io, n, op->retval);
+#endif
             delete op;
         };
         break;
     case DDIR_WRITE:
-        op->opcode = BS_OP_WRITE_STABLE;
+        op->opcode = BS_OP_WRITE;
         op->buf = io->xfer_buf;
         op->oid = {
             .inode = 1,
@@ -212,28 +198,30 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->version = 0; // assign automatically
         op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+        op->callback = [io](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
             bs_data *bsd = (bs_data*)io->engine_data;
             bsd->inflight--;
             bsd->completed.push_back(io);
-            if (bsd->trace)
+#ifdef BLOCKSTORE_DEBUG
-                printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+            printf("--- OP_WRITE %llx n=%d retval=%d\n", io, n, op->retval);
+#endif
             delete op;
         };
         bsd->last_sync = false;
         break;
     case DDIR_SYNC:
-        op->opcode = BS_OP_SYNC;
+        op->opcode = BS_OP_SYNC_STAB_ALL;
-        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+        op->callback = [io](blockstore_op_t *op)
         {
             bs_data *bsd = (bs_data*)io->engine_data;
             io->error = op->retval < 0 ? -op->retval : 0;
             bsd->completed.push_back(io);
             bsd->inflight--;
-            if (bsd->trace)
+#ifdef BLOCKSTORE_DEBUG
-                printf("--- OP_SYNC %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+            printf("--- OP_SYNC %llx n=%d retval=%d\n", io, n, op->retval);
+#endif
             delete op;
         };
         bsd->last_sync = true;
@@ -244,8 +232,9 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         return FIO_Q_COMPLETED;
     }
 
-    if (bsd->trace)
+#ifdef BLOCKSTORE_DEBUG
-        printf("+++ %s %zx n=%d\n", op->opcode == BS_OP_READ ? "OP_READ" : (op->opcode == BS_OP_WRITE_STABLE ? "OP_WRITE" : "OP_SYNC"), (size_t)op, bsd->op_n);
+    printf("+++ %s %llx n=%d\n", op->opcode == OP_READ ? "OP_READ" : (op->opcode == OP_WRITE ? "OP_WRITE" : "OP_SYNC"), io, n);
+#endif
     io->error = 0;
     bsd->inflight++;
     bsd->bs->enqueue_op(op);

src/client/cluster_client.cpp

@@ -57,7 +57,6 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
     st_cli.on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); };
     st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
-    st_cli.on_change_pg_config_hook = [this]() { on_change_pool_config_hook(); };
     st_cli.on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); };
     st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
     st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
@@ -564,7 +563,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & os
 
 void cluster_client_t::on_load_pgs_hook(bool success)
 {
-    for (auto & pool_item: st_cli.pool_config)
+    for (auto pool_item: st_cli.pool_config)
     {
         pg_counts[pool_item.first] = pool_item.second.real_pg_count;
     }
@@ -584,15 +583,10 @@ void cluster_client_t::on_load_pgs_hook(bool success)
 
 void cluster_client_t::on_change_pool_config_hook()
 {
-    for (auto & pool_item: st_cli.pool_config)
+    for (auto pool_item: st_cli.pool_config)
     {
         if (pg_counts[pool_item.first] != pool_item.second.real_pg_count)
         {
-            if (log_level > 2 && pg_counts[pool_item.first])
-            {
-                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
-                    pg_counts[pool_item.first], pool_item.second.real_pg_count);
-            }
             // At this point, all pool operations should have been suspended
             // And now they have to be resliced!
             for (auto op = op_queue_head; op; op = op->next)

src/client/etcd_state_client.cpp

@@ -1056,10 +1056,6 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             }
             pool_item.second.real_pg_count = n;
         }
-        if (on_change_pg_config_hook)
-        {
-            on_change_pg_config_hook();
-        }
     }
     else if (key.substr(0, etcd_prefix.length()+12) == etcd_prefix+"/pg/history/")
     {
@@ -1186,7 +1182,9 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
         osd_num_t peer_osd = std::stoull(key.substr(etcd_prefix.length()+11));
         if (peer_osd > 0)
         {
-            if (value.is_object() && value["state"] == "up")
+            if (value.is_object() && value["state"] == "up" &&
+                value["addresses"].is_array() &&
+                value["port"].int64_value() > 0 && value["port"].int64_value() < 65536)
             {
                 this->peer_states[peer_osd] = value;
                 this->seen_peers.insert(peer_osd);

src/client/etcd_state_client.h

@@ -142,7 +142,6 @@ public:
     std::function<json11::Json()> load_pgs_checks_hook;
     std::function<void(bool)> on_load_pgs_hook;
     std::function<void()> on_change_pool_config_hook;
-    std::function<void()> on_change_pg_config_hook;
     std::function<void(pool_id_t)> on_change_backfillfull_hook;
     std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
     std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;

src/client/msgr_rdmacm.cpp

@@ -515,7 +515,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
     cl->peer_fd = conn->peer_fd;
     cl->peer_state = PEER_RDMA;
     cl->connect_timeout_id = -1;
-    cl->osd_num = peer_osd;
+    cl->in_osd_num = peer_osd;
     cl->in_buf = malloc_or_die(receive_buffer_size);
     cl->rdma_conn = rc;
     clients[conn->peer_fd] = cl;

src/client/qemu_driver.c

@@ -14,7 +14,6 @@
 #endif
 #include "block/block_int.h"
 #include "qapi/error.h"
-#include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
@@ -59,14 +58,12 @@ typedef struct VitastorClient
 {
     void *proxy;
     int uring_eventfd;
-    int auto_loop;
 
     void *watch;
     char *config_path;
     char *etcd_host;
     char *etcd_prefix;
     char *image;
-    char *file_mirror_path;
     int skip_parents;
     uint64_t inode;
     uint64_t pool;
@@ -158,19 +155,16 @@ static void qemu_vitastor_unescape(char *src)
 // vitastor[:key=value]*
 // vitastor[:etcd_host=127.0.0.1]:inode=1:pool=1[:rdma_gid_index=3]
 // vitastor:config_path=/etc/vitastor/vitastor.conf:image=testimg
-// vitastor://?config_path=/etc/vitastor/vitastor.conf&image=testimg
 static void vitastor_parse_filename(const char *filename, QDict *options, Error **errp)
 {
     const char *start;
     char *p, *buf;
 
-    int url_style = strstart(filename, "vitastor://?", &start);
+    if (!strstart(filename, "vitastor:", &start))
-    if (!url_style && !strstart(filename, "vitastor:", &start))
     {
         error_setg(errp, "File name must start with 'vitastor:'");
         return;
     }
-    char delim = url_style ? '&' : ':';
 
     buf = g_strdup(start);
     p = buf;
@@ -190,7 +184,7 @@ static void vitastor_parse_filename(const char *filename, QDict *options, Error
             if (name[i] == '_')
                 name[i] = '-';
         qemu_vitastor_unescape(name);
-        value = qemu_vitastor_next_tok(p, delim, &p);
+        value = qemu_vitastor_next_tok(p, ':', &p);
         qemu_vitastor_unescape(value);
         if (!strcmp(name, "inode") ||
             !strcmp(name, "pool") ||
@@ -300,8 +294,9 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
-    if (!client->auto_loop)
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
-        vitastor_schedule_uring_handler(client);
+    vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task->complete)
@@ -434,70 +429,6 @@ static void strarray_free(str_array *a)
 }
 #endif
 
-static void check_config(VitastorClient *client)
-{
-    size_t bufsize = 64*1024, maxsize = 1024*1024, done = 0;
-    ssize_t r = 0;
-    char *data = NULL;
-    QObject *obj = NULL;
-    QDict *qd = NULL;
-    int fd = open(client->config_path, O_RDONLY);
-    if (fd < 0)
-    {
-        fprintf(stderr, "vitastor: failed to read %s: %s\n", client->config_path, strerror(errno));
-        return;
-    }
-    data = (char*)malloc(bufsize);
-    if (!data)
-    {
-        fprintf(stderr, "vitastor: memory allocation failed\n");
-        goto end_free;
-    }
-    while (1)
-    {
-        if (bufsize <= done)
-        {
-            if (bufsize >= maxsize)
-            {
-                fprintf(stderr, "vitastor: config file exceeds %zu bytes\n", maxsize);
-                goto end_free;
-            }
-            bufsize *= 2;
-            data = (char*)realloc(data, bufsize);
-            if (!data)
-            {
-                fprintf(stderr, "vitastor: memory allocation failed\n");
-                goto end_free;
-            }
-        }
-        r = read(fd, data+done, bufsize-done);
-        if (r < 0)
-        {
-            if (errno == EAGAIN || errno == EINTR)
-                continue;
-            fprintf(stderr, "vitastor: failed to read %s: %s\n", client->config_path, strerror(errno));
-            break;
-        }
-        done += r;
-    }
-    obj = qobject_from_json(data
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 9 || QEMU_VERSION_MAJOR >= 3
-        , NULL
-#endif
-    );
-    qd = qobject_to(QDict, obj);
-    if (qd)
-    {
-        client->file_mirror_path = g_strdup(qdict_get_try_str(qd, "qemu_file_mirror_path"));
-    }
-end_free:
-    if (obj)
-        qobject_unref(obj);
-    if (data)
-        free(data);
-    close(fd);
-}
-
 static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, Error **errp)
 {
     VitastorRPC task;
@@ -516,15 +447,8 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
     client->rdma_gid_index = qdict_get_try_int(options, "rdma-gid-index", 0);
     client->rdma_mtu = qdict_get_try_int(options, "rdma-mtu", 0);
     client->ctx = bdrv_get_aio_context(bs);
-    if (client->config_path && strlen(client->config_path))
-    {
-        check_config(client);
-    }
 #if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
     str_array opt = {};
-    char version_buffer[32];
-    memset(version_buffer, ' ', sizeof(version_buffer));
-    version_buffer[sizeof(version_buffer)-1] = '\0';
     strarray_push_kv(&opt, "config_path", qdict_get_try_str(options, "config-path"));
     strarray_push_kv(&opt, "etcd_address", qdict_get_try_str(options, "etcd-host"));
     strarray_push_kv(&opt, "etcd_prefix", qdict_get_try_str(options, "etcd-prefix"));
@@ -534,9 +458,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
     strarray_push_kv(&opt, "rdma_gid_index", qdict_get_try_str(options, "rdma-gid-index"));
     strarray_push_kv(&opt, "rdma_mtu", qdict_get_try_str(options, "rdma-mtu"));
     strarray_push_kv(&opt, "client_writeback_allowed", (flags & BDRV_O_NOCACHE) ? "0" : "1");
-    strarray_push_kv(&opt, "__version_check_buffer", version_buffer);
     client->proxy = vitastor_c_create_uring_json(opt.items, opt.len);
-    client->auto_loop = 0;
     strarray_free(&opt);
     if (client->proxy)
     {
@@ -549,12 +471,6 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
             return -1;
         }
         universal_aio_set_fd_handler(client->ctx, client->uring_eventfd, vitastor_uring_handler, NULL, client);
-        int vitastor_version[3] = { 0 };
-        if (sscanf(version_buffer, "%d.%d.%d", &vitastor_version[0], &vitastor_version[1], &vitastor_version[2]) == 3 &&
-            vitastor_version[0] >= 2)
-        {
-            client->auto_loop = 1;
-        }
     }
     else
     {
@@ -688,34 +604,6 @@ static void vitastor_close(BlockDriverState *bs)
     client->last_bitmap = NULL;
 }
 
-#if QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 2
-static void vitastor_refresh_filename(BlockDriverState *bs)
-{
-    VitastorClient *client = bs->opaque;
-    size_t len = 0;
-    int n = 0;
-    if (client->image && client->file_mirror_path)
-    {
-        len = snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s%s", client->file_mirror_path, client->image);
-        return;
-    }
-    len = snprintf(bs->exact_filename, sizeof(bs->exact_filename), "vitastor://");
-    if (len < sizeof(bs->exact_filename))
-    {
-        if (client->image)
-            len += snprintf(bs->exact_filename+len, sizeof(bs->exact_filename)-len, "%cimage=%s", (n++ ? '&' : '?'), client->image);
-        else
-            len += snprintf(bs->exact_filename+len, sizeof(bs->exact_filename)-len, "%cpool=%ju&inode=%ju&size=%ju", (n++ ? '&' : '?'), client->pool, client->inode, client->size);
-    }
-    if (client->config_path && len < sizeof(bs->exact_filename))
-        len += snprintf(bs->exact_filename+len, sizeof(bs->exact_filename)-len, "%cconfig_path=%s", (n++ ? '&' : '?'), client->config_path);
-    if (client->etcd_host && len < sizeof(bs->exact_filename))
-        len += snprintf(bs->exact_filename+len, sizeof(bs->exact_filename)-len, "%cetcd_host=%s", (n++ ? '&' : '?'), client->etcd_host);
-    if (client->etcd_prefix && len < sizeof(bs->exact_filename))
-        len += snprintf(bs->exact_filename+len, sizeof(bs->exact_filename)-len, "%cetcd_prefix=%s", (n++ ? '&' : '?'), client->etcd_prefix);
-}
-#endif
-
 #if QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR > 2
 static int vitastor_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
 {
@@ -879,8 +767,9 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
-    if (!client->auto_loop)
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
-        vitastor_schedule_uring_handler(client);
+    vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -914,8 +803,9 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
-    if (!client->auto_loop)
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
-        vitastor_schedule_uring_handler(client);
+    vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -995,8 +885,9 @@ static int coroutine_fn vitastor_co_block_status(
         task.bitmap = client->last_bitmap = NULL;
         qemu_mutex_lock(&client->mutex);
         vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
-        if (!client->auto_loop)
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
-            vitastor_schedule_uring_handler(client);
+        vitastor_schedule_uring_handler(client);
+#endif
         qemu_mutex_unlock(&client->mutex);
         while (!task.complete)
         {
@@ -1083,8 +974,9 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
-    if (!client->auto_loop)
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
-        vitastor_schedule_uring_handler(client);
+    vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -1146,9 +1038,6 @@ static BlockDriver bdrv_vitastor = {
     .bdrv_get_info                  = vitastor_get_info,
     .bdrv_getlength                 = vitastor_getlength,
 #endif
-#if QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 2
-    .bdrv_refresh_filename          = vitastor_refresh_filename,
-#endif
 #if QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR > 2
     .bdrv_probe_blocksizes          = vitastor_probe_blocksizes,
 #endif

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.2.3
+Version: 2.2.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/client/vitastor_c.cpp

@@ -178,14 +178,7 @@ vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
     json11::Json::object cfg;
     for (int i = 0; i < options_len-1; i += 2)
     {
-        if (!strcmp(options[i], "__version_check_buffer"))
+        cfg[options[i]] = std::string(options[i+1]);
-        {
-            int buf_size = strlen(options[i+1])+1;
-            if (buf_size > strlen(VITASTOR_VERSION))
-                strncpy((char*)options[i+1], VITASTOR_VERSION, buf_size);
-        }
-        else
-            cfg[options[i]] = std::string(options[i+1]);
     }
     json11::Json cfg_json(cfg);
     vitastor_c *self = new vitastor_c;

src/client/vitastor_c.h

@@ -7,7 +7,7 @@
 #define VITASTOR_QEMU_PROXY_H
 
 // C API wrapper version
-#define VITASTOR_C_API_VERSION 4
+#define VITASTOR_C_API_VERSION 5
 
 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16

src/nfs/nfs_proxy.cpp

@@ -72,14 +72,12 @@ static const char* help_text =
     "  --port <PORT>         use port <PORT> for NFS services (default is 2049)\n"
     "                        specify \"auto\" to auto-select and print port\n"
     "  --portmap 0           do not listen on port 111 (portmap/rpcbind, requires root)\n"
-#ifdef WITH_RDMACM
     "  --nfs_rdma <PORT>     enable NFS-RDMA at RDMA-CM port <PORT> (you can try 20049)\n"
     "                        if RDMA is enabled and --port is set to 0, TCP will be disabled\n"
     "  --nfs_rdma_credit 16  maximum operation credit for RDMA clients (max iodepth)\n"
     "  --nfs_rdma_send 1024  maximum RDMA send operation count (should be larger than iodepth)\n"
     "  --nfs_rdma_alloc 1M   RDMA memory allocation rounding\n"
     "  --nfs_rdma_gc 64M     maximum unused RDMA buffers\n"
-#endif
     "\n"
     "vitastor-nfs --fs <NAME> upgrade\n"
     "  Upgrade FS metadata. Can be run online, but server(s) should be restarted\n"
@@ -207,15 +205,12 @@ void nfs_proxy_t::run(json11::Json cfg)
     default_pool = cfg["pool"].as_string();
     portmap_enabled = !json_is_false(cfg["portmap"]);
     nfs_port = cfg["port"].uint64_value() & 0xffff;
-#ifdef WITH_RDMACM
     nfs_rdma_port = cfg["nfs_rdma"].uint64_value() & 0xffff;
-#endif
     // Allow RDMA-only mode if port is explicitly set to 0
     // Allow port auto-selection in server mode if explicitly set to --port auto
     nfs_port_auto = cfg["port"] == "auto";
     if (!nfs_port)
         nfs_port = nfs_port_auto ? 0 : (!cfg["port"].is_null() && nfs_rdma_port ? -1 : 2049);
-#ifdef WITH_RDMACM
     nfs_rdma_credit = cfg["nfs_rdma_credit"].uint64_value();
     if (!nfs_rdma_credit)
         nfs_rdma_credit = 16;
@@ -228,7 +223,6 @@ void nfs_proxy_t::run(json11::Json cfg)
     nfs_rdma_gc = cfg["nfs_rdma_gc"].uint64_value();
     if (!nfs_rdma_gc)
         nfs_rdma_gc = 64*1048576;
-#endif
     export_root = cfg["nfspath"].string_value();
     if (!export_root.size())
         export_root = "/";
@@ -397,12 +391,10 @@ void nfs_proxy_t::run_server(json11::Json cfg)
             }
         });
     }
-#ifdef WITH_RDMACM
     if (nfs_rdma_port)
     {
         rdma_context = create_rdma(bind_address, nfs_rdma_port, nfs_rdma_credit, nfs_rdma_max_send, nfs_rdma_alloc, nfs_rdma_gc);
     }
-#endif
     if (mountpoint != "")
     {
         mount_fs();

src/osd/osd.cpp

@@ -196,6 +196,7 @@ void osd_t::parse_config(bool init)
             etcd_stats_interval = 30;
         readonly = json_is_true(config["readonly"]);
         run_primary = !json_is_false(config["run_primary"]);
+        allow_test_ops = json_is_true(config["allow_test_ops"]);
     }
     log_level = config["log_level"].uint64_value();
     auto old_no_rebalance = no_rebalance;
@@ -469,7 +470,11 @@ void osd_t::exec_op(osd_op_t *cur_op)
         finish_op(cur_op, -EROFS);
         return;
     }
-    if (cur_op->req.hdr.opcode == OSD_OP_SHOW_CONFIG)
+    if (cur_op->req.hdr.opcode == OSD_OP_TEST_SYNC_STAB_ALL)
+    {
+        exec_sync_stab_all(cur_op);
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SHOW_CONFIG)
     {
         exec_show_config(cur_op);
     }

src/osd/osd.h

@@ -119,6 +119,7 @@ class osd_t
     bool disable_tcp = false;
     // FIXME: Implement client queue depth limit
     int client_queue_depth = 128;
+    bool allow_test_ops = false;
     int print_stats_interval = 3;
     int slow_log_interval = 10;
     int immediate_commit = IMMEDIATE_NONE;
@@ -318,6 +319,7 @@ class osd_t
 
     // secondary ops
     bool sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid);
+    void exec_sync_stab_all(osd_op_t *cur_op);
     void exec_show_config(osd_op_t *cur_op);
     void exec_secondary(osd_op_t *cur_op);
     void exec_secondary_real(osd_op_t *cur_op);

src/osd/osd_flush.cpp

@@ -189,7 +189,7 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
             {
                 .len = (uint32_t)count,
             },
-            .buf = (uint8_t*)op->buf,
+            .buf = op->buf,
         });
         bs->enqueue_op(op->bs_op);
     }

src/osd/osd_peering.cpp

@@ -488,14 +488,15 @@ void osd_t::relock_pg(pg_t & pg)
             }
             else if (op->reply.hdr.retval != -EPIPE)
             {
-                printf("Failed to %s PG %u/%u on OSD %ju - ", op->req.sec_lock.flags == OSD_SEC_UNLOCK_PG ? "unlock" : "lock",
+                printf(
-                    pg_id.pool_id, pg_id.pg_num, peer_osd);
+                    (op->reply.hdr.retval == -ENOENT
-                if (op->reply.hdr.retval == -ENOENT)
+                        ? "Failed to %1$s PG %2$u/%3$u on OSD %4$ju - peer didn't load PG info yet\n"
-                    printf("peer didn't load PG info yet\n");
+                        : (op->reply.sec_lock.cur_primary
-                else if (op->reply.sec_lock.cur_primary)
+                            ? "Failed to %1$s PG %2$u/%3$u on OSD %4$ju - taken by OSD %6$ju (retval=%5$jd)\n"
-                    printf("taken by OSD %ju (retval=%jd)\n", op->reply.sec_lock.cur_primary, op->reply.hdr.retval);
+                            : "Failed to %1$s PG %2$u/%3$u on OSD %4$ju - retval=%5$jd\n")),
-                else
+                    op->req.sec_lock.flags == OSD_SEC_UNLOCK_PG ? "unlock" : "lock",
-                    printf("retval=%jd\n", op->reply.hdr.retval);
+                    pg_id.pool_id, pg_id.pg_num, peer_osd, op->reply.hdr.retval, op->reply.sec_lock.cur_primary
+                );
                 // Retry relocking/unlocking PG after a short time
                 pg.lock_waiting = true;
                 tfd->set_timer(pg_lock_retry_interval_ms, false, [this, pg_id](int)

src/osd/osd_primary_subops.cpp

@@ -46,6 +46,11 @@ void osd_t::autosync()
 void osd_t::finish_op(osd_op_t *cur_op, int retval)
 {
     inflight_ops--;
+    if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
+    {
+        printf("%jx %jx+%x p%jx v%jx r=%x\n", cur_op->req.rw.inode, cur_op->req.rw.offset, cur_op->req.rw.len,
+            cur_op->op_data ? cur_op->op_data->orig_ver : 0, cur_op->reply.rw.version, retval);
+    }
     if (cur_op->req.hdr.opcode == OSD_OP_READ ||
         cur_op->req.hdr.opcode == OSD_OP_WRITE ||
         cur_op->req.hdr.opcode == OSD_OP_DELETE)
@@ -209,8 +214,8 @@ void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
                 .offset = wr ? si->write_start : si->read_start,
                 .len = subop_len,
             } },
-            .buf = (uint8_t*)(wr ? si->write_buf : si->read_buf),
+            .buf = wr ? si->write_buf : si->read_buf,
-            .bitmap = (uint8_t*)si->bmp_buf,
+            .bitmap = si->bmp_buf,
         });
 #ifdef OSD_DEBUG
          printf(
@@ -290,6 +295,7 @@ static uint64_t bs_op_to_osd_op[] = {
     OSD_OP_SEC_DELETE,          // BS_OP_DELETE = 6
     OSD_OP_SEC_LIST,            // BS_OP_LIST = 7
     OSD_OP_SEC_ROLLBACK,        // BS_OP_ROLLBACK = 8
+    OSD_OP_TEST_SYNC_STAB_ALL,  // BS_OP_SYNC_STAB_ALL = 9
 };
 
 void osd_t::handle_primary_bs_subop(osd_op_t *subop)
@@ -419,14 +425,15 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
             ? msgr.clients[subop->peer_fd]->osd_num : 0);
         if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
         {
-            printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode],
+            printf(
-                subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version);
+                subop->peer_fd >= 0
-            if (subop->peer_fd >= 0 && peer_osd > 0)
+                    ? (peer_osd > 0
-                printf("on osd %ju: retval = %d (expected %d)\n", peer_osd, retval, expected);
+                        ? "%1$s subop to %2$jx:%3$jx v%4$ju failed on osd %7$ju: retval = %5$d (expected %6$d)\n"
-            else if (peer_osd > 0)
+                        : "%1$s subop to %2$jx:%3$jx v%4$ju failed on peer %8$d: retval = %5$d (expected %6$d)\n")
-                printf("on peer %d: retval = %d (expected %d)\n", subop->peer_fd, retval, expected);
+                    : "%1$s subop to %2$jx:%3$jx v%4$ju failed locally: retval = %5$d (expected %6$d)\n",
-            else
+                osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version,
-                printf("locally: retval = %d (expected %d)\n", retval, expected);
+                retval, expected, peer_osd, subop->peer_fd
+            );
         }
         else if (opcode == OSD_OP_SEC_DELETE)
         {
@@ -702,7 +709,7 @@ void osd_t::submit_primary_stab_subops(osd_op_t *cur_op)
                 {
                     .len = (uint32_t)stab_osd.len,
                 },
-                .buf = (uint8_t*)(op_data->unstable_writes + stab_osd.start),
+                .buf = (void*)(op_data->unstable_writes + stab_osd.start),
             });
             bs->enqueue_op(subops[i].bs_op);
         }
@@ -789,7 +796,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
                     {
                         .len = 1,
                     },
-                    .buf = (uint8_t*)(op_data->unstable_writes + i),
+                    .buf = (void*)(op_data->unstable_writes + i),
                 });
 #ifdef OSD_DEBUG
                 printf(

src/osd/osd_scrub.cpp

@@ -505,7 +505,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 if (votes[role] > 0 && votes[role] < votes[best])
                 {
                     printf(
-                        "[PG %u/%u] Object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
+                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[role].osd_num, votes[best],
@@ -528,7 +528,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 best = -1;
                 inconsistent = true;
                 printf(
-                    "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
+                    "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
                     INODE_POOL(op_data->oid.inode), op_data->pg_num,
                     op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
                 );
@@ -547,7 +547,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
         {
             inconsistent = true;
             printf(
-                "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
+                "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
                 INODE_POOL(op_data->oid.inode), op_data->pg_num,
                 op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
             );
@@ -574,7 +574,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 {
                     op_data->stripes[i].read_error = true;
                     printf(
-                        "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
+                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[i].role, op_data->stripes[i].osd_num,

src/osd/osd_secondary.cpp

@@ -161,8 +161,8 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         cur_op->bs_op->version = cur_op->req.sec_rw.version;
         cur_op->bs_op->offset = cur_op->req.sec_rw.offset;
         cur_op->bs_op->len = cur_op->req.sec_rw.len;
-        cur_op->bs_op->buf = (uint8_t*)cur_op->buf;
+        cur_op->bs_op->buf = cur_op->buf;
-        cur_op->bs_op->bitmap = (uint8_t*)cur_op->bitmap;
+        cur_op->bs_op->bitmap = cur_op->bitmap;
 #ifdef OSD_STUB
         cur_op->bs_op->retval = cur_op->bs_op->len;
 #endif
@@ -186,7 +186,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)
     {
         cur_op->bs_op->len = cur_op->req.sec_stab.len/sizeof(obj_ver_id);
-        cur_op->bs_op->buf = (uint8_t*)cur_op->buf;
+        cur_op->bs_op->buf = cur_op->buf;
 #ifdef OSD_STUB
         cur_op->bs_op->retval = 0;
 #endif
@@ -378,3 +378,27 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
     cur_op->iov.push_back(cur_op->buf, cfg_str.size()+1);
     finish_op(cur_op, cfg_str.size()+1);
 }
+
+void osd_t::exec_sync_stab_all(osd_op_t *cur_op)
+{
+    // Sync and stabilize all objects
+    // This command is only valid for tests
+    cur_op->bs_op = new blockstore_op_t();
+    if (!allow_test_ops)
+    {
+        cur_op->bs_op->retval = -EINVAL;
+        secondary_op_callback(cur_op);
+        return;
+    }
+    cur_op->bs_op->opcode = BS_OP_SYNC_STAB_ALL;
+    cur_op->bs_op->callback = [this, cur_op](blockstore_op_t *bs_op)
+    {
+        secondary_op_callback(cur_op);
+    };
+#ifdef OSD_STUB
+    cur_op->bs_op->retval = 0;
+    secondary_op_callback(cur_op);
+#else
+    bs->enqueue_op(cur_op->bs_op);
+#endif
+}

tests/run_tests.sh

@@ -10,8 +10,6 @@ cd $(dirname $0)
 ./test_change_pg_count.sh
 SCHEME=ec ./test_change_pg_count.sh
 
-./test_change_pg_count_online.sh
-
 ./test_change_pg_size.sh
 
 ./test_create_nomaxid.sh

tests/test_change_pg_count_online.sh

@@ -1,61 +0,0 @@
-#!/bin/bash -ex
-
-OSD_COUNT=${OSD_COUNT:-6}
-PG_COUNT=16
-GLOBAL_CONFIG=',"client_retry_interval":1000'
-
-. `dirname $0`/run_3osds.sh
-
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M
-
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randrw \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -loops=100 -cluster_log_level=3 -runtime=60 &>./testdata/fio.log &
-FIO_PID=$!
-
-try_change()
-{
-    n=$1
-
-    for i in {1..6}; do
-        echo --- Change PG count to $n --- >>testdata/osd$i.log
-    done
-    echo --- Change PG count to $n --- >>testdata/mon.log
-
-    $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}'
-    echo "Pool 1 (testpool) PG count changed from $PG_COUNT to $n" >> testdata/pgr.log
-    PG_COUNT=$n
-
-    sleep 10
-}
-
-sleep 5
-
-# 16 -> 32
-
-try_change 32
-
-# 32 -> 16
-
-try_change 16
-
-# 16 -> 25
-
-try_change 25
-
-# 25 -> 17
-
-try_change 17
-
-# 17 -> 16
-
-try_change 16
-
-wait $FIO_PID
-
-grep -P 'Pool 1 \(testpool\) PG count changed from \d+ to \d+' testdata/fio.log > testdata/pgc.log
-diff testdata/pgr.log testdata/pgc.log
-
-format_green OK

tests/test_heal.sh

@@ -55,6 +55,20 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
         -serialize_overlap=1 -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/bin/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
 
+# Wait for active/clean
+wait_up 120
+
+# Trigger scrub
+for i in $(seq 1 $PG_COUNT); do
+    $ETCDCTL put /vitastor/pg/history/1/$i `$ETCDCTL get --print-value-only /vitastor/pg/history/1/$i | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
+done
+
+# Wait for scrub to finish
+wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
+
+# Check that everything is ok
+wait_up 1
+
 qemu-img convert -S 4096 -p \
     -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
     -O raw ./testdata/bin/read.bin
@@ -64,7 +78,7 @@ if ! diff -q ./testdata/bin/read.bin ./testdata/bin/mirror.bin; then
     format_error Data lost during self-heal
 fi
 
-if grep -qP 'Checksum mismatch|BUG' ./testdata/osd*.log; then
+if grep -qP 'Checksum mismatch|BUG|Scrub error' ./testdata/osd*.log; then
     format_error Checksum mismatches or BUGs detected during test
 fi