Release 0.9.3

- Add patch for libvirt 9.0
- Add support for Proxmox VE 8.0
- Fix compatibility of the QEMU driver with iothread (QEMU rebuilds are coming)
- Fix vitastor-cli rm-data/rm/merge hanging when some OSDs are down.
  Allow deletions in unclean cluster at the cost of some data possibly
  "reappearing" when those OSDs start back. In that case you can just repeat
  the deletion request using rm-data.
- A bunch of bug fixes for snapshots:
  - Fix snapshot reads often not working at all with snapshot chain size > 2
  - Fix optimized snapshot data merge (children to parent)
  - Fix updating of image name index key during optimized merge
  - Fix auto-selection preventing the use of optimized merge with only 1 snapshot
  - Fix incorrect CAS retries during snapshot merge
  - Fix snapshot merge progress reporting
- Fix primary_read bitmap buffers use-after-free which could lead to
  incorrect allocation map reads
- Remove /usr/local/bin path from make-etcd
- Some documentation fixes

.gitea/workflows/test.Dockerfile

@@ -10,6 +10,9 @@ RUN set -e -x; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     ln -s /root/qemu-build/qemu-*/ ./qemu; \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
+    cd mon; \
+    npm install; \
+    cd ..; \
     mkdir build; \
     cd build; \
     cmake .. -DWITH_ASAN=yes -DWITH_QEMU=yes; \

.gitea/workflows/test.yml

@@ -190,24 +190,6 @@ jobs:
           echo ""
         done
 
-  test_failure_domain:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_failure_domain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_interrupted_rebalance:
     runs-on: ubuntu-latest
     needs: build
@@ -280,6 +262,60 @@ jobs:
           echo ""
         done
 
+  test_failure_domain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_failure_domain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_minsize_1:
     runs-on: ubuntu-latest
     needs: build
@@ -316,6 +352,114 @@ jobs:
           echo ""
         done
 
+  test_rm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_rm.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_chain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_chain_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_down:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_down_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_splitbrain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_splitbrain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_rebalance_verify:
     runs-on: ubuntu-latest
     needs: build
@@ -388,78 +532,6 @@ jobs:
           echo ""
         done
 
-  test_rm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_rm.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_snapshot.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec /root/vitastor/tests/test_snapshot.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_splitbrain:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_splitbrain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_write:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "0.9.2")
+set(VERSION "0.9.3")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.9.2
+VERSION ?= v0.9.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.9.2
+          image: vitalif/vitastor-csi:v0.9.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.9.2
+          image: vitalif/vitastor-csi:v0.9.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.9.2"
+    vitastorCSIDriverVersion = "0.9.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-pve-qemu.sh

@@ -0,0 +1,58 @@
+exit
+
+git clone https://git.yourcmc.ru/vitalif/pve-qemu .
+
+# bookworm
+
+docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-bullseye debian:bullseye bash
+
+perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources
+echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve bookworm pve-no-subscription' >> /etc/apt/sources.list
+echo 'deb https://vitastor.io/debian bookworm main' >> /etc/apt/sources.list
+echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
+echo 'ru_RU UTF-8' >> /etc/locale.gen
+echo 'en_US UTF-8' >> /etc/locale.gen
+apt-get update
+apt-get install wget ca-certificates
+wget https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
+wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
+apt-get update
+apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
+mk-build-deps --install ./control
+
+# bullseye
+
+docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-bullseye debian:bullseye bash
+
+grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb /deb-src /' >> /etc/apt/sources.list
+echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve bullseye pve-no-subscription' >> /etc/apt/sources.list
+echo 'deb https://vitastor.io/debian bullseye main' >> /etc/apt/sources.list
+echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
+echo 'ru_RU UTF-8' >> /etc/locale.gen
+echo 'en_US UTF-8' >> /etc/locale.gen
+apt-get update
+apt-get install wget
+wget https://enterprise.proxmox.com/debian/proxmox-release-bullseye.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg
+wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
+apt-get update
+apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
+mk-build-deps --install ./control
+
+# buster
+
+docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-buster debian:buster bash
+
+grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb /deb-src /' >> /etc/apt/sources.list
+echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve buster pve-no-subscription' >> /etc/apt/sources.list
+echo 'deb https://vitastor.io/debian buster main' >> /etc/apt/sources.list
+echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list
+echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
+echo 'ru_RU UTF-8' >> /etc/locale.gen
+echo 'en_US UTF-8' >> /etc/locale.gen
+apt-get update
+apt-get install wget ca-certificates
+wget http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg -O /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
+apt-get update
+apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
+mk-build-deps --install ./control

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.9.2-1) unstable; urgency=medium
+vitastor (0.9.3-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.9.2-1) unstable; urgency=medium
+vitastor (0.9.3-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/patched-qemu.Dockerfile

@@ -20,7 +20,7 @@ RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install qemu fio liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
 RUN apt-get -y build-dep qemu
 # To build a custom version
 #RUN cp /root/packages/qemu-orig/* /root
@@ -40,8 +40,9 @@ RUN set -e; \
     cd /root/packages/qemu-$REL; \
     dpkg-source -x /root/qemu*.dsc; \
     QEMU_VER=$(ls -d qemu*/ | perl -pe 's!^.*(\d+\.\d+).*!$1!'); \
-    cp /root/vitastor/patches/qemu-$QEMU_VER-vitastor.patch qemu-*/debian/patches; \
-    echo qemu-$QEMU_VER-vitastor.patch >> qemu-*/debian/patches/series; \
+    D=$(ls -d qemu*/); \
+    cp /root/vitastor/patches/qemu-$QEMU_VER-vitastor.patch ./qemu-*/debian/patches; \
+    echo qemu-$QEMU_VER-vitastor.patch >> $D/debian/patches/series; \
     cd /root/packages/qemu-$REL/qemu-*/; \
     quilt push -a; \
     quilt add block/vitastor.c; \

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.9.2; \
-    cd vitastor-0.9.2; \
+    cp -r /root/vitastor vitastor-0.9.3; \
+    cd vitastor-0.9.3; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.2.orig.tar.xz vitastor-0.9.2; \
-    cd vitastor-0.9.2; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.3.orig.tar.xz vitastor-0.9.3; \
+    cd vitastor-0.9.3; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config.en.md

@@ -21,7 +21,7 @@ Configuration parameters can be set in 3 places:
   mon, fio and QEMU options, OpenStack/Proxmox/etc configuration. The latter
   doesn't allow to set all variables directly, but it allows to override the
   configuration file and set everything you need inside it.
-- OSD superblocks created by [vitastor-disk](../usage/disk.en.md) contain
+- OSD superblocks created by [vitastor-disk](usage/disk.en.md) contain
   primarily disk layout parameters of specific OSDs. In fact, these parameters
   are automatically passed into the command line of vitastor-osd process, so
   they have the same "status" as command-line parameters.

docs/config.ru.md

@@ -23,7 +23,7 @@
   монитора, опциях fio и QEMU, настроек OpenStack, Proxmox и т.п. Последние,
   как правило, не включают полный набор параметров напрямую, но позволяют
   определить путь к файлу конфигурации и задать любые параметры в нём.
-- В суперблоке OSD, записываемом [vitastor-disk](../usage/disk.ru.md) - параметры,
+- В суперблоке OSD, записываемом [vitastor-disk](usage/disk.ru.md) - параметры,
   связанные с дисковым форматом и с этим конкретным OSD. На самом деле,
   при запуске OSD эти параметры автоматически передаются в командную строку
   процесса vitastor-osd, то есть по "статусу" они эквивалентны параметрам

docs/config/src/include.js

@@ -0,0 +1,145 @@
+#!/usr/bin/nodejs
+
+const fsp = require('fs').promises;
+
+run(process.argv).catch(console.error);
+
+async function run(argv)
+{
+    if (argv.length < 3)
+    {
+        console.log('Markdown preprocessor\nUSAGE: ./include.js file.md');
+        return;
+    }
+    const index_file = await fsp.realpath(argv[2]);
+    const re = /(\{\{[\s\S]*?\}\}|\[[^\]]+\]\([^\)]+\)|(?:^|\n)#[^\n]+)/;
+    let text = await fsp.readFile(index_file, { encoding: 'utf-8' });
+    text = text.split(re);
+    let included = {};
+    let heading = 0, heading_name = '', m;
+    for (let i = 0; i < text.length; i++)
+    {
+        if (text[i].substr(0, 2) == '{{')
+        {
+            // Inclusion
+            let incfile = text[i].substr(2, text[i].length-4);
+            let section = null;
+            let indent = heading;
+            incfile = incfile.replace(/\s*\|\s*indent\s*=\s*(-?\d+)\s*$/, (m, m1) => { indent = parseInt(m1); return ''; });
+            incfile = incfile.replace(/\s*#\s*([^#]+)$/, (m, m1) => { section = m1; return ''; });
+            let inc_heading = section;
+            incfile = rel2abs(index_file, incfile);
+            let inc = await fsp.readFile(incfile, { encoding: 'utf-8' });
+            inc = inc.trim().replace(/^[\s\S]+?\n#/, '#'); // remove until the first header
+            inc = inc.split(re);
+            const indent_str = new Array(indent+1).join('#');
+            let section_start = -1, section_end = -1;
+            for (let j = 0; j < inc.length; j++)
+            {
+                if ((m = /^(\n?)(#+\s*)([\s\S]+)$/.exec(inc[j])))
+                {
+                    if (!inc_heading)
+                    {
+                        inc_heading = m[3].trim();
+                    }
+                    if (section)
+                    {
+                        if (m[3].trim() == section)
+                            section_start = j;
+                        else if (section_start >= 0)
+                        {
+                            section_end = j;
+                            break;
+                        }
+                    }
+                    inc[j] = m[1] + indent_str + m[2] + m[3];
+                }
+                else if ((m = /^(\[[^\]]+\]\()([^\)]+)(\))$/.exec(inc[j])) && !/^https?:(\/\/)|^#/.exec(m[2]))
+                {
+                    const abs_m2 = rel2abs(incfile, m[2]);
+                    const rel_m = abs2rel(__filename, abs_m2);
+                    if (rel_m.substr(0, 9) == '../../../') // outside docs
+                        inc[j] = m[1] + 'https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/'+rel2abs('docs/config/src/include.js', rel_m) + m[3];
+                    else
+                        inc[j] = m[1] + abs_m2 + m[3];
+                }
+            }
+            if (section)
+            {
+                inc = section_start >= 0 ? inc.slice(section_start, section_end < 0 ? inc.length : section_end) : [];
+            }
+            if (inc.length)
+            {
+                if (!inc_heading)
+                    inc_heading = heading_name||'';
+                included[incfile+(section ? '#'+section : '')] = '#'+inc_heading.toLowerCase().replace(/\P{L}+/ug, '-').replace(/^-|-$/g, '');
+                inc[0] = inc[0].replace(/^\s+/, '');
+                inc[inc.length-1] = inc[inc.length-1].replace(/\s+$/, '');
+            }
+            text.splice(i, 1, ...inc);
+            i = i + inc.length - 1;
+        }
+        else if ((m = /^\n?(#+)\s*([\s\S]+)$/.exec(text[i])))
+        {
+            // Heading
+            heading = m[1].length;
+            heading_name = m[2].trim();
+        }
+    }
+    for (let i = 0; i < text.length; i++)
+    {
+        if ((m = /^(\[[^\]]+\]\()([^\)]+)(\))$/.exec(text[i])) && !/^https?:(\/\/)|^#/.exec(m[2]))
+        {
+            const p = m[2].indexOf('#');
+            if (included[m[2]])
+            {
+                text[i] = m[1]+included[m[2]]+m[3];
+            }
+            else if (p >= 0 && included[m[2].substr(0, p)])
+            {
+                text[i] = m[1]+m[2].substr(p)+m[3];
+            }
+        }
+    }
+    console.log(text.join(''));
+}
+
+function rel2abs(ref, rel)
+{
+    rel = [ ...ref.replace(/^(.*)\/[^\/]+$/, '$1').split(/\/+/), ...rel.split(/\/+/) ];
+    return killdots(rel).join('/');
+}
+
+function abs2rel(ref, abs)
+{
+    ref = ref.split(/\/+/);
+    abs = abs.split(/\/+/);
+    while (ref.length > 1 && ref[0] == abs[0])
+    {
+        ref.shift();
+        abs.shift();
+    }
+    for (let i = 1; i < ref.length; i++)
+    {
+        abs.unshift('..');
+    }
+    return killdots(abs).join('/');
+}
+
+function killdots(rel)
+{
+    for (let i = 0; i < rel.length; i++)
+    {
+        if (rel[i] == '.')
+        {
+            rel.splice(i, 1);
+            i--;
+        }
+        else if (i >= 1 && rel[i] == '..' && rel[i-1] != '..')
+        {
+            rel.splice(i-1, 2);
+            i -= 2;
+        }
+    }
+    return rel;
+}

docs/config/src/included.en.md

@@ -0,0 +1,65 @@
+# Vitastor
+
+{{../../../README.md#The Idea}}
+
+{{../../../README.md#Talks and presentations}}
+
+{{../../intro/features.en.md}}
+
+{{../../intro/quickstart.en.md}}
+
+{{../../intro/architecture.en.md}}
+
+## Installation
+
+{{../../installation/packages.en.md}}
+
+{{../../installation/proxmox.en.md}}
+
+{{../../installation/openstack.en.md}}
+
+{{../../installation/kubernetes.en.md}}
+
+{{../../installation/source.en.md}}
+
+{{../../config.en.md|indent=1}}
+
+{{../../config/common.en.md|indent=2}}
+
+{{../../config/network.en.md|indent=2}}
+
+{{../../config/layout-cluster.en.md|indent=2}}
+
+{{../../config/layout-osd.en.md|indent=2}}
+
+{{../../config/osd.en.md|indent=2}}
+
+{{../../config/monitor.en.md|indent=2}}
+
+{{../../config/pool.en.md|indent=2}}
+
+{{../../config/inode.en.md|indent=2}}
+
+## Usage
+
+{{../../usage/cli.en.md}}
+
+{{../../usage/disk.en.md}}
+
+{{../../usage/fio.en.md}}
+
+{{../../usage/nbd.en.md}}
+
+{{../../usage/qemu.en.md}}
+
+{{../../usage/nfs.en.md}}
+
+## Performance
+
+{{../../performance/understanding.en.md}}
+
+{{../../performance/theoretical.en.md}}
+
+{{../../performance/comparison1.en.md}}
+
+{{../../intro/author.en.md|indent=1}}

docs/config/src/included.ru.md

@@ -0,0 +1,65 @@
+# Vitastor
+
+{{../../../README-ru.md#Идея|indent=0}}
+
+{{../../../README-ru.md#Презентации и записи докладов|indent=0}}
+
+{{../../intro/features.ru.md}}
+
+{{../../intro/quickstart.ru.md}}
+
+{{../../intro/architecture.ru.md}}
+
+## Установка
+
+{{../../installation/packages.ru.md}}
+
+{{../../installation/proxmox.ru.md}}
+
+{{../../installation/openstack.ru.md}}
+
+{{../../installation/kubernetes.ru.md}}
+
+{{../../installation/source.ru.md}}
+
+{{../../config.ru.md|indent=1}}
+
+{{../../config/common.ru.md|indent=2}}
+
+{{../../config/network.ru.md|indent=2}}
+
+{{../../config/layout-cluster.ru.md|indent=2}}
+
+{{../../config/layout-osd.ru.md|indent=2}}
+
+{{../../config/osd.ru.md|indent=2}}
+
+{{../../config/monitor.ru.md|indent=2}}
+
+{{../../config/pool.ru.md|indent=2}}
+
+{{../../config/inode.ru.md|indent=2}}
+
+## Использование
+
+{{../../usage/cli.ru.md}}
+
+{{../../usage/disk.ru.md}}
+
+{{../../usage/fio.ru.md}}
+
+{{../../usage/nbd.ru.md}}
+
+{{../../usage/qemu.ru.md}}
+
+{{../../usage/nfs.ru.md}}
+
+## Производительность
+
+{{../../performance/understanding.ru.md}}
+
+{{../../performance/theoretical.ru.md}}
+
+{{../../performance/comparison1.ru.md}}
+
+{{../../intro/author.ru.md|indent=1}}

docs/installation/kubernetes.ru.md

@@ -8,13 +8,13 @@
 
 У Vitastor есть CSI-плагин для Kubernetes, поддерживающий RWO, а также блочные RWX, тома.
 
-Для установки возьмите манифесты из директории [csi/deploy/](../csi/deploy/), поместите
-вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../csi/deploy/001-csi-config-map.yaml),
-настройте StorageClass в [csi/deploy/009-storage-class.yaml](../csi/deploy/009-storage-class.yaml)
+Для установки возьмите манифесты из директории [csi/deploy/](../../csi/deploy/), поместите
+вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
+настройте StorageClass в [csi/deploy/009-storage-class.yaml](../../csi/deploy/009-storage-class.yaml)
 и примените все `NNN-*.yaml` к вашей инсталляции Kubernetes.
 
 ```
 for i in ./???-*.yaml; do kubectl apply -f $i; done
 ```
 
-После этого вы сможете создавать PersistentVolume. Пример смотрите в файле [csi/deploy/example-pvc.yaml](../csi/deploy/example-pvc.yaml).
+После этого вы сможете создавать PersistentVolume. Пример смотрите в файле [csi/deploy/example-pvc.yaml](../../csi/deploy/example-pvc.yaml).

docs/installation/openstack.en.md

@@ -36,5 +36,5 @@ vitastor_pool_id = 1
 image_upload_use_cinder_backend = True
 ```
 
-To put Glance images in Vitastor, use [https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html](volume-backed images),
+To put Glance images in Vitastor, use [volume-backed images](https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html),
 although the support has not been verified yet.

docs/installation/openstack.ru.md

@@ -36,5 +36,5 @@ image_upload_use_cinder_backend = True
 ```
 
 Чтобы помещать в Vitastor Glance-образы, нужно использовать
-[https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html](образы на основе томов Cinder),
+[образы на основе томов Cinder](https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html),
 однако, поддержка этой функции ещё не проверялась.

docs/installation/packages.en.md

@@ -11,7 +11,8 @@
 - Trust Vitastor package signing key:
   `wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg`
 - Add Vitastor package repository to your /etc/apt/sources.list:
-  - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
+  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
+  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
 - For Debian 10 (Buster) also enable backports repository:
   `deb http://deb.debian.org/debian buster-backports main`

docs/installation/packages.ru.md

@@ -11,7 +11,8 @@
 - Добавьте ключ репозитория Vitastor:
   `wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg`
 - Добавьте репозиторий Vitastor в /etc/apt/sources.list:
-  - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
+  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
+  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
 - Для Debian 10 (Buster) также включите репозиторий backports:
   `deb http://deb.debian.org/debian buster-backports main`

docs/installation/proxmox.en.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-7.4 are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-8.0 are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
-  buster for 6.4, bullseye for 7.4, pve7.1 for 7.1, pve7.2 for 7.2, pve7.3 for 7.3
+  bookworm for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
 - Install vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* or see note) packages from Vitastor repository
 - Define storage in `/etc/pve/storage.cfg` (see below)
 - Block network access from VMs to Vitastor network (to OSDs and etcd),
@@ -35,5 +35,5 @@ vitastor: vitastor
     vitastor_nbd 0
 ```
 
-\* Note: you can also manually copy [patches/VitastorPlugin.pm](patches/VitastorPlugin.pm) to Proxmox hosts
+\* Note: you can also manually copy [patches/VitastorPlugin.pm](../../patches/VitastorPlugin.pm) to Proxmox hosts
 as `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm` instead of installing pve-storage-vitastor.

docs/installation/proxmox.ru.md

@@ -4,12 +4,12 @@
 
 [Read in English](proxmox.en.md)
 
-# Proxmox
+# Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-7.4):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.0):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
-  buster для 6.4, bullseye для 7.4, pve7.1 для 7.1, pve7.2 для 7.2, pve7.3 для 7.3
+  bookworm для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
 - Установите пакеты vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* или см. сноску) из репозитория Vitastor
 - Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже)
 - Обязательно заблокируйте доступ от виртуальных машин к сети Vitastor (OSD и etcd), т.к. Vitastor (пока) не поддерживает аутентификацию
@@ -35,5 +35,5 @@ vitastor: vitastor
 ```
 
 \* Примечание: вместо установки пакета pve-storage-vitastor вы можете вручную скопировать файл
-[patches/VitastorPlugin.pm](patches/VitastorPlugin.pm) на хосты Proxmox как
+[patches/VitastorPlugin.pm](../../patches/VitastorPlugin.pm) на хосты Proxmox как
 `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm`.

docs/intro/architecture.en.md

@@ -44,7 +44,7 @@
   depends linearly on drive capacity and data store block size which is 128 KB by default.
   With 128 KB blocks metadata takes around 512 MB per 1 TB (which is still less than Ceph wants).
   Journal is also kept in memory by default, but in SSD-only clusters it's only 32 MB, and in SSD+HDD
-  clusters, where it's beneficial to increase it, [inmemory_journal](docs/config/osd.en.md#inmemory_journal) can be disabled.
+  clusters, where it's beneficial to increase it, [inmemory_journal](../config/osd.en.md#inmemory_journal) can be disabled.
 - Vitastor storage layer doesn't have internal copy-on-write or redirect-write. I know that maybe
   it's possible to create a good copy-on-write storage, but it's much harder and makes performance
   less deterministic, so CoW isn't used in Vitastor.

docs/intro/architecture.ru.md

@@ -156,7 +156,7 @@
   блока хранилища (block_size, по умолчанию 128 КБ). С 128 КБ блоком потребление памяти
   составляет примерно 512 МБ на 1 ТБ данных. Журналы по умолчанию тоже хранятся в памяти,
   но в SSD-кластерах нужный размер журнала составляет всего 32 МБ, а в гибридных (SSD+HDD)
-  кластерах, в которых есть смысл делать журналы больше, можно отключить [inmemory_journal](../docs/config/osd.ru.md#inmemory_journal).
+  кластерах, в которых есть смысл делать журналы больше, можно отключить [inmemory_journal](../config/osd.ru.md#inmemory_journal).
 - В Vitastor нет внутреннего copy-on-write. Я считаю, что реализация CoW-хранилища гораздо сложнее,
   поэтому сложнее добиться устойчиво хороших результатов. Возможно, в один прекрасный день
   я придумаю красивый алгоритм для CoW-хранилища, но пока нет — внутреннего CoW в Vitastor не будет.

docs/intro/features.en.md

@@ -35,7 +35,7 @@
 
 - [Debian and CentOS packages](../installation/packages.en.md)
 - [Image management CLI (vitastor-cli)](../usage/cli.en.md)
-- [Disk management CLI (vitastor-disk)](docs/usage/disk.en.md)
+- [Disk management CLI (vitastor-disk)](../usage/disk.en.md)
 - Generic user-space client library
 - [Native QEMU driver](../usage/qemu.en.md)
 - [Loadable fio engine for benchmarks](../usage/fio.en.md)

docs/intro/features.ru.md

@@ -13,7 +13,7 @@
 ## Серверные функции
 
 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
-- [Производительность](../comparison1.ru.md) ;-D
+- [Производительность](../performance/comparison1.ru.md) ;-D
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
   Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd
@@ -37,7 +37,7 @@
 
 - [Пакеты для Debian и CentOS](../installation/packages.ru.md)
 - [Консольный интерфейс управления образами (vitastor-cli)](../usage/cli.ru.md)
-- [Инструмент управления дисками (vitastor-disk)](docs/usage/disk.ru.md)
+- [Инструмент управления дисками (vitastor-disk)](../usage/disk.ru.md)
 - Общая пользовательская клиентская библиотека для работы с кластером
 - [Драйвер диска для QEMU](../usage/qemu.ru.md)
 - [Драйвер диска для утилиты тестирования производительности fio](../usage/fio.ru.md)

docs/usage/nfs.ru.md

@@ -29,7 +29,7 @@ vitastor-nfs [--etcd_address ADDR] [ДРУГИЕ ОПЦИИ]
 --bind <IP>       принимать соединения по адресу <IP> (по умолчанию 0.0.0.0 - на всех)
 --nfspath <PATH>  установить путь NFS-экспорта в <PATH> (по умолчанию /)
 --port <PORT>     использовать порт <PORT> для NFS-сервисов (по умолчанию 2049)
---pool <POOL>     исползовать пул <POOL> для новых образов (обязательно, если пул в кластере не один)
+--pool <POOL>     использовать пул <POOL> для новых образов (обязательно, если пул в кластере не один)
 --foreground 1    не уходить в фон после запуска
 ```
 

mon/make-etcd

@@ -63,7 +63,7 @@ Wants=network-online.target local-fs.target time-sync.target
 
 [Service]
 Restart=always
-ExecStart=/usr/local/bin/etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
+ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
     --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
     --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
     --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.9.2'
+VERSION = '0.9.3'
 
 LOG = logging.getLogger(__name__)
 

patches/libvirt-9.0-vitastor.diff

@@ -0,0 +1,644 @@
+commit e6f935157944279c2c0634915c3c00feeec748c9
+Author: Vitaliy Filippov <vitalif@yourcmc.ru>
+Date:   Mon Jun 19 00:58:19 2023 +0300
+
+    Add Vitastor support
+
+diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
+index aaad4a3..5f5daa8 100644
+--- a/include/libvirt/libvirt-storage.h
++++ b/include/libvirt/libvirt-storage.h
+@@ -326,6 +326,7 @@ typedef enum {
+     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
+     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
+     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
++    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
+ } virConnectListAllStoragePoolsFlags;
+ 
+ int                     virConnectListAllStoragePools(virConnectPtr conn,
+diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
+index 45965fa..b7c23d3 100644
+--- a/src/conf/domain_conf.c
++++ b/src/conf/domain_conf.c
+@@ -7103,7 +7103,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
+     src->configFile = virXPathString("string(./config/@file)", ctxt);
+ 
+     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
+-        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
++        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
++        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
+         src->query = virXMLPropString(node, "query");
+ 
+     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
+@@ -30121,6 +30122,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
+ 
+     case VIR_STORAGE_POOL_MPATH:
+     case VIR_STORAGE_POOL_RBD:
++    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_SHEEPDOG:
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_LAST:
+diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
+index 5a9bf20..05058b8 100644
+--- a/src/conf/domain_validate.c
++++ b/src/conf/domain_validate.c
+@@ -494,6 +494,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+             break;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+@@ -541,7 +542,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+         }
+     }
+ 
+-    /* internal snapshots and config files are currently supported only with rbd: */
++    /* internal snapshots are currently supported only with rbd: */
+     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
+         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
+         if (src->snapshot) {
+@@ -550,11 +551,15 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+                              "only with 'rbd' disks"));
+             return -1;
+         }
+-
++    }
++    /* config files are currently supported only with rbd and vitastor: */
++    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
++        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
++        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
+         if (src->configFile) {
+             virReportError(VIR_ERR_XML_ERROR, "%s",
+                            _("<config> element is currently supported "
+-                             "only with 'rbd' disks"));
++                             "only with 'rbd' and 'vitastor' disks"));
+             return -1;
+         }
+     }
+diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
+index 6cb0a20..8bf7de9 100644
+--- a/src/conf/schemas/domaincommon.rng
++++ b/src/conf/schemas/domaincommon.rng
+@@ -1972,6 +1972,35 @@
+     </element>
+   </define>
+ 
++  <define name="diskSourceNetworkProtocolVitastor">
++    <element name="source">
++      <interleave>
++        <attribute name="protocol">
++          <value>vitastor</value>
++        </attribute>
++        <ref name="diskSourceCommon"/>
++        <optional>
++          <attribute name="name"/>
++        </optional>
++        <optional>
++          <attribute name="query"/>
++        </optional>
++        <zeroOrMore>
++          <ref name="diskSourceNetworkHost"/>
++        </zeroOrMore>
++        <optional>
++          <element name="config">
++            <attribute name="file">
++              <ref name="absFilePath"/>
++            </attribute>
++            <empty/>
++          </element>
++        </optional>
++        <empty/>
++      </interleave>
++    </element>
++  </define>
++
+   <define name="diskSourceNetworkProtocolISCSI">
+     <element name="source">
+       <attribute name="protocol">
+@@ -2264,6 +2293,7 @@
+       <ref name="diskSourceNetworkProtocolSimple"/>
+       <ref name="diskSourceNetworkProtocolVxHS"/>
+       <ref name="diskSourceNetworkProtocolNFS"/>
++      <ref name="diskSourceNetworkProtocolVitastor"/>
+     </choice>
+   </define>
+ 
+diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
+index f5a9636..8339bc4 100644
+--- a/src/conf/storage_conf.c
++++ b/src/conf/storage_conf.c
+@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
+               "logical", "disk", "iscsi",
+               "iscsi-direct", "scsi", "mpath",
+               "rbd", "sheepdog", "gluster",
+-              "zfs", "vstorage",
++              "zfs", "vstorage", "vitastor",
+ );
+ 
+ VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
+@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
+           .formatToString = virStorageFileFormatTypeToString,
+       }
+     },
++    {.poolType = VIR_STORAGE_POOL_VITASTOR,
++     .poolOptions = {
++         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
++                   VIR_STORAGE_POOL_SOURCE_NETWORK |
++                   VIR_STORAGE_POOL_SOURCE_NAME),
++      },
++      .volOptions = {
++          .defaultFormat = VIR_STORAGE_FILE_RAW,
++          .formatFromString = virStorageVolumeFormatFromString,
++          .formatToString = virStorageFileFormatTypeToString,
++      }
++    },
+     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
+      .poolOptions = {
+          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
+@@ -542,6 +554,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
+                        _("element 'name' is mandatory for RBD pool"));
+         return -1;
+     }
++    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
++        virReportError(VIR_ERR_XML_ERROR, "%s",
++                       _("element 'name' is mandatory for Vitastor pool"));
++        return -1;
++    }
+ 
+     if (options->formatFromString) {
+         g_autofree char *format = NULL;
+@@ -1132,6 +1149,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
+     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
+      * files, so they don't have a target */
+     if (def->type != VIR_STORAGE_POOL_RBD &&
++        def->type != VIR_STORAGE_POOL_VITASTOR &&
+         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
+         def->type != VIR_STORAGE_POOL_GLUSTER &&
+         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
+diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
+index fc67957..720c07e 100644
+--- a/src/conf/storage_conf.h
++++ b/src/conf/storage_conf.h
+@@ -103,6 +103,7 @@ typedef enum {
+     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
+     VIR_STORAGE_POOL_ZFS,      /* ZFS */
+     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
++    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
+ 
+     VIR_STORAGE_POOL_LAST,
+ } virStoragePoolType;
+@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
+                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
++                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
+diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
+index cecd7e8..d7b79a4 100644
+--- a/src/conf/storage_source_conf.c
++++ b/src/conf/storage_source_conf.c
+@@ -87,6 +87,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
+               "ssh",
+               "vxhs",
+               "nfs",
++              "vitastor",
+ );
+ 
+ 
+@@ -1286,6 +1287,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+             return 24007;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+             /* we don't provide a default for RBD */
+             return 0;
+diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
+index 14a6825..eb4acac 100644
+--- a/src/conf/storage_source_conf.h
++++ b/src/conf/storage_source_conf.h
+@@ -128,6 +128,7 @@ typedef enum {
+     VIR_STORAGE_NET_PROTOCOL_SSH,
+     VIR_STORAGE_NET_PROTOCOL_VXHS,
+     VIR_STORAGE_NET_PROTOCOL_NFS,
++    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
+ 
+     VIR_STORAGE_NET_PROTOCOL_LAST
+ } virStorageNetProtocol;
+diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
+index e6c187e..035b423 100644
+--- a/src/conf/virstorageobj.c
++++ b/src/conf/virstorageobj.c
+@@ -1433,6 +1433,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
+             return 1;
+         break;
+ 
++    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_ISCSI_DIRECT:
+     case VIR_STORAGE_POOL_RBD:
+     case VIR_STORAGE_POOL_LAST:
+@@ -1918,6 +1919,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
+                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
+                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
++              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
++               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
+                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
+diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
+index 8490034..ab2cdaa 100644
+--- a/src/libvirt-storage.c
++++ b/src/libvirt-storage.c
+@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
+  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
+  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
+  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
++ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
+  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
+  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
+  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
+diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
+index 17ac880..59711b5 100644
+--- a/src/libxl/libxl_conf.c
++++ b/src/libxl/libxl_conf.c
+@@ -970,6 +970,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
+     case VIR_STORAGE_NET_PROTOCOL_SSH:
+     case VIR_STORAGE_NET_PROTOCOL_VXHS:
+     case VIR_STORAGE_NET_PROTOCOL_NFS:
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+         virReportError(VIR_ERR_NO_SUPPORT,
+diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
+index 6919325..55ffc32 100644
+--- a/src/libxl/xen_xl.c
++++ b/src/libxl/xen_xl.c
+@@ -1445,6 +1445,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
+     case VIR_STORAGE_NET_PROTOCOL_SSH:
+     case VIR_STORAGE_NET_PROTOCOL_VXHS:
+     case VIR_STORAGE_NET_PROTOCOL_NFS:
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+         virReportError(VIR_ERR_NO_SUPPORT,
+diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
+index e865aa1..40162af 100644
+--- a/src/qemu/qemu_block.c
++++ b/src/qemu/qemu_block.c
+@@ -604,6 +604,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
+ }
+ 
+ 
++static virJSONValue *
++qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
++{
++    virJSONValue *ret = NULL;
++    virStorageNetHostDef *host;
++    size_t i;
++    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
++    g_autofree char *etcd = NULL;
++
++    for (i = 0; i < src->nhosts; i++) {
++        host = src->hosts + i;
++        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
++            return NULL;
++        }
++        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
++    }
++    if (src->nhosts > 0) {
++        etcd = virBufferContentAndReset(&buf);
++    }
++
++    if (virJSONValueObjectAdd(&ret,
++                              "S:etcd-host", etcd,
++                              "S:etcd-prefix", src->query,
++                              "S:config-path", src->configFile,
++                              "s:image", src->path,
++                              NULL) < 0)
++        return NULL;
++
++    return ret;
++}
++
++
+ static virJSONValue *
+ qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src)
+ {
+@@ -917,6 +949,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
+                 return NULL;
+             break;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
++            driver = "vitastor";
++            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
++                return NULL;
++            break;
++
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+             driver = "sheepdog";
+             if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src)))
+@@ -1860,6 +1898,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
+ 
+             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+             case VIR_STORAGE_NET_PROTOCOL_RBD:
++            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+             case VIR_STORAGE_NET_PROTOCOL_VXHS:
+             case VIR_STORAGE_NET_PROTOCOL_NFS:
+             case VIR_STORAGE_NET_PROTOCOL_SSH:
+@@ -2242,6 +2281,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
+                 return -1;
+             break;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
++            driver = "vitastor";
++            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
++                return -1;
++            break;
++
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+             driver = "sheepdog";
+             if (!(location = qemuBlockStorageSourceGetSheepdogProps(src)))
+diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
+index 2eb5653..60ee82d 100644
+--- a/src/qemu/qemu_domain.c
++++ b/src/qemu/qemu_domain.c
+@@ -4958,7 +4958,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
+     if (src->query &&
+         (actualType != VIR_STORAGE_TYPE_NETWORK ||
+          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
+-          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
++          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
++          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
+         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                        _("query is supported only with HTTP(S) protocols"));
+         return -1;
+@@ -10129,6 +10130,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
+         break;
+ 
+     case VIR_STORAGE_NET_PROTOCOL_RBD:
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
+index b841680..a6be771 100644
+--- a/src/qemu/qemu_snapshot.c
++++ b/src/qemu/qemu_snapshot.c
+@@ -373,6 +373,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
+         case VIR_STORAGE_NET_PROTOCOL_NONE:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+@@ -578,6 +579,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
+         case VIR_STORAGE_NET_PROTOCOL_NONE:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
+index d90c1c9..e853457 100644
+--- a/src/storage/storage_driver.c
++++ b/src/storage/storage_driver.c
+@@ -1627,6 +1627,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
+ 
+         case VIR_STORAGE_POOL_GLUSTER:
+         case VIR_STORAGE_POOL_RBD:
++        case VIR_STORAGE_POOL_VITASTOR:
+         case VIR_STORAGE_POOL_SHEEPDOG:
+         case VIR_STORAGE_POOL_ZFS:
+         case VIR_STORAGE_POOL_LAST:
+diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
+index e48ae72..2017ccc 100644
+--- a/src/storage_file/storage_source_backingstore.c
++++ b/src/storage_file/storage_source_backingstore.c
+@@ -284,6 +284,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
+ }
+ 
+ 
++static int
++virStorageSourceParseVitastorColonString(const char *colonstr,
++                                         virStorageSource *src)
++{
++    char *p, *e, *next;
++    g_autofree char *options = NULL;
++
++    /* optionally skip the "vitastor:" prefix if provided */
++    if (STRPREFIX(colonstr, "vitastor:"))
++        colonstr += strlen("vitastor:");
++
++    options = g_strdup(colonstr);
++
++    p = options;
++    while (*p) {
++        /* find : delimiter or end of string */
++        for (e = p; *e && *e != ':'; ++e) {
++            if (*e == '\\') {
++                e++;
++                if (*e == '\0')
++                    break;
++            }
++        }
++        if (*e == '\0') {
++            next = e;    /* last kv pair */
++        } else {
++            next = e + 1;
++            *e = '\0';
++        }
++
++        if (STRPREFIX(p, "image=")) {
++            src->path = g_strdup(p + strlen("image="));
++        } else if (STRPREFIX(p, "etcd-prefix=")) {
++            src->query = g_strdup(p + strlen("etcd-prefix="));
++        } else if (STRPREFIX(p, "config-path=")) {
++            src->configFile = g_strdup(p + strlen("config-path="));
++        } else if (STRPREFIX(p, "etcd-host=")) {
++            char *h, *sep;
++
++            h = p + strlen("etcd-host=");
++            while (h < e) {
++                for (sep = h; sep < e; ++sep) {
++                    if (*sep == '\\' && (sep[1] == ',' ||
++                                         sep[1] == ';' ||
++                                         sep[1] == ' ')) {
++                        *sep = '\0';
++                        sep += 2;
++                        break;
++                    }
++                }
++
++                if (virStorageSourceRBDAddHost(src, h) < 0)
++                    return -1;
++
++                h = sep;
++            }
++        }
++
++        p = next;
++    }
++
++    if (!src->path) {
++        return -1;
++    }
++
++    return 0;
++}
++
++
+ static int
+ virStorageSourceParseNBDColonString(const char *nbdstr,
+                                     virStorageSource *src)
+@@ -396,6 +465,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
+             return -1;
+         break;
+ 
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
++        if (virStorageSourceParseVitastorColonString(path, src) < 0)
++            return -1;
++        break;
++
+     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+@@ -984,6 +1058,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
+     return 0;
+ }
+ 
++static int
++virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
++                                         virJSONValue *json,
++                                         const char *jsonstr G_GNUC_UNUSED,
++                                         int opaque G_GNUC_UNUSED)
++{
++    const char *filename;
++    const char *image = virJSONValueObjectGetString(json, "image");
++    const char *conf = virJSONValueObjectGetString(json, "config-path");
++    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
++    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
++    size_t nservers;
++    size_t i;
++
++    src->type = VIR_STORAGE_TYPE_NETWORK;
++    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
++
++    /* legacy syntax passed via 'filename' option */
++    if ((filename = virJSONValueObjectGetString(json, "filename")))
++        return virStorageSourceParseVitastorColonString(filename, src);
++
++    if (!image) {
++        virReportError(VIR_ERR_INVALID_ARG, "%s",
++                       _("missing image name in Vitastor backing volume "
++                         "JSON specification"));
++        return -1;
++    }
++
++    src->path = g_strdup(image);
++    src->configFile = g_strdup(conf);
++    src->query = g_strdup(etcd_prefix);
++
++    if (servers) {
++        nservers = virJSONValueArraySize(servers);
++
++        src->hosts = g_new0(virStorageNetHostDef, nservers);
++        src->nhosts = nservers;
++
++        for (i = 0; i < nservers; i++) {
++            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
++                                                                  virJSONValueArrayGet(servers, i)) < 0)
++                return -1;
++        }
++    }
++
++    return 0;
++}
++
+ static int
+ virStorageSourceParseBackingJSONRaw(virStorageSource *src,
+                                     virJSONValue *json,
+@@ -1162,6 +1284,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
+     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
+     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
+     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
++    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
+     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
+     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
+     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
+diff --git a/src/test/test_driver.c b/src/test/test_driver.c
+index bd6f063..cce34e1 100644
+--- a/src/test/test_driver.c
++++ b/src/test/test_driver.c
+@@ -7338,6 +7338,7 @@ testStorageVolumeTypeForPool(int pooltype)
+     case VIR_STORAGE_POOL_ISCSI_DIRECT:
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_RBD:
++    case VIR_STORAGE_POOL_VITASTOR:
+         return VIR_STORAGE_VOL_NETWORK;
+     case VIR_STORAGE_POOL_LOGICAL:
+     case VIR_STORAGE_POOL_DISK:
+diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+index eee75af..8bd0a57 100644
+--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
++++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+@@ -204,4 +204,11 @@
+       </enum>
+     </volOptions>
+   </pool>
++  <pool type='vitastor' supported='no'>
++    <volOptions>
++      <defaultFormat type='raw'/>
++      <enum name='targetFormatType'>
++      </enum>
++    </volOptions>
++  </pool>
+ </storagepoolCapabilities>
+diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
+index 805950a..852df0d 100644
+--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
++++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
+@@ -204,4 +204,11 @@
+       </enum>
+     </volOptions>
+   </pool>
++  <pool type='vitastor' supported='yes'>
++    <volOptions>
++      <defaultFormat type='raw'/>
++      <enum name='targetFormatType'>
++      </enum>
++    </volOptions>
++  </pool>
+ </storagepoolCapabilities>
+diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
+index e8e40d6..db55fe5 100644
+--- a/tests/storagepoolxml2argvtest.c
++++ b/tests/storagepoolxml2argvtest.c
+@@ -65,6 +65,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_ZFS:
+     case VIR_STORAGE_POOL_VSTORAGE:
++    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_LAST:
+     default:
+         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
+diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
+index 8a98c6a..4b1bbd4 100644
+--- a/tools/virsh-pool.c
++++ b/tools/virsh-pool.c
+@@ -1221,6 +1221,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
+             case VIR_STORAGE_POOL_VSTORAGE:
+                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
+                 break;
++            case VIR_STORAGE_POOL_VITASTOR:
++                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
++                break;
+             case VIR_STORAGE_POOL_LAST:
+                 break;
+             }

patches/pve-qemu-8.0-vitastor.patch

@@ -0,0 +1,190 @@
+diff --git a/block/meson.build b/block/meson.build
+index 382bec0e7d..af6207dbce 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index c44d05a13f..ebedb42843 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1028,6 +1028,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'), kwargs: static_kwargs)
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -1882,6 +1902,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_SDL', sdl.found())
+ config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
+@@ -4020,6 +4041,7 @@ if spice_protocol.found()
+   summary_info += {'  spice server support': spice}
+ endif
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index fc9447d267..c4ac55c283 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -173,6 +173,8 @@ option('lzo', type : 'feature', value : 'auto',
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index c05ad0c07e..f5eb701604 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3308,7 +3308,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4338,6 +4338,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4787,6 +4809,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5187,6 +5210,17 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5385,6 +5419,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
+index 6e8983f39c..1b0b9fcf3e 100755
+--- a/scripts/ci/org.centos/stream/8/x86_64/configure
++++ b/scripts/ci/org.centos/stream/8/x86_64/configure
+@@ -32,7 +32,7 @@
+ --with-git=meson \
+ --with-git-submodules=update \
+ --target-list="x86_64-softmmu" \
+---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
++--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
+ --audio-drv-list="" \
+ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
+ --with-coroutine=ucontext \
+@@ -179,6 +179,7 @@
+ --enable-opengl \
+ --enable-pie \
+ --enable-rbd \
++--enable-vitastor \
+ --enable-rdma \
+ --enable-seccomp \
+ --enable-snappy \
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index 009fab1515..95914e6ebc 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -144,6 +144,7 @@ meson_options_help() {
+   printf "%s\n" '  qed             qed image format support'
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  sdl             SDL user interface'
+@@ -392,6 +393,8 @@ _meson_option_parse() {
+     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-replication) printf "%s" -Dreplication=enabled ;;

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.9.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.2$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.9.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.3$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el7.tar.gz
+Source0:        vitastor-0.9.3.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el8.tar.gz
+Source0:        vitastor-0.9.3.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el9.tar.gz
+Source0:        vitastor-0.9.3.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.9.2")
+add_definitions(-DVERSION="0.9.3")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/cli_merge.cpp

@@ -41,7 +41,7 @@ struct snap_merger_t
     int fsync_interval = 128;
 
     // -- STATE --
-    inode_t target;
+    inode_t target, to_num;
     int target_rank;
     bool inside_continue = false;
     int state = 0;
@@ -98,6 +98,7 @@ struct snap_merger_t
             state = 100;
             return;
         }
+        to_num = to_cfg->num;
         // Check that to_cfg is actually a child of from_cfg and target_cfg is somewhere between them
         std::vector<inode_t> chain_list;
         inode_config_t *cur = to_cfg;
@@ -451,7 +452,7 @@ struct snap_merger_t
     {
         cluster_op_t *op = &rwo->op;
         op->opcode = OSD_OP_READ;
-        op->inode = target;
+        op->inode = to_num;
         op->offset = rwo->offset;
         op->len = target_block_size;
         op->iov.push_back(rwo->buf, target_block_size);
@@ -483,7 +484,7 @@ struct snap_merger_t
                 {
                     // write start->end
                     rwo->todo++;
-                    write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas ? 1+rwo->op.version : 0);
+                    write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas && to_num == target ? 1+rwo->op.version : 0);
                     rwo->start = rwo->end;
                     if (use_cas)
                     {
@@ -502,7 +503,7 @@ struct snap_merger_t
         {
             // write start->end
             rwo->todo++;
-            write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas ? 1+rwo->op.version : 0);
+            write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas && to_num == target ? 1+rwo->op.version : 0);
             rwo->start = rwo->end;
             if (use_cas)
             {
@@ -532,7 +533,7 @@ struct snap_merger_t
                 if (use_cas && subop->retval == -EINTR)
                 {
                     // CAS failure - reread and repeat optimistically
-                    rwo->start = subop->offset - rwo->offset;
+                    rwo->start = rwo->end = 0;
                     rwo_read(rwo);
                     delete subop;
                     return;
@@ -542,7 +543,7 @@ struct snap_merger_t
                 rwo->error_read = false;
             }
             // Increment CAS version
-            rwo->op.version++;
+            rwo->op.version = subop->version;
             if (use_cas)
                 next_write(rwo);
             else

src/cli_rm.cpp

@@ -65,6 +65,9 @@ struct snap_remover_t
     int current_child = 0;
     std::function<bool(cli_result_t &)> cb;
 
+    std::vector<std::string> rebased_images, deleted_images;
+    std::vector<uint64_t> deleted_ids;
+    std::string inverse_child_name, inverse_parent_name;
     cli_result_t result;
 
     bool is_done()
@@ -122,6 +125,7 @@ resume_1:
         {
             if (merge_children[current_child] == inverse_child)
                 continue;
+            rebased_images.push_back(parent->cli->st_cli.inode_config.at(merge_children[current_child]).name);
             start_merge_child(merge_children[current_child], merge_children[current_child]);
             if (state == 100)
                 return;
@@ -134,9 +138,12 @@ resume_2:
             cb = NULL;
             if (result.err)
             {
+                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
+            else if (parent->progress)
+                printf("%s\n", result.text.c_str());
             parent->change_parent(merge_children[current_child], new_parent, &result);
             state = 3;
 resume_3:
@@ -144,6 +151,7 @@ resume_3:
                 return;
             if (result.err)
             {
+                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
@@ -165,9 +173,12 @@ resume_4:
             cb = NULL;
             if (result.err)
             {
+                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
+            else if (parent->progress)
+                printf("%s\n", result.text.c_str());
             // Delete "inverse" child data
             start_delete_source(inverse_child);
             if (state == 100)
@@ -181,9 +192,12 @@ resume_5:
             cb = NULL;
             if (result.err)
             {
+                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
+            else if (parent->progress)
+                printf("%s\n", result.text.c_str());
             // Delete "inverse" child metadata, rename parent over it,
             // and also change parent links of the previous "inverse" child
             rename_inverse_parent();
@@ -199,6 +213,12 @@ resume_6:
         {
             if (chain_list[current_child] == inverse_parent)
                 continue;
+            {
+                auto parent_it = parent->cli->st_cli.inode_config.find(chain_list[current_child]);
+                if (parent_it != parent->cli->st_cli.inode_config.end())
+                    deleted_images.push_back(parent_it->second.name);
+                deleted_ids.push_back(chain_list[current_child]);
+            }
             start_delete_source(chain_list[current_child]);
 resume_7:
             while (!cb(result))
@@ -209,9 +229,12 @@ resume_7:
             cb = NULL;
             if (result.err)
             {
+                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
+            else if (parent->progress)
+                printf("%s\n", result.text.c_str());
             delete_inode_config(chain_list[current_child]);
             if (state == 100)
                 return;
@@ -221,11 +244,26 @@ resume_8:
                 return;
         }
         state = 100;
+        result = (cli_result_t){
+            .text = "",
+            .data = my_result(result.data),
+        };
 resume_100:
         // Done
         return;
     }
 
+    json11::Json my_result(json11::Json src)
+    {
+        auto obj = src.object_items();
+        obj["deleted_ids"] = deleted_ids;
+        obj["deleted_images"] = deleted_images;
+        obj["rebased_images"] = rebased_images;
+        obj["renamed_from"] = inverse_parent_name;
+        obj["renamed_to"] = inverse_child_name;
+        return obj;
+    }
+
     void get_merge_children()
     {
         // Get all children of from..to
@@ -338,7 +376,11 @@ resume_100:
             }
             for (auto inode_result: data["responses"].array_items())
             {
-                auto kv = parent->cli->st_cli.parse_etcd_kv(inode_result["kvs"][0]);
+                if (inode_result["response_range"]["kvs"].array_items().size() == 0)
+                {
+                    continue;
+                }
+                auto kv = parent->cli->st_cli.parse_etcd_kv(inode_result["response_range"]["kvs"][0]);
                 pool_id_t pool_id = 0;
                 inode_t inode = 0;
                 char null_byte = 0;
@@ -377,7 +419,7 @@ resume_100:
             inode_t child = cp.first;
             uint64_t child_used = inode_used[child];
             int rank = cp.second;
-            for (int i = chain_list.size()-rank; i < chain_list.size(); i++)
+            for (int i = chain_list.size()-1-rank; i < chain_list.size(); i++)
             {
                 inode_t parent = chain_list[i];
                 uint64_t parent_used = inode_used[parent];
@@ -413,8 +455,8 @@ resume_100:
         }
         inode_config_t *child_cfg = &child_it->second;
         inode_config_t *target_cfg = &target_it->second;
-        std::string child_name = child_cfg->name;
-        std::string target_name = target_cfg->name;
+        inverse_child_name = child_cfg->name;
+        inverse_parent_name = target_cfg->name;
         std::string child_cfg_key = base64_encode(
             parent->cli->st_cli.etcd_prefix+
             "/config/inode/"+std::to_string(INODE_POOL(inverse_child))+
@@ -425,6 +467,9 @@ resume_100:
             "/config/inode/"+std::to_string(INODE_POOL(inverse_parent))+
             "/"+std::to_string(INODE_NO_POOL(inverse_parent))
         );
+        std::string target_idx_key = base64_encode(
+            parent->cli->st_cli.etcd_prefix+"/index/image/"+inverse_parent_name
+        );
         // Fill new configuration
         inode_config_t new_cfg = *child_cfg;
         new_cfg.num = target_cfg->num;
@@ -449,6 +494,11 @@ resume_100:
                     { "key", child_cfg_key },
                 } },
             },
+            json11::Json::object {
+                { "request_delete_range", json11::Json::object {
+                    { "key", target_idx_key },
+                } },
+            },
             json11::Json::object {
                 { "request_put", json11::Json::object {
                     { "key", target_cfg_key },
@@ -495,12 +545,12 @@ resume_100:
         parent->cli->st_cli.etcd_txn_slow(json11::Json::object {
             { "compare", cmp },
             { "success", txn },
-        }, [this, target_name, child_name](std::string err, json11::Json res)
+        }, [this](std::string err, json11::Json res)
         {
             parent->waiting--;
             if (err != "")
             {
-                result = (cli_result_t){ .err = EIO, .text = "Error renaming "+target_name+" to "+child_name+": "+err };
+                result = (cli_result_t){ .err = EIO, .text = "Error renaming "+inverse_parent_name+" to "+inverse_child_name+": "+err };
                 state = 100;
                 return;
             }
@@ -508,14 +558,14 @@ resume_100:
             {
                 result = (cli_result_t){
                     .err = EAGAIN,
-                    .text = "Parent ("+target_name+"), child ("+child_name+"), or one of its children"
+                    .text = "Parent ("+inverse_parent_name+"), child ("+inverse_child_name+"), or one of its children"
                         " configuration was modified during rename",
                 };
                 state = 100;
                 return;
             }
             if (parent->progress)
-                printf("Layer %s renamed to %s\n", target_name.c_str(), child_name.c_str());
+                printf("Layer %s renamed to %s\n", inverse_parent_name.c_str(), inverse_child_name.c_str());
             parent->ringloop->wakeup();
         });
     }

src/cli_rm_data.cpp

@@ -28,6 +28,7 @@ struct rm_inode_t
     cli_tool_t *parent = NULL;
     inode_list_t *lister = NULL;
     std::vector<rm_pg_t*> lists;
+    std::vector<osd_num_t> inactive_osds;
     uint64_t total_count = 0, total_done = 0, total_prev_pct = 0;
     uint64_t pgs_to_list = 0;
     bool lists_done = false;
@@ -86,6 +87,16 @@ struct rm_inode_t
             state = 100;
             return;
         }
+        inactive_osds = parent->cli->list_inode_get_inactive_osds(lister);
+        if (inactive_osds.size() && !parent->json_output)
+        {
+            fprintf(stderr, "Some data may remain after delete on OSDs which are currently down: ");
+            for (int i = 0; i < inactive_osds.size(); i++)
+            {
+                fprintf(stderr, i > 0 ? ", %lu" : "%lu", inactive_osds[i]);
+            }
+            fprintf(stderr, "\n");
+        }
         pgs_to_list = parent->cli->list_pg_count(lister);
         parent->cli->list_inode_next(lister, parent->parallel_osds);
     }
@@ -167,16 +178,33 @@ struct rm_inode_t
         }
         if (parent->progress && total_count > 0 && total_done*1000/total_count != total_prev_pct)
         {
-            printf("\rRemoved %lu/%lu objects, %lu more PGs to list...", total_done, total_count, pgs_to_list);
+            fprintf(stderr, "\rRemoved %lu/%lu objects, %lu more PGs to list...", total_done, total_count, pgs_to_list);
             total_prev_pct = total_done*1000/total_count;
         }
         if (lists_done && !lists.size())
         {
+            if (parent->progress && total_count > 0)
+            {
+                fprintf(stderr, "\n");
+            }
+            if (parent->progress && (total_done < total_count || inactive_osds.size() > 0))
+            {
+                fprintf(
+                    stderr, "Warning: Pool:%u,ID:%lu inode data may not have been fully removed.\n"
+                    " Use `vitastor-cli rm-data --pool %u --inode %lu` if you encounter it in listings.\n",
+                    pool_id, INODE_NO_POOL(inode), pool_id, INODE_NO_POOL(inode)
+                );
+            }
             result = (cli_result_t){
                 .err = error_count > 0 ? EIO : 0,
                 .text = error_count > 0 ? "Some blocks were not removed" : (
                     "Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
                     std::to_string(pool_id)+" removed"),
+                .data = json11::Json::object {
+                    { "removed_objects", total_done },
+                    { "total_objects", total_count },
+                    { "inactive_osds", inactive_osds },
+                },
             };
             state = 100;
         }

src/cluster_client.cpp

@@ -1209,6 +1209,10 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             copy_part_bitmap(op, part);
             op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }
+        else if (op->opcode == OSD_OP_WRITE)
+        {
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+        }
         if (op->inflight_count == 0)
         {
             if (op->opcode == OSD_OP_SYNC)

src/cluster_client.h

@@ -130,6 +130,7 @@ public:
     inode_list_t *list_inode_start(inode_t inode,
         std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback);
     int list_pg_count(inode_list_t *lst);
+    const std::vector<osd_num_t> & list_inode_get_inactive_osds(inode_list_t *lst);
     void list_inode_next(inode_list_t *lst, int next_pgs);
     //inline uint32_t get_bs_bitmap_granularity() { return st_cli.global_bitmap_granularity; }
     //inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }

src/cluster_client_list.cpp

@@ -36,6 +36,7 @@ struct inode_list_t
     inode_t inode = 0;
     int done_pgs = 0;
     int want = 0;
+    std::vector<osd_num_t> inactive_osds;
     std::vector<inode_list_pg_t*> pgs;
     std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback;
 };
@@ -60,6 +61,7 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
     lst->inode = inode;
     lst->callback = callback;
     auto pool_cfg = st_cli.pool_config[pool_id];
+    std::set<osd_num_t> inactive_osd_set;
     for (auto & pg_item: pool_cfg.pg_config)
     {
         auto & pg = pg_item.second;
@@ -106,11 +108,18 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
             }
             for (osd_num_t peer_osd: all_peers)
             {
-                r->list_osds.push_back((inode_list_osd_t){
-                    .pg = r,
-                    .osd_num = peer_osd,
-                    .sent = false,
-                });
+                if (st_cli.peer_states.find(peer_osd) != st_cli.peer_states.end())
+                {
+                    r->list_osds.push_back((inode_list_osd_t){
+                        .pg = r,
+                        .osd_num = peer_osd,
+                        .sent = false,
+                    });
+                }
+                else
+                {
+                    inactive_osd_set.insert(peer_osd);
+                }
             }
         }
         else
@@ -132,6 +141,7 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
     {
         lst->pgs[i]->pos = i;
     }
+    lst->inactive_osds.insert(lst->inactive_osds.end(), inactive_osd_set.begin(), inactive_osd_set.end());
     lists.push_back(lst);
     return lst;
 }
@@ -141,6 +151,11 @@ int cluster_client_t::list_pg_count(inode_list_t *lst)
     return lst->pgs.size();
 }
 
+const std::vector<osd_num_t> & cluster_client_t::list_inode_get_inactive_osds(inode_list_t *lst)
+{
+    return lst->inactive_osds;
+}
+
 void cluster_client_t::list_inode_next(inode_list_t *lst, int next_pgs)
 {
     if (next_pgs >= 0)

src/msgr_op.cpp

@@ -9,6 +9,10 @@ osd_op_t::~osd_op_t()
 {
     assert(!bs_op);
     assert(!op_data);
+    if (bitmap_buf)
+    {
+        free(bitmap_buf);
+    }
     if (rmw_buf)
     {
         free(rmw_buf);

src/msgr_op.h

@@ -165,6 +165,7 @@ struct osd_op_t
     void *bitmap = NULL;
     unsigned bitmap_len = 0;
     unsigned bmp_data = 0;
+    void *bitmap_buf = NULL;
     void *rmw_buf = NULL;
     osd_primary_op_data_t* op_data = NULL;
     std::function<void(osd_op_t*)> callback;

src/msgr_receive.cpp

@@ -369,7 +369,7 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
         op->buf = malloc_or_die(op->reply.hdr.retval);
         cl->recv_list.push_back(op->buf, op->reply.hdr.retval);
     }
-    else if (op->reply.hdr.opcode == OSD_OP_DESCRIBE && op->reply.hdr.retval > 0)
+    else if (op->reply.hdr.opcode == OSD_OP_DESCRIBE && op->reply.describe.result_bytes > 0)
     {
         delete cl->read_op;
         cl->read_op = op;

src/msgr_send.cpp

@@ -84,9 +84,12 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
     {
         for (int i = 0; i < cur_op->iov.count; i++)
         {
-            assert(cur_op->iov.buf[i].iov_base);
-            to_send_list.push_back(cur_op->iov.buf[i]);
-            to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = 0 });
+            if (cur_op->iov.buf[i].iov_len > 0)
+            {
+                assert(cur_op->iov.buf[i].iov_base);
+                to_send_list.push_back(cur_op->iov.buf[i]);
+                to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = 0 });
+            }
         }
     }
     if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)

src/nbd_proxy.cpp

@@ -149,7 +149,7 @@ public:
             "  --dev_num N\n"
             "    Use the specified device /dev/nbdN instead of automatic selection.\n"
             "  --foreground 1\n"
-            "    Stay in foreground, do not daemonize.n",
+            "    Stay in foreground, do not daemonize.\n",
             exe_name, exe_name, exe_name
         );
         exit(0);

src/osd_ops.h

@@ -220,7 +220,7 @@ struct __attribute__((__packed__)) osd_reply_rw_t
     // for reads: bitmap length
     uint32_t bitmap_len;
     uint32_t pad0;
-    // for reads: object version
+    // for reads and writes: object version
     uint64_t version;
 };
 

src/osd_primary.cpp

@@ -87,8 +87,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         // - op_data
         1, sizeof(osd_primary_op_data_t) +
         // - stripes
-        // - resulting bitmap buffers
-        stripe_count * (clean_entry_bitmap_size + sizeof(osd_rmw_stripe_t)) +
+        stripe_count * sizeof(osd_rmw_stripe_t) +
         chain_size * (
             // - copy of the chain
             sizeof(inode_t) +
@@ -110,11 +109,12 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     op_data->pg_size = pg_it->second.pg_size;
     cur_op->op_data = op_data;
     split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
-    // Allocate bitmaps along with stripes to avoid extra allocations and fragmentation
+    // Resulting bitmaps have to survive op_data and be freed with the op itself
+    assert(!cur_op->bitmap_buf);
+    cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count);
     for (int i = 0; i < stripe_count; i++)
     {
-        op_data->stripes[i].bmp_buf = data_buf;
-        data_buf = (uint8_t*)data_buf + clean_entry_bitmap_size;
+        op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
     }
     op_data->chain_size = chain_size;
     if (chain_size > 0)
@@ -129,16 +129,19 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         data_buf = (uint8_t*)data_buf + chain_size * (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 0 : pg_it->second.pg_size);
         // Copy chain
         int chain_num = 0;
-        op_data->read_chain[chain_num++] = cur_op->req.rw.inode;
+        op_data->read_chain[chain_num] = cur_op->req.rw.inode;
+        op_data->chain_states[chain_num] = NULL;
+        chain_num++;
         auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode);
         while (inode_it != st_cli.inode_config.end() && inode_it->second.parent_id &&
             INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id &&
             // Check for loops
             inode_it->second.parent_id != cur_op->req.rw.inode)
         {
-            op_data->read_chain[chain_num++] = inode_it->second.parent_id;
-            op_data->chain_states[chain_num++] = NULL;
+            op_data->read_chain[chain_num] = inode_it->second.parent_id;
+            op_data->chain_states[chain_num] = NULL;
             inode_it = st_cli.inode_config.find(inode_it->second.parent_id);
+            chain_num++;
         }
     }
     pg_it->second.inflight++;
@@ -644,12 +647,6 @@ void osd_t::continue_primary_del(osd_op_t *cur_op)
     else if (op_data->st == 4) goto resume_4;
     else if (op_data->st == 5) goto resume_5;
     assert(op_data->st == 0);
-    // Delete is forbidden even in active PGs if they're also degraded or have previous dead OSDs
-    if (pg.state & (PG_DEGRADED | PG_LEFT_ON_DEAD))
-    {
-        finish_op(cur_op, -EBUSY);
-        return;
-    }
     if (!check_write_queue(cur_op, pg))
     {
         return;

src/osd_primary_write.cpp

@@ -83,11 +83,13 @@ retry_1:
             // Object is degraded/misplaced and will be moved to <write_osd_set>
             op_data->stripes[0].read_start = 0;
             op_data->stripes[0].read_end = bs_block_size;
+            assert(!cur_op->rmw_buf);
             cur_op->rmw_buf = op_data->stripes[0].read_buf = memalign_or_die(MEM_ALIGNMENT, bs_block_size);
         }
     }
     else
     {
+        assert(!cur_op->rmw_buf);
         cur_op->rmw_buf = calc_rmw(cur_op->buf, op_data->stripes, op_data->prev_set,
             pg.pg_size, op_data->pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
         if (!cur_op->rmw_buf)

src/qemu_driver.c

@@ -35,6 +35,11 @@
 #define qdict_put_str(options, name, value) qdict_put_obj(options, name, QOBJECT(qstring_from_str(value)))
 #define qobject_unref QDECREF
 #endif
+#if QEMU_VERSION_MAJOR == 4 && QEMU_VERSION_MINOR >= 2 || QEMU_VERSION_MAJOR > 4
+#include "sysemu/replay.h"
+#else
+#include "sysemu/sysemu.h"
+#endif
 
 #include "vitastor_c.h"
 
@@ -48,6 +53,8 @@ void DSO_STAMP_FUN(void)
 }
 #endif
 
+typedef struct VitastorFdData VitastorFdData;
+
 typedef struct VitastorClient
 {
     void *proxy;
@@ -67,12 +74,23 @@ typedef struct VitastorClient
     int rdma_gid_index;
     int rdma_mtu;
     QemuMutex mutex;
+    AioContext *ctx;
+    VitastorFdData **fds;
+    int fd_count, fd_alloc;
 
     uint64_t last_bitmap_inode, last_bitmap_offset, last_bitmap_len;
     uint32_t last_bitmap_granularity;
     uint8_t *last_bitmap;
 } VitastorClient;
 
+typedef struct VitastorFdData
+{
+    VitastorClient *cli;
+    int fd;
+    IOHandler *fd_read, *fd_write;
+    void *opaque;
+} VitastorFdData;
+
 typedef struct VitastorRPC
 {
     BlockDriverState *bs;
@@ -83,10 +101,13 @@ typedef struct VitastorRPC
     uint64_t inode, offset, len;
     uint32_t bitmap_granularity;
     uint8_t *bitmap;
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
+    QEMUBH *bh;
+#endif
 } VitastorRPC;
 
 static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task);
-static void vitastor_co_generic_bh_cb(void *opaque, long retval);
+static void vitastor_co_generic_cb(void *opaque, long retval);
 static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version);
 static void vitastor_close(BlockDriverState *bs);
 
@@ -209,7 +230,7 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
     task->co = qemu_coroutine_self();
 
     qemu_mutex_lock(&client->mutex);
-    vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_bh_cb, task);
+    vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task->complete)
@@ -218,14 +239,70 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
     }
 }
 
-// FIXME: Fix thread safety of the driver - now it segfaults when iothread is enabled in QEMU
-static void vitastor_aio_set_fd_handler(void *ctx, int fd, int unused1, IOHandler *fd_read, IOHandler *fd_write, void *unused2, void *opaque)
+static void vitastor_aio_fd_read(void *fddv)
 {
-    aio_set_fd_handler(ctx, fd,
+    VitastorFdData *fdd = (VitastorFdData*)fddv;
+    qemu_mutex_lock(&fdd->cli->mutex);
+    fdd->fd_read(fdd->opaque);
+    qemu_mutex_unlock(&fdd->cli->mutex);
+}
+
+static void vitastor_aio_fd_write(void *fddv)
+{
+    VitastorFdData *fdd = (VitastorFdData*)fddv;
+    qemu_mutex_lock(&fdd->cli->mutex);
+    fdd->fd_write(fdd->opaque);
+    qemu_mutex_unlock(&fdd->cli->mutex);
+}
+
+static void vitastor_aio_set_fd_handler(void *vcli, int fd, int unused1, IOHandler *fd_read, IOHandler *fd_write, void *unused2, void *opaque)
+{
+    VitastorClient *client = (VitastorClient*)vcli;
+    VitastorFdData *fdd = NULL;
+    int i;
+    for (i = 0; i < client->fd_count; i++)
+    {
+        if (client->fds[i]->fd == fd)
+        {
+            if (fd_read || fd_write)
+            {
+                fdd = client->fds[i];
+                fdd->opaque = opaque;
+                fdd->fd_read = fd_read;
+                fdd->fd_write = fd_write;
+            }
+            else
+            {
+                for (int j = i+1; j < client->fd_count; j++)
+                    client->fds[j-1] = client->fds[j];
+                client->fd_count--;
+            }
+            break;
+        }
+    }
+    if ((fd_read || fd_write) && !fdd)
+    {
+        fdd = (VitastorFdData*)malloc(sizeof(VitastorFdData));
+        fdd->cli = client;
+        fdd->fd = fd;
+        fdd->fd_read = fd_read;
+        fdd->fd_write = fd_write;
+        fdd->opaque = opaque;
+        if (client->fd_count >= client->fd_alloc)
+        {
+            client->fd_alloc = client->fd_alloc*2;
+            if (client->fd_alloc < 16)
+                client->fd_alloc = 16;
+            client->fds = (VitastorFdData**)realloc(client->fds, sizeof(VitastorFdData*) * client->fd_alloc);
+        }
+        client->fds[client->fd_count++] = fdd;
+    }
+    aio_set_fd_handler(client->ctx, fd,
 #if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 5 || QEMU_VERSION_MAJOR >= 3
         0 /*is_external*/,
 #endif
-        fd_read, fd_write,
+        fd_read ? vitastor_aio_fd_read : NULL,
+        fd_write ? vitastor_aio_fd_write : NULL,
 #if QEMU_VERSION_MAJOR == 1 && QEMU_VERSION_MINOR <= 6 || QEMU_VERSION_MAJOR < 1
         NULL /*io_flush*/,
 #endif
@@ -235,7 +312,7 @@ static void vitastor_aio_set_fd_handler(void *ctx, int fd, int unused1, IOHandle
 #if QEMU_VERSION_MAJOR >= 7
         NULL /*io_poll_ready*/,
 #endif
-        opaque);
+        fdd);
 }
 
 static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, Error **errp)
@@ -255,8 +332,9 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
     client->rdma_port_num = qdict_get_try_int(options, "rdma-port-num", 0);
     client->rdma_gid_index = qdict_get_try_int(options, "rdma-gid-index", 0);
     client->rdma_mtu = qdict_get_try_int(options, "rdma-mtu", 0);
+    client->ctx = bdrv_get_aio_context(bs);
     client->proxy = vitastor_c_create_qemu(
-        vitastor_aio_set_fd_handler, bdrv_get_aio_context(bs), client->config_path, client->etcd_host, client->etcd_prefix,
+        vitastor_aio_set_fd_handler, client, client->config_path, client->etcd_host, client->etcd_prefix,
         client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0
     );
     image = client->image = g_strdup(qdict_get_try_str(options, "image"));
@@ -338,6 +416,12 @@ static void vitastor_close(BlockDriverState *bs)
 {
     VitastorClient *client = bs->opaque;
     vitastor_c_destroy(client->proxy);
+    if (client->fds)
+    {
+        free(client->fds);
+        client->fds = NULL;
+        client->fd_alloc = client->fd_count = 0;
+    }
     qemu_mutex_destroy(&client->mutex);
     if (client->config_path)
         g_free(client->config_path);
@@ -454,25 +538,43 @@ static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task)
     };
 }
 
-static void vitastor_co_generic_bh_cb(void *opaque, long retval)
+static void vitastor_co_generic_bh_cb(void *opaque)
 {
     VitastorRPC *task = opaque;
-    task->ret = retval;
     task->complete = 1;
     if (qemu_coroutine_self() != task->co)
     {
 #if QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR > 8
         aio_co_wake(task->co);
 #else
+#if QEMU_VERSION_MAJOR == 2
+        qemu_bh_delete(task->bh);
+#endif
         qemu_coroutine_enter(task->co, NULL);
         qemu_aio_release(task);
 #endif
     }
 }
 
+static void vitastor_co_generic_cb(void *opaque, long retval)
+{
+    VitastorRPC *task = opaque;
+    task->ret = retval;
+#if QEMU_VERSION_MAJOR > 4 || QEMU_VERSION_MAJOR == 4 && QEMU_VERSION_MINOR >= 2
+    replay_bh_schedule_oneshot_event(bdrv_get_aio_context(task->bs), vitastor_co_generic_bh_cb, opaque);
+#elif QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 8
+    aio_bh_schedule_oneshot(bdrv_get_aio_context(task->bs), vitastor_co_generic_bh_cb, opaque);
+#elif QEMU_VERSION_MAJOR >= 2
+    task->bh = aio_bh_new(bdrv_get_aio_context(task->bs), vitastor_co_generic_bh_cb, opaque);
+    qemu_bh_schedule(task->bh);
+#else
+    vitastor_co_generic_bh_cb(opaque);
+#endif
+}
+
 static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version)
 {
-    vitastor_co_generic_bh_cb(opaque, retval);
+    vitastor_co_generic_cb(opaque, retval);
 }
 
 static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
@@ -523,7 +625,7 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
 
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
-    vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task);
+    vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -687,7 +789,7 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
     vitastor_co_init_task(bs, &task);
 
     qemu_mutex_lock(&client->mutex);
-    vitastor_c_sync(client->proxy, vitastor_co_generic_bh_cb, &task);
+    vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.9.2
+Version: 0.9.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/run_3osds.sh

@@ -36,9 +36,6 @@ for i in $(seq 1 $OSD_COUNT); do
     start_osd $i
 done
 
-cd mon
-npm install
-cd ..
 (while true; do node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 || true; done) &>./testdata/mon.log &
 MON_PID=$!
 

tests/run_tests.sh

@@ -16,29 +16,35 @@ SCHEME=ec ./test_change_pg_count.sh
 
 ./test_etcd_fail.sh
 
-./test_failure_domain.sh
-
 ./test_interrupted_rebalance.sh
 IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
 SCHEME=ec ./test_interrupted_rebalance.sh
 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
 
+./test_failure_domain.sh
+
+./test_snapshot.sh
+SCHEME=ec ./test_snapshot.sh
+
 ./test_minsize_1.sh
 
 ./test_move_reappear.sh
 
+./test_rm.sh
+
+./test_snapshot_chain.sh
+SCHEME=ec ./test_snapshot_chain.sh
+
+./test_snapshot_down.sh
+SCHEME=ec ./test_snapshot_down.sh
+
+./test_splitbrain.sh
+
 ./test_rebalance_verify.sh
 IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 SCHEME=ec ./test_rebalance_verify.sh
 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 
-./test_rm.sh
-
-./test_snapshot.sh
-SCHEME=ec ./test_snapshot.sh
-
-./test_splitbrain.sh
-
 ./test_write.sh
 SCHEME=xor ./test_write.sh
 

tests/test_failure_domain.sh

@@ -15,9 +15,6 @@ $ETCDCTL put /vitastor/osd/stats/7 '{"host":"host4","size":1073741824,"time":"'$
 $ETCDCTL put /vitastor/osd/stats/8 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":4,"failure_domain":"rack"}}'
 
-cd mon
-npm install
-cd ..
 node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
 MON_PID=$!
 

tests/test_snapshot_chain.sh

@@ -0,0 +1,48 @@
+#!/bin/bash -ex
+
+. `dirname $0`/run_3osds.sh
+check_qemu
+
+# Test multiple snapshots
+
+build/src/vitastor-cli --etcd_address $ETCD_URL create -s 32M testchain
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/mirror.bin
+
+for i in {1..10}; do
+    # Create a snapshot
+    build/src/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@$i
+    # Check that the new snapshot is see-through
+    qemu-img convert -p \
+        -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testchain" \
+        -O raw ./testdata/check.bin
+    cmp ./testdata/check.bin ./testdata/mirror.bin
+    # Write something to it
+    LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=32 -rw=randwrite \
+        -randrepeat=$((i <= 2)) -buffer_pattern=0x$((10+i))$((10+i))$((10+i))$((10+i)) \
+        -etcd=$ETCD_URL -image=testchain -number_ios=1024 -mirror_file=./testdata/mirror.bin
+    # Check the new content
+    qemu-img convert -p \
+        -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testchain" \
+        -O raw ./testdata/layer1.bin
+    cmp ./testdata/layer1.bin ./testdata/mirror.bin
+done
+
+build/src/vitastor-cli --etcd_address $ETCD_URL rm testchain@1 testchain@9
+
+# Check the final image
+qemu-img convert -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testchain" \
+    -O raw ./testdata/layer1.bin
+cmp ./testdata/layer1.bin ./testdata/mirror.bin
+
+# Check the last remaining snapshot
+qemu-img convert -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testchain@10" \
+    -O raw ./testdata/layer0.bin
+cmp ./testdata/layer0.bin ./testdata/check.bin
+
+format_green OK

tests/test_snapshot_down.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -ex
+
+. `dirname $0`/run_3osds.sh
+check_qemu
+
+# Test merge to child (without "inverse rename" optimisation)
+
+build/src/vitastor-cli --etcd_address $ETCD_URL create -s 128M testchain
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/mirror.bin
+
+# Create a snapshot
+build/src/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@0
+
+# Write something to it
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 -rw=randwrite \
+    -randrepeat=0 -etcd=$ETCD_URL -image=testchain -number_ios=8 -mirror_file=./testdata/mirror.bin
+
+# Check the new content
+qemu-img convert -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testchain" \
+    -O raw ./testdata/layer1.bin
+cmp ./testdata/layer1.bin ./testdata/mirror.bin
+
+# Merge
+build/src/vitastor-cli --etcd_address $ETCD_URL rm testchain@0
+
+# Check the final image
+qemu-img convert -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testchain" \
+    -O raw ./testdata/layer1.bin
+cmp ./testdata/layer1.bin ./testdata/mirror.bin
+
+format_green OK