Test bug similar to 307c1731c1

.gitea/workflows/test.Dockerfile

@@ -10,9 +10,6 @@ RUN set -e -x; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     ln -s /root/qemu-build/qemu-*/ ./qemu; \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
-    cd mon; \
-    npm install; \
-    cd ..; \
     mkdir build; \
     cd build; \
     cmake .. -DWITH_ASAN=yes -DWITH_QEMU=yes; \

.gitea/workflows/test.yml

@@ -71,7 +71,7 @@ jobs:
     steps:
     - name: Run test
       id: test
-      timeout-minutes: 10
+      timeout-minutes: 3
       run: /root/vitastor/tests/test_add_osd.sh
     - name: Print logs
       if: always() && steps.test.outcome == 'failure'
@@ -190,6 +190,24 @@ jobs:
           echo ""
         done
 
+  test_failure_domain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_failure_domain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_interrupted_rebalance:
     runs-on: ubuntu-latest
     needs: build
@@ -262,7 +280,7 @@ jobs:
           echo ""
         done
 
-  test_failure_domain:
+  test_minsize_1:
     runs-on: ubuntu-latest
     needs: build
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
@@ -270,7 +288,115 @@ jobs:
     - name: Run test
       id: test
       timeout-minutes: 3
-      run: /root/vitastor/tests/test_failure_domain.sh
+      run: /root/vitastor/tests/test_minsize_1.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_move_reappear:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_move_reappear.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_rm.sh
     - name: Print logs
       if: always() && steps.test.outcome == 'failure'
       run: |
@@ -316,132 +442,6 @@ jobs:
           echo ""
         done
 
-  test_minsize_1:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_minsize_1.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_move_reappear:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_move_reappear.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_rm.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot_chain:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_snapshot_chain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot_chain_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot_down:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_snapshot_down.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot_down_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_splitbrain:
     runs-on: ubuntu-latest
     needs: build
@@ -460,78 +460,6 @@ jobs:
           echo ""
         done
 
-  test_rebalance_verify:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify_ec_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_write:
     runs-on: ubuntu-latest
     needs: build
@@ -622,7 +550,7 @@ jobs:
           echo ""
         done
 
-  test_heal_csum_32k_dmj:
+  test_heal_imm_ec:
     runs-on: ubuntu-latest
     needs: build
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
@@ -630,97 +558,7 @@ jobs:
     - name: Run test
       id: test
       timeout-minutes: 10
-      run: TEST_NAME=csum_32k_dmj OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_csum_32k_dj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=csum_32k_dj  OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_csum_32k:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=csum_32k     OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_csum_4k_dmj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=csum_4k_dmj  OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_csum_4k_dj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=csum_4k_dj   OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_csum_4k:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=csum_4k      OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+      run: IMMEDIATE_COMMIT=1 SCHEME=ec /root/vitastor/tests/test_heal.sh
     - name: Print logs
       if: always() && steps.test.outcome == 'failure'
       run: |

.gitea/workflows/tests-to-yaml.pl

@@ -7,22 +7,15 @@ for my $line (<>)
     if ($line =~ /\.\/(test_[^\.]+)/s)
     {
         chomp $line;
-        my $base_name = $1;
-        my $test_name = $base_name;
+        my $test_name = $1;
         my $timeout = 3;
-        if ($test_name eq 'test_etcd_fail' || $test_name eq 'test_heal' || $test_name eq 'test_add_osd' ||
-            $test_name eq 'test_interrupted_rebalance' || $test_name eq 'test_rebalance_verify')
+        if ($test_name eq 'test_etcd_fail' || $test_name eq 'test_heal' || $test_name eq 'test_interrupted_rebalance')
         {
             $timeout = 10;
         }
         while ($line =~ /([^\s=]+)=(\S+)/gs)
         {
-            if ($1 eq 'TEST_NAME')
-            {
-                $test_name = $base_name.'_'.$2;
-                last;
-            }
-            elsif ($1 eq 'SCHEME' && $2 eq 'ec')
+            if ($1 eq 'SCHEME' && $2 eq 'ec')
             {
                 $test_name .= '_ec';
             }

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "1.0.0")
+set(VERSION "0.9.0")
 
 add_subdirectory(src)

README-ru.md

@@ -15,7 +15,7 @@ Vitastor архитектурно похож на Ceph, что означает
 и автоматическое распределение данных по любому числу дисков любого размера с настраиваемыми схемами
 избыточности - репликацией или с произвольными кодами коррекции ошибок.
 
-Vitastor нацелен в первую очередь на SSD и SSD+HDD кластеры с как минимум 10 Гбит/с сетью, поддерживает
+Vitastor нацелен на SSD и SSD+HDD кластеры с как минимум 10 Гбит/с сетью, поддерживает
 TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
 что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
 

README.md

@@ -14,8 +14,8 @@ Vitastor is architecturally similar to Ceph which means strong consistency,
 primary-replication, symmetric clustering and automatic data distribution over any
 number of drives of any size with configurable redundancy (replication or erasure codes/XOR).
 
-Vitastor targets primarily SSD and SSD+HDD clusters with at least 10 Gbit/s network,
-supports TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1 ms
+Vitastor targets SSD and SSD+HDD clusters with at least 10 Gbit/s network, supports
+TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1 ms
 with proper hardware which is ~10 times faster than other popular SDS's like Ceph
 or internal systems of public clouds.
 

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v1.0.0
+VERSION ?= v0.9.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.0.0
+          image: vitalif/vitastor-csi:v0.9.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.0.0
+          image: vitalif/vitastor-csi:v0.9.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.0.0"
+    vitastorCSIDriverVersion = "0.9.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-pve-qemu.sh

@@ -1,58 +0,0 @@
-exit
-
-git clone https://git.yourcmc.ru/vitalif/pve-qemu .
-
-# bookworm
-
-docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-bullseye debian:bullseye bash
-
-perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources
-echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve bookworm pve-no-subscription' >> /etc/apt/sources.list
-echo 'deb https://vitastor.io/debian bookworm main' >> /etc/apt/sources.list
-echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
-echo 'ru_RU UTF-8' >> /etc/locale.gen
-echo 'en_US UTF-8' >> /etc/locale.gen
-apt-get update
-apt-get install wget ca-certificates
-wget https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
-wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
-apt-get update
-apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
-mk-build-deps --install ./control
-
-# bullseye
-
-docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-bullseye debian:bullseye bash
-
-grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb /deb-src /' >> /etc/apt/sources.list
-echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve bullseye pve-no-subscription' >> /etc/apt/sources.list
-echo 'deb https://vitastor.io/debian bullseye main' >> /etc/apt/sources.list
-echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
-echo 'ru_RU UTF-8' >> /etc/locale.gen
-echo 'en_US UTF-8' >> /etc/locale.gen
-apt-get update
-apt-get install wget
-wget https://enterprise.proxmox.com/debian/proxmox-release-bullseye.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg
-wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
-apt-get update
-apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
-mk-build-deps --install ./control
-
-# buster
-
-docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-buster debian:buster bash
-
-grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb /deb-src /' >> /etc/apt/sources.list
-echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve buster pve-no-subscription' >> /etc/apt/sources.list
-echo 'deb https://vitastor.io/debian buster main' >> /etc/apt/sources.list
-echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list
-echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
-echo 'ru_RU UTF-8' >> /etc/locale.gen
-echo 'en_US UTF-8' >> /etc/locale.gen
-apt-get update
-apt-get install wget ca-certificates
-wget http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg -O /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
-wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
-apt-get update
-apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
-mk-build-deps --install ./control

debian/build-vitastor-bookworm.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-cat < vitastor.Dockerfile > ../Dockerfile
-cd ..
-mkdir -p packages
-sudo podman build --build-arg REL=bookworm -v `pwd`/packages:/root/packages -f Dockerfile .
-rm Dockerfile

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (1.0.0-1) unstable; urgency=medium
+vitastor (0.9.0-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (1.0.0-1) unstable; urgency=medium
+vitastor (0.9.0-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/patched-qemu.Dockerfile

@@ -1,4 +1,4 @@
-# Build patched QEMU for Debian inside a container
+# Build patched QEMU for Debian Buster or Bullseye/Sid inside a container
 # cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .
 
 ARG REL=
@@ -15,46 +15,47 @@ RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
         echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     fi; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
-    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y install qemu fio liburing1 liburing-dev libgoogle-perftools-dev devscripts
 RUN apt-get -y build-dep qemu
 # To build a custom version
 #RUN cp /root/packages/qemu-orig/* /root
 RUN apt-get --download-only source qemu
 
-ADD patches /root/vitastor/patches
-ADD src/qemu_driver.c /root/vitastor/src/qemu_driver.c
-
-#RUN set -e; \
-#    apt-get install -y wget; \
-#    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg; \
-#    (echo deb http://vitastor.io/debian $REL main > /etc/apt/sources.list.d/vitastor.list); \
-#    (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
-#    apt-get update; \
-#    apt-get install -y vitastor-client vitastor-client-dev quilt
-
+ADD patches/qemu-5.0-vitastor.patch patches/qemu-5.1-vitastor.patch patches/qemu-6.1-vitastor.patch src/qemu_driver.c /root/vitastor/patches/
 RUN set -e; \
-    dpkg -i /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
+    apt-get install -y wget; \
+    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg; \
+    (echo deb http://vitastor.io/debian $REL main > /etc/apt/sources.list.d/vitastor.list); \
+    (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
     apt-get update; \
-    apt-get install -y quilt; \
+    apt-get install -y vitastor-client vitastor-client-dev quilt; \
     mkdir -p /root/packages/qemu-$REL; \
     rm -rf /root/packages/qemu-$REL/*; \
     cd /root/packages/qemu-$REL; \
     dpkg-source -x /root/qemu*.dsc; \
-    QEMU_VER=$(ls -d qemu*/ | perl -pe 's!^.*(\d+\.\d+).*!$1!'); \
-    D=$(ls -d qemu*/); \
-    cp /root/vitastor/patches/qemu-$QEMU_VER-vitastor.patch ./qemu-*/debian/patches; \
-    echo qemu-$QEMU_VER-vitastor.patch >> $D/debian/patches/series; \
+    if ls -d /root/packages/qemu-$REL/qemu-5.0*; then \
+        D=$(ls -d /root/packages/qemu-$REL/qemu-5.0*); \
+        cp /root/vitastor/patches/qemu-5.0-vitastor.patch $D/debian/patches; \
+        echo qemu-5.0-vitastor.patch >> $D/debian/patches/series; \
+    elif ls /root/packages/qemu-$REL/qemu-6.1*; then \
+        D=$(ls -d /root/packages/qemu-$REL/qemu-6.1*); \
+        cp /root/vitastor/patches/qemu-6.1-vitastor.patch $D/debian/patches; \
+        echo qemu-6.1-vitastor.patch >> $D/debian/patches/series; \
+    else \
+        cp /root/vitastor/patches/qemu-5.1-vitastor.patch /root/packages/qemu-$REL/qemu-*/debian/patches; \
+        P=`ls -d /root/packages/qemu-$REL/qemu-*/debian/patches`; \
+        echo qemu-5.1-vitastor.patch >> $P/series; \
+    fi; \
     cd /root/packages/qemu-$REL/qemu-*/; \
     quilt push -a; \
     quilt add block/vitastor.c; \
-    cp /root/vitastor/src/qemu_driver.c block/vitastor.c; \
+    cp /root/vitastor/patches/qemu_driver.c block/vitastor.c; \
     quilt refresh; \
-    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor3; \
+    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor1; \
     DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
     rm -rf /root/packages/qemu-$REL/qemu-*/

debian/vitastor.Dockerfile

@@ -1,4 +1,4 @@
-# Build Vitastor packages for Debian inside a container
+# Build Vitastor packages for Debian Buster or Bullseye/Sid inside a container
 # cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
 
 ARG REL=
@@ -15,12 +15,11 @@ RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
         echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     fi; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
-    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y install fio liburing1 liburing-dev libgoogle-perftools-dev devscripts
 RUN apt-get -y build-dep fio
 RUN apt-get --download-only source fio
 RUN apt-get update && apt-get -y install libjerasure-dev cmake libibverbs-dev libisal-dev
@@ -35,8 +34,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-1.0.0; \
-    cd vitastor-1.0.0; \
+    cp -r /root/vitastor vitastor-0.9.0; \
+    cd vitastor-0.9.0; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +48,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.0.0.orig.tar.xz vitastor-1.0.0; \
-    cd vitastor-1.0.0; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.0.orig.tar.xz vitastor-0.9.0; \
+    cd vitastor-0.9.0; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config.en.md

@@ -21,7 +21,7 @@ Configuration parameters can be set in 3 places:
   mon, fio and QEMU options, OpenStack/Proxmox/etc configuration. The latter
   doesn't allow to set all variables directly, but it allows to override the
   configuration file and set everything you need inside it.
-- OSD superblocks created by [vitastor-disk](usage/disk.en.md) contain
+- OSD superblocks created by [vitastor-disk](../usage/disk.en.md) contain
   primarily disk layout parameters of specific OSDs. In fact, these parameters
   are automatically passed into the command line of vitastor-osd process, so
   they have the same "status" as command-line parameters.

docs/config.ru.md

@@ -23,7 +23,7 @@
   монитора, опциях fio и QEMU, настроек OpenStack, Proxmox и т.п. Последние,
   как правило, не включают полный набор параметров напрямую, но позволяют
   определить путь к файлу конфигурации и задать любые параметры в нём.
-- В суперблоке OSD, записываемом [vitastor-disk](usage/disk.ru.md) - параметры,
+- В суперблоке OSD, записываемом [vitastor-disk](../usage/disk.ru.md) - параметры,
   связанные с дисковым форматом и с этим конкретным OSD. На самом деле,
   при запуске OSD эти параметры автоматически передаются в командную строку
   процесса vitastor-osd, то есть по "статусу" они эквивалентны параметрам

docs/config/layout-cluster.en.md

@@ -33,13 +33,12 @@ Size of objects (data blocks) into which all physical and virtual drives
 in Vitastor, affects memory usage, write amplification and I/O load
 distribution effectiveness.
 
-Recommended default block size is 128 KB for SSD and 1 MB for HDD. In fact,
-it's possible to use 1 MB for SSD too - it will lower memory usage, but
+Recommended default block size is 128 KB for SSD and 4 MB for HDD. In fact,
+it's possible to use 4 MB for SSD too - it will lower memory usage, but
 may increase average WA and reduce linear performance.
 
 OSD memory usage is roughly (SIZE / BLOCK * 68 bytes) which is roughly
 544 MB per 1 TB of used disk space with the default 128 KB block size.
-With 1 MB it's 8 times lower.
 
 ## bitmap_granularity
 

docs/config/layout-cluster.ru.md

@@ -33,14 +33,14 @@ OSD) могут сосуществовать в одном кластере Vita
 настроек, влияет на потребление памяти, объём избыточной записи (write
 amplification) и эффективность распределения нагрузки по OSD.
 
-Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 1 мегабайт
-для HDD. В принципе, для SSD можно тоже использовать блок размером 1 мегабайт,
+Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 4
+мегабайта для HDD. В принципе, для SSD можно тоже использовать 4 мегабайта,
 это понизит использование памяти, но ухудшит распределение нагрузки и в
 среднем увеличит WA.
 
 Потребление памяти OSD составляет примерно (РАЗМЕР / БЛОК * 68 байт),
 т.е. примерно 544 МБ памяти на 1 ТБ занятого места на диске при
-стандартном 128 КБ блоке. При 1 МБ блоке памяти нужно в 8 раз меньше.
+стандартном 128 КБ блоке.
 
 ## bitmap_granularity
 

docs/config/layout-osd.en.md

@@ -24,8 +24,6 @@ initialization and can't be changed after it without losing data.
 - [disable_journal_fsync](#disable_journal_fsync)
 - [disable_device_lock](#disable_device_lock)
 - [disk_alignment](#disk_alignment)
-- [data_csum_type](#data_csum_type)
-- [csum_block_size](#csum_block_size)
 
 ## data_device
 
@@ -176,43 +174,3 @@ Intel Optane (probably, not tested yet).
 
 Clients don't need to be aware of disk_alignment, so it's not required to
 put a modified value into etcd key /vitastor/config/global.
-
-## data_csum_type
-
-- Type: string
-- Default: none
-
-Data checksum type to use. May be "crc32c" or "none". Set to "crc32c" to
-enable data checksums.
-
-## csum_block_size
-
-- Type: integer
-- Default: 4096
-
-Checksum calculation block size.
-
-Must be equal or a multiple of [bitmap_granularity](layout-cluster.en.md#bitmap_granularity)
-(which is usually 4 KB).
-
-Checksums increase metadata size by 4 bytes per each csum_block_size of data.
-
-Checksums are always a tradeoff:
-1. You either sacrifice +1 GB RAM per 1 TB of data
-2. Or you raise csum_block_size, for example, to 32k and sacrifice
-   50% random write iops due to checksum read-modify-write
-3. Or you turn off [inmemory_metadata](osd.en.md#inmemory_metadata) and
-   sacrifice 50% random read iops due to checksum reads
-
-All-flash clusters usually have enough RAM to use default csum_block_size,
-which uses 1 GB RAM per 1 TB of data. HDD clusters usually don't.
-
-Thus, recommended setups are:
-1. All-flash, 1 GB RAM per 1 TB data: default (csum_block_size=4k)
-2. All-flash, less RAM: csum_block_size=4k + inmemory_metadata=false
-3. Hybrid HDD+SSD: csum_block_size=4k + inmemory_metadata=false
-4. HDD-only, faster random read: csum_block_size=32k
-5. HDD-only, faster random write: csum_block_size=4k +
-   inmemory_metadata=false + cached_io_meta=true
-
-See also [cached_io_meta](osd.en.md#cached_io_meta).

docs/config/layout-osd.ru.md

@@ -25,8 +25,6 @@
 - [disable_journal_fsync](#disable_journal_fsync)
 - [disable_device_lock](#disable_device_lock)
 - [disk_alignment](#disk_alignment)
-- [data_csum_type](#data_csum_type)
-- [csum_block_size](#csum_block_size)
 
 ## data_device
 
@@ -185,47 +183,3 @@ journal_block_size и meta_block_size. Однако единственные SSD
 
 Клиентам не обязательно знать про disk_alignment, так что помещать значение
 этого параметра в etcd в /vitastor/config/global не нужно.
-
-## data_csum_type
-
-- Тип: строка
-- Значение по умолчанию: none
-
-Тип используемых OSD контрольных сумм данных. Может быть "crc32c" или "none".
-Установите в "crc32c", чтобы включить расчёт и проверку контрольных сумм данных.
-
-Следует понимать, что контрольные суммы в зависимости от размера блока их
-расчёта либо увеличивают потребление памяти, либо снижают производительность.
-Подробнее смотрите в описании параметра [csum_block_size](#csum_block_size).
-
-## csum_block_size
-
-- Тип: целое число
-- Значение по умолчанию: 4096
-
-Размер блока расчёта контрольных сумм.
-
-Должен быть равен или кратен [bitmap_granularity](layout-cluster.ru.md#bitmap_granularity)
-(который обычно равен 4 КБ).
-
-Контрольные суммы увеличивают размер метаданных на 4 байта на каждые
-csum_block_size данных.
-
-Контрольные суммы - это всегда компромисс:
-1. Вы либо жертвуете потреблением +1 ГБ памяти на 1 ТБ дискового пространства
-2. Либо вы повышаете csum_block_size до, скажем, 32k и жертвуете 50%
-   скорости случайной записи из-за цикла чтения-изменения-записи для расчёта
-   новых контрольных сумм
-3. Либо вы отключаете [inmemory_metadata](osd.ru.md#inmemory_metadata) и
-   жертвуете 50% скорости случайного чтения из-за чтения контрольных сумм
-   с диска
-
-Таким образом, рекомендуются следующие варианты настроек:
-1. All-flash, 1 ГБ памяти на 1 ТБ данных: по умолчанию (csum_block_size=4k)
-2. All-flash, меньше памяти: csum_block_size=4k + inmemory_metadata=false
-3. Гибридные HDD+SSD: csum_block_size=4k + inmemory_metadata=false
-4. Только HDD, быстрее случайное чтение: csum_block_size=32k
-5. Только HDD, быстрее случайная запись: csum_block_size=4k +
-   inmemory_metadata=false + cached_io_meta=true
-
-Смотрите также [cached_io_meta](osd.ru.md#cached_io_meta).

docs/config/osd.en.md

@@ -31,9 +31,6 @@ them, even without restarting by updating configuration in etcd.
 - [max_flusher_count](#max_flusher_count)
 - [inmemory_metadata](#inmemory_metadata)
 - [inmemory_journal](#inmemory_journal)
-- [cached_io_data](#cached_io_data)
-- [cached_io_meta](#cached_io_meta)
-- [cached_io_journal](#cached_io_journal)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
 - [throttle_small_writes](#throttle_small_writes)
@@ -258,48 +255,6 @@ is typically very small because it's sufficient to have 16-32 MB journal
 for SSD OSDs. However, in theory it's possible that you'll want to turn it
 off for hybrid (HDD+SSD) OSDs with large journals on quick devices.
 
-## cached_io_data
-
-- Type: boolean
-- Default: false
-
-Read and write *data* through Linux page cache, i.e. use a file descriptor
-opened with O_SYNC, but without O_DIRECT for I/O. May improve read
-performance for hot data and slower disks - HDDs and maybe SATA SSDs.
-Not recommended for desktop SSDs without capacitors because O_SYNC flushes
-disk cache on every write.
-
-## cached_io_meta
-
-- Type: boolean
-- Default: false
-
-Read and write *metadata* through Linux page cache. May improve read
-performance only if your drives are relatively slow (HDD, SATA SSD), and
-only if checksums are enabled and [inmemory_metadata](#inmemory_metadata)
-is disabled, because in this case metadata blocks are read from disk
-on every read request to verify checksums and caching them may reduce this
-extra read load.
-
-Absolutely pointless to enable with enabled inmemory_metadata because all
-metadata is kept in memory anyway, and likely pointless without checksums,
-because in that case, metadata blocks are read from disk only during journal
-flushing.
-
-If the same device is used for data and metadata, enabling [cached_io_data](#cached_io_data)
-also enables this parameter, given that it isn't turned off explicitly.
-
-## cached_io_journal
-
-- Type: boolean
-- Default: false
-
-Read and write *journal* through Linux page cache. May improve read
-performance if [inmemory_journal](#inmemory_journal) is turned off.
-
-If the same device is used for metadata and journal, enabling [cached_io_meta](#cached_io_meta)
-also enables this parameter, given that it isn't turned off explicitly.
-
 ## journal_sector_buffer_count
 
 - Type: integer

docs/config/osd.ru.md

@@ -32,9 +32,6 @@
 - [max_flusher_count](#max_flusher_count)
 - [inmemory_metadata](#inmemory_metadata)
 - [inmemory_journal](#inmemory_journal)
-- [cached_io_data](#cached_io_data)
-- [cached_io_meta](#cached_io_meta)
-- [cached_io_journal](#cached_io_journal)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
 - [throttle_small_writes](#throttle_small_writes)
@@ -266,52 +263,6 @@ Flusher - это микро-поток (корутина), которая коп
 параметра может оказаться полезным для гибридных OSD (HDD+SSD) с большими
 журналами, расположенными на быстром по сравнению с HDD устройстве.
 
-## cached_io_data
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Читать и записывать *данные* через системный кэш Linux (page cache), то есть,
-использовать для данных файловый дескриптор, открытый без флага O_DIRECT, но
-с флагом O_SYNC. Может улучшить скорость чтения для относительно медленных
-дисков - HDD и, возможно, SATA SSD. Не рекомендуется для потребительских
-SSD без конденсаторов, так как O_SYNC сбрасывает кэш диска при каждой записи.
-
-## cached_io_meta
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Читать и записывать *метаданные* через системный кэш Linux. Может улучшить
-скорость чтения, если у вас медленные диски, и только если контрольные суммы
-включены, а параметр [inmemory_metadata](#inmemory_metadata) отключён, так
-как в этом случае блоки метаданных читаются с диска при каждом запросе чтения
-для проверки контрольных сумм и их кэширование может снизить дополнительную
-нагрузку на диск.
-
-Абсолютно бессмысленно включать данный параметр, если параметр
-inmemory_metadata включён (по умолчанию это так), и также вероятно
-бессмысленно включать его, если не включены контрольные суммы, так как в
-этом случае блоки метаданных читаются с диска только во время сброса
-журнала.
-
-Если одно и то же устройство используется для данных и метаданных, включение
-[cached_io_data](#cached_io_data) также включает данный параметр, при
-условии, что он не отключён явным образом.
-
-## cached_io_journal
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Читать и записывать *журнал* через системный кэш Linux. Может улучшить
-скорость чтения, если параметр [inmemory_journal](#inmemory_journal)
-отключён.
-
-Если одно и то же устройство используется для метаданных и журнала,
-включение [cached_io_meta](#cached_io_meta) также включает данный
-параметр, при условии, что он не отключён явным образом.
-
 ## journal_sector_buffer_count
 
 - Тип: целое число

docs/config/src/include.js

@@ -1,145 +0,0 @@
-#!/usr/bin/nodejs
-
-const fsp = require('fs').promises;
-
-run(process.argv).catch(console.error);
-
-async function run(argv)
-{
-    if (argv.length < 3)
-    {
-        console.log('Markdown preprocessor\nUSAGE: ./include.js file.md');
-        return;
-    }
-    const index_file = await fsp.realpath(argv[2]);
-    const re = /(\{\{[\s\S]*?\}\}|\[[^\]]+\]\([^\)]+\)|(?:^|\n)#[^\n]+)/;
-    let text = await fsp.readFile(index_file, { encoding: 'utf-8' });
-    text = text.split(re);
-    let included = {};
-    let heading = 0, heading_name = '', m;
-    for (let i = 0; i < text.length; i++)
-    {
-        if (text[i].substr(0, 2) == '{{')
-        {
-            // Inclusion
-            let incfile = text[i].substr(2, text[i].length-4);
-            let section = null;
-            let indent = heading;
-            incfile = incfile.replace(/\s*\|\s*indent\s*=\s*(-?\d+)\s*$/, (m, m1) => { indent = parseInt(m1); return ''; });
-            incfile = incfile.replace(/\s*#\s*([^#]+)$/, (m, m1) => { section = m1; return ''; });
-            let inc_heading = section;
-            incfile = rel2abs(index_file, incfile);
-            let inc = await fsp.readFile(incfile, { encoding: 'utf-8' });
-            inc = inc.trim().replace(/^[\s\S]+?\n#/, '#'); // remove until the first header
-            inc = inc.split(re);
-            const indent_str = new Array(indent+1).join('#');
-            let section_start = -1, section_end = -1;
-            for (let j = 0; j < inc.length; j++)
-            {
-                if ((m = /^(\n?)(#+\s*)([\s\S]+)$/.exec(inc[j])))
-                {
-                    if (!inc_heading)
-                    {
-                        inc_heading = m[3].trim();
-                    }
-                    if (section)
-                    {
-                        if (m[3].trim() == section)
-                            section_start = j;
-                        else if (section_start >= 0)
-                        {
-                            section_end = j;
-                            break;
-                        }
-                    }
-                    inc[j] = m[1] + indent_str + m[2] + m[3];
-                }
-                else if ((m = /^(\[[^\]]+\]\()([^\)]+)(\))$/.exec(inc[j])) && !/^https?:(\/\/)|^#/.exec(m[2]))
-                {
-                    const abs_m2 = rel2abs(incfile, m[2]);
-                    const rel_m = abs2rel(__filename, abs_m2);
-                    if (rel_m.substr(0, 9) == '../../../') // outside docs
-                        inc[j] = m[1] + 'https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/'+rel2abs('docs/config/src/include.js', rel_m) + m[3];
-                    else
-                        inc[j] = m[1] + abs_m2 + m[3];
-                }
-            }
-            if (section)
-            {
-                inc = section_start >= 0 ? inc.slice(section_start, section_end < 0 ? inc.length : section_end) : [];
-            }
-            if (inc.length)
-            {
-                if (!inc_heading)
-                    inc_heading = heading_name||'';
-                included[incfile+(section ? '#'+section : '')] = '#'+inc_heading.toLowerCase().replace(/\P{L}+/ug, '-').replace(/^-|-$/g, '');
-                inc[0] = inc[0].replace(/^\s+/, '');
-                inc[inc.length-1] = inc[inc.length-1].replace(/\s+$/, '');
-            }
-            text.splice(i, 1, ...inc);
-            i = i + inc.length - 1;
-        }
-        else if ((m = /^\n?(#+)\s*([\s\S]+)$/.exec(text[i])))
-        {
-            // Heading
-            heading = m[1].length;
-            heading_name = m[2].trim();
-        }
-    }
-    for (let i = 0; i < text.length; i++)
-    {
-        if ((m = /^(\[[^\]]+\]\()([^\)]+)(\))$/.exec(text[i])) && !/^https?:(\/\/)|^#/.exec(m[2]))
-        {
-            const p = m[2].indexOf('#');
-            if (included[m[2]])
-            {
-                text[i] = m[1]+included[m[2]]+m[3];
-            }
-            else if (p >= 0 && included[m[2].substr(0, p)])
-            {
-                text[i] = m[1]+m[2].substr(p)+m[3];
-            }
-        }
-    }
-    console.log(text.join(''));
-}
-
-function rel2abs(ref, rel)
-{
-    rel = [ ...ref.replace(/^(.*)\/[^\/]+$/, '$1').split(/\/+/), ...rel.split(/\/+/) ];
-    return killdots(rel).join('/');
-}
-
-function abs2rel(ref, abs)
-{
-    ref = ref.split(/\/+/);
-    abs = abs.split(/\/+/);
-    while (ref.length > 1 && ref[0] == abs[0])
-    {
-        ref.shift();
-        abs.shift();
-    }
-    for (let i = 1; i < ref.length; i++)
-    {
-        abs.unshift('..');
-    }
-    return killdots(abs).join('/');
-}
-
-function killdots(rel)
-{
-    for (let i = 0; i < rel.length; i++)
-    {
-        if (rel[i] == '.')
-        {
-            rel.splice(i, 1);
-            i--;
-        }
-        else if (i >= 1 && rel[i] == '..' && rel[i-1] != '..')
-        {
-            rel.splice(i-1, 2);
-            i -= 2;
-        }
-    }
-    return rel;
-}

docs/config/src/included.en.md

@@ -1,65 +0,0 @@
-# Vitastor
-
-{{../../../README.md#The Idea}}
-
-{{../../../README.md#Talks and presentations}}
-
-{{../../intro/features.en.md}}
-
-{{../../intro/quickstart.en.md}}
-
-{{../../intro/architecture.en.md}}
-
-## Installation
-
-{{../../installation/packages.en.md}}
-
-{{../../installation/proxmox.en.md}}
-
-{{../../installation/openstack.en.md}}
-
-{{../../installation/kubernetes.en.md}}
-
-{{../../installation/source.en.md}}
-
-{{../../config.en.md|indent=1}}
-
-{{../../config/common.en.md|indent=2}}
-
-{{../../config/network.en.md|indent=2}}
-
-{{../../config/layout-cluster.en.md|indent=2}}
-
-{{../../config/layout-osd.en.md|indent=2}}
-
-{{../../config/osd.en.md|indent=2}}
-
-{{../../config/monitor.en.md|indent=2}}
-
-{{../../config/pool.en.md|indent=2}}
-
-{{../../config/inode.en.md|indent=2}}
-
-## Usage
-
-{{../../usage/cli.en.md}}
-
-{{../../usage/disk.en.md}}
-
-{{../../usage/fio.en.md}}
-
-{{../../usage/nbd.en.md}}
-
-{{../../usage/qemu.en.md}}
-
-{{../../usage/nfs.en.md}}
-
-## Performance
-
-{{../../performance/understanding.en.md}}
-
-{{../../performance/theoretical.en.md}}
-
-{{../../performance/comparison1.en.md}}
-
-{{../../intro/author.en.md|indent=1}}

docs/config/src/included.ru.md

@@ -1,65 +0,0 @@
-# Vitastor
-
-{{../../../README-ru.md#Идея|indent=0}}
-
-{{../../../README-ru.md#Презентации и записи докладов|indent=0}}
-
-{{../../intro/features.ru.md}}
-
-{{../../intro/quickstart.ru.md}}
-
-{{../../intro/architecture.ru.md}}
-
-## Установка
-
-{{../../installation/packages.ru.md}}
-
-{{../../installation/proxmox.ru.md}}
-
-{{../../installation/openstack.ru.md}}
-
-{{../../installation/kubernetes.ru.md}}
-
-{{../../installation/source.ru.md}}
-
-{{../../config.ru.md|indent=1}}
-
-{{../../config/common.ru.md|indent=2}}
-
-{{../../config/network.ru.md|indent=2}}
-
-{{../../config/layout-cluster.ru.md|indent=2}}
-
-{{../../config/layout-osd.ru.md|indent=2}}
-
-{{../../config/osd.ru.md|indent=2}}
-
-{{../../config/monitor.ru.md|indent=2}}
-
-{{../../config/pool.ru.md|indent=2}}
-
-{{../../config/inode.ru.md|indent=2}}
-
-## Использование
-
-{{../../usage/cli.ru.md}}
-
-{{../../usage/disk.ru.md}}
-
-{{../../usage/fio.ru.md}}
-
-{{../../usage/nbd.ru.md}}
-
-{{../../usage/qemu.ru.md}}
-
-{{../../usage/nfs.ru.md}}
-
-## Производительность
-
-{{../../performance/understanding.ru.md}}
-
-{{../../performance/theoretical.ru.md}}
-
-{{../../performance/comparison1.ru.md}}
-
-{{../../intro/author.ru.md|indent=1}}

docs/config/src/layout-cluster.yml

@@ -7,27 +7,26 @@
     in Vitastor, affects memory usage, write amplification and I/O load
     distribution effectiveness.
 
-    Recommended default block size is 128 KB for SSD and 1 MB for HDD. In fact,
-    it's possible to use 1 MB for SSD too - it will lower memory usage, but
+    Recommended default block size is 128 KB for SSD and 4 MB for HDD. In fact,
+    it's possible to use 4 MB for SSD too - it will lower memory usage, but
     may increase average WA and reduce linear performance.
 
     OSD memory usage is roughly (SIZE / BLOCK * 68 bytes) which is roughly
     544 MB per 1 TB of used disk space with the default 128 KB block size.
-    With 1 MB it's 8 times lower.
   info_ru: |
     Размер объектов (блоков данных), на которые делятся физические и виртуальные
     диски в Vitastor (в рамках каждого пула). Одна из ключевых на данный момент
     настроек, влияет на потребление памяти, объём избыточной записи (write
     amplification) и эффективность распределения нагрузки по OSD.
 
-    Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 1 мегабайт
-    для HDD. В принципе, для SSD можно тоже использовать блок размером 1 мегабайт,
+    Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 4
+    мегабайта для HDD. В принципе, для SSD можно тоже использовать 4 мегабайта,
     это понизит использование памяти, но ухудшит распределение нагрузки и в
     среднем увеличит WA.
 
     Потребление памяти OSD составляет примерно (РАЗМЕР / БЛОК * 68 байт),
     т.е. примерно 544 МБ памяти на 1 ТБ занятого места на диске при
-    стандартном 128 КБ блоке. При 1 МБ блоке памяти нужно в 8 раз меньше.
+    стандартном 128 КБ блоке.
 - name: bitmap_granularity
   type: int
   default: 4096

docs/config/src/layout-osd.yml

@@ -204,73 +204,3 @@
 
     Клиентам не обязательно знать про disk_alignment, так что помещать значение
     этого параметра в etcd в /vitastor/config/global не нужно.
-- name: data_csum_type
-  type: string
-  default: none
-  info: |
-    Data checksum type to use. May be "crc32c" or "none". Set to "crc32c" to
-    enable data checksums.
-  info_ru: |
-    Тип используемых OSD контрольных сумм данных. Может быть "crc32c" или "none".
-    Установите в "crc32c", чтобы включить расчёт и проверку контрольных сумм данных.
-
-    Следует понимать, что контрольные суммы в зависимости от размера блока их
-    расчёта либо увеличивают потребление памяти, либо снижают производительность.
-    Подробнее смотрите в описании параметра [csum_block_size](#csum_block_size).
-- name: csum_block_size
-  type: int
-  default: 4096
-  info: |
-    Checksum calculation block size.
-
-    Must be equal or a multiple of [bitmap_granularity](layout-cluster.en.md#bitmap_granularity)
-    (which is usually 4 KB).
-
-    Checksums increase metadata size by 4 bytes per each csum_block_size of data.
-
-    Checksums are always a tradeoff:
-    1. You either sacrifice +1 GB RAM per 1 TB of data
-    2. Or you raise csum_block_size, for example, to 32k and sacrifice
-       50% random write iops due to checksum read-modify-write
-    3. Or you turn off [inmemory_metadata](osd.en.md#inmemory_metadata) and
-       sacrifice 50% random read iops due to checksum reads
-
-    All-flash clusters usually have enough RAM to use default csum_block_size,
-    which uses 1 GB RAM per 1 TB of data. HDD clusters usually don't.
-
-    Thus, recommended setups are:
-    1. All-flash, 1 GB RAM per 1 TB data: default (csum_block_size=4k)
-    2. All-flash, less RAM: csum_block_size=4k + inmemory_metadata=false
-    3. Hybrid HDD+SSD: csum_block_size=4k + inmemory_metadata=false
-    4. HDD-only, faster random read: csum_block_size=32k
-    5. HDD-only, faster random write: csum_block_size=4k +
-       inmemory_metadata=false + cached_io_meta=true
-
-    See also [cached_io_meta](osd.en.md#cached_io_meta).
-  info_ru: |
-    Размер блока расчёта контрольных сумм.
-
-    Должен быть равен или кратен [bitmap_granularity](layout-cluster.ru.md#bitmap_granularity)
-    (который обычно равен 4 КБ).
-
-    Контрольные суммы увеличивают размер метаданных на 4 байта на каждые
-    csum_block_size данных.
-
-    Контрольные суммы - это всегда компромисс:
-    1. Вы либо жертвуете потреблением +1 ГБ памяти на 1 ТБ дискового пространства
-    2. Либо вы повышаете csum_block_size до, скажем, 32k и жертвуете 50%
-       скорости случайной записи из-за цикла чтения-изменения-записи для расчёта
-       новых контрольных сумм
-    3. Либо вы отключаете [inmemory_metadata](osd.ru.md#inmemory_metadata) и
-       жертвуете 50% скорости случайного чтения из-за чтения контрольных сумм
-       с диска
-
-    Таким образом, рекомендуются следующие варианты настроек:
-    1. All-flash, 1 ГБ памяти на 1 ТБ данных: по умолчанию (csum_block_size=4k)
-    2. All-flash, меньше памяти: csum_block_size=4k + inmemory_metadata=false
-    3. Гибридные HDD+SSD: csum_block_size=4k + inmemory_metadata=false
-    4. Только HDD, быстрее случайное чтение: csum_block_size=32k
-    5. Только HDD, быстрее случайная запись: csum_block_size=4k +
-       inmemory_metadata=false + cached_io_meta=true
-
-    Смотрите также [cached_io_meta](osd.ru.md#cached_io_meta).

docs/config/src/osd.yml

@@ -260,73 +260,6 @@
     достаточно 16- или 32-мегабайтного журнала. Однако в теории отключение
     параметра может оказаться полезным для гибридных OSD (HDD+SSD) с большими
     журналами, расположенными на быстром по сравнению с HDD устройстве.
-- name: cached_io_data
-  type: bool
-  default: false
-  info: |
-    Read and write *data* through Linux page cache, i.e. use a file descriptor
-    opened with O_SYNC, but without O_DIRECT for I/O. May improve read
-    performance for hot data and slower disks - HDDs and maybe SATA SSDs.
-    Not recommended for desktop SSDs without capacitors because O_SYNC flushes
-    disk cache on every write.
-  info_ru: |
-    Читать и записывать *данные* через системный кэш Linux (page cache), то есть,
-    использовать для данных файловый дескриптор, открытый без флага O_DIRECT, но
-    с флагом O_SYNC. Может улучшить скорость чтения для относительно медленных
-    дисков - HDD и, возможно, SATA SSD. Не рекомендуется для потребительских
-    SSD без конденсаторов, так как O_SYNC сбрасывает кэш диска при каждой записи.
-- name: cached_io_meta
-  type: bool
-  default: false
-  info: |
-    Read and write *metadata* through Linux page cache. May improve read
-    performance only if your drives are relatively slow (HDD, SATA SSD), and
-    only if checksums are enabled and [inmemory_metadata](#inmemory_metadata)
-    is disabled, because in this case metadata blocks are read from disk
-    on every read request to verify checksums and caching them may reduce this
-    extra read load.
-
-    Absolutely pointless to enable with enabled inmemory_metadata because all
-    metadata is kept in memory anyway, and likely pointless without checksums,
-    because in that case, metadata blocks are read from disk only during journal
-    flushing.
-
-    If the same device is used for data and metadata, enabling [cached_io_data](#cached_io_data)
-    also enables this parameter, given that it isn't turned off explicitly.
-  info_ru: |
-    Читать и записывать *метаданные* через системный кэш Linux. Может улучшить
-    скорость чтения, если у вас медленные диски, и только если контрольные суммы
-    включены, а параметр [inmemory_metadata](#inmemory_metadata) отключён, так
-    как в этом случае блоки метаданных читаются с диска при каждом запросе чтения
-    для проверки контрольных сумм и их кэширование может снизить дополнительную
-    нагрузку на диск.
-
-    Абсолютно бессмысленно включать данный параметр, если параметр
-    inmemory_metadata включён (по умолчанию это так), и также вероятно
-    бессмысленно включать его, если не включены контрольные суммы, так как в
-    этом случае блоки метаданных читаются с диска только во время сброса
-    журнала.
-
-    Если одно и то же устройство используется для данных и метаданных, включение
-    [cached_io_data](#cached_io_data) также включает данный параметр, при
-    условии, что он не отключён явным образом.
-- name: cached_io_journal
-  type: bool
-  default: false
-  info: |
-    Read and write *journal* through Linux page cache. May improve read
-    performance if [inmemory_journal](#inmemory_journal) is turned off.
-
-    If the same device is used for metadata and journal, enabling [cached_io_meta](#cached_io_meta)
-    also enables this parameter, given that it isn't turned off explicitly.
-  info_ru: |
-    Читать и записывать *журнал* через системный кэш Linux. Может улучшить
-    скорость чтения, если параметр [inmemory_journal](#inmemory_journal)
-    отключён.
-
-    Если одно и то же устройство используется для метаданных и журнала,
-    включение [cached_io_meta](#cached_io_meta) также включает данный
-    параметр, при условии, что он не отключён явным образом.
 - name: journal_sector_buffer_count
   type: int
   default: 32

docs/installation/kubernetes.ru.md

@@ -8,13 +8,13 @@
 
 У Vitastor есть CSI-плагин для Kubernetes, поддерживающий RWO, а также блочные RWX, тома.
 
-Для установки возьмите манифесты из директории [csi/deploy/](../../csi/deploy/), поместите
-вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
-настройте StorageClass в [csi/deploy/009-storage-class.yaml](../../csi/deploy/009-storage-class.yaml)
+Для установки возьмите манифесты из директории [csi/deploy/](../csi/deploy/), поместите
+вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../csi/deploy/001-csi-config-map.yaml),
+настройте StorageClass в [csi/deploy/009-storage-class.yaml](../csi/deploy/009-storage-class.yaml)
 и примените все `NNN-*.yaml` к вашей инсталляции Kubernetes.
 
 ```
 for i in ./???-*.yaml; do kubectl apply -f $i; done
 ```
 
-После этого вы сможете создавать PersistentVolume. Пример смотрите в файле [csi/deploy/example-pvc.yaml](../../csi/deploy/example-pvc.yaml).
+После этого вы сможете создавать PersistentVolume. Пример смотрите в файле [csi/deploy/example-pvc.yaml](../csi/deploy/example-pvc.yaml).

docs/installation/openstack.en.md

@@ -36,5 +36,5 @@ vitastor_pool_id = 1
 image_upload_use_cinder_backend = True
 ```
 
-To put Glance images in Vitastor, use [volume-backed images](https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html),
+To put Glance images in Vitastor, use [https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html](volume-backed images),
 although the support has not been verified yet.

docs/installation/openstack.ru.md

@@ -36,5 +36,5 @@ image_upload_use_cinder_backend = True
 ```
 
 Чтобы помещать в Vitastor Glance-образы, нужно использовать
-[образы на основе томов Cinder](https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html),
+[https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html](образы на основе томов Cinder),
 однако, поддержка этой функции ещё не проверялась.

docs/installation/packages.en.md

@@ -11,11 +11,8 @@
 - Trust Vitastor package signing key:
   `wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg`
 - Add Vitastor package repository to your /etc/apt/sources.list:
-  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
-  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
+  - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
-  - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
-    stable version from 0.9.x branch instead of 1.x
 - For Debian 10 (Buster) also enable backports repository:
   `deb http://deb.debian.org/debian buster-backports main`
 - Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu`

docs/installation/packages.ru.md

@@ -11,11 +11,8 @@
 - Добавьте ключ репозитория Vitastor:
   `wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg`
 - Добавьте репозиторий Vitastor в /etc/apt/sources.list:
-  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
-  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
+  - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
-  - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
-    установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Для Debian 10 (Buster) также включите репозиторий backports:
   `deb http://deb.debian.org/debian buster-backports main`
 - Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu`

docs/installation/proxmox.en.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.0 are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-7.4 are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
-  bookworm for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
+  buster for 6.4, bullseye for 7.4, pve7.1 for 7.1, pve7.2 for 7.2, pve7.3 for 7.3
 - Install vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* or see note) packages from Vitastor repository
 - Define storage in `/etc/pve/storage.cfg` (see below)
 - Block network access from VMs to Vitastor network (to OSDs and etcd),
@@ -35,5 +35,5 @@ vitastor: vitastor
     vitastor_nbd 0
 ```
 
-\* Note: you can also manually copy [patches/VitastorPlugin.pm](../../patches/VitastorPlugin.pm) to Proxmox hosts
+\* Note: you can also manually copy [patches/VitastorPlugin.pm](patches/VitastorPlugin.pm) to Proxmox hosts
 as `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm` instead of installing pve-storage-vitastor.

docs/installation/proxmox.ru.md

@@ -1,15 +1,15 @@
-[Документация](../../README-ru.md#документация) → Установка → Proxmox VE
+[Документация](../../README-ru.md#документация) → Установка → Proxmox
 
 -----
 
 [Read in English](proxmox.en.md)
 
-# Proxmox VE
+# Proxmox
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.0):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-7.4):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
-  bookworm для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
+  buster для 6.4, bullseye для 7.4, pve7.1 для 7.1, pve7.2 для 7.2, pve7.3 для 7.3
 - Установите пакеты vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* или см. сноску) из репозитория Vitastor
 - Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже)
 - Обязательно заблокируйте доступ от виртуальных машин к сети Vitastor (OSD и etcd), т.к. Vitastor (пока) не поддерживает аутентификацию
@@ -35,5 +35,5 @@ vitastor: vitastor
 ```
 
 \* Примечание: вместо установки пакета pve-storage-vitastor вы можете вручную скопировать файл
-[patches/VitastorPlugin.pm](../../patches/VitastorPlugin.pm) на хосты Proxmox как
+[patches/VitastorPlugin.pm](patches/VitastorPlugin.pm) на хосты Proxmox как
 `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm`.

docs/installation/source.en.md

@@ -21,7 +21,7 @@
 
 ## Basic instructions
 
-Download source, for example using git: `git clone --recurse-submodules https://git.yourcmc.ru/vitalif/vitastor/`
+Download source, for example using git: `git clone --recurse-submodules https://yourcmc.ru/git/vitalif/vitastor/`
 
 Get `fio` source and symlink it into `<vitastor>/fio`. If you don't want to build fio engine,
 you can disable it by passing `-DWITH_FIO=no` to cmake.
@@ -41,7 +41,7 @@ It's recommended to build the QEMU driver (qemu_driver.c) in-tree, as a part of
 QEMU build process. To do that:
 - Install vitastor client library headers (from source or from vitastor-client-dev package)
 - Take a corresponding patch from `patches/qemu-*-vitastor.patch` and apply it to QEMU source
-- Copy `src/qemu_driver.c` to QEMU source directory as `block/vitastor.c`
+- Copy `src/qemu_driver.c` to QEMU source directory as `block/block-vitastor.c`
 - Build QEMU as usual
 
 But it is also possible to build it out-of-tree. To do that:

docs/installation/source.ru.md

@@ -21,7 +21,7 @@
 
 ## Базовая инструкция
 
-Скачайте исходные коды, например, из git: `git clone --recurse-submodules https://git.yourcmc.ru/vitalif/vitastor/`
+Скачайте исходные коды, например, из git: `git clone --recurse-submodules https://yourcmc.ru/git/vitalif/vitastor/`
 
 Скачайте исходные коды пакета `fio`, распакуйте их и создайте символическую ссылку на них
 в директории исходников Vitastor: `<vitastor>/fio`. Либо, если вы не хотите собирать плагин fio,
@@ -41,7 +41,7 @@ cmake .. && make -j8 install
 Драйвер QEMU (qemu_driver.c) рекомендуется собирать вместе с самим QEMU. Для этого:
 - Установите заголовки клиентской библиотеки Vitastor (из исходников или из пакета vitastor-client-dev)
 - Возьмите соответствующий патч из `patches/qemu-*-vitastor.patch` и примените его к исходникам QEMU
-- Скопируйте [src/qemu_driver.c](../../src/qemu_driver.c) в директорию исходников QEMU как `block/vitastor.c`
+- Скопируйте [src/qemu_driver.c](../../src/qemu_driver.c) в директорию исходников QEMU как `block/block-vitastor.c`
 - Соберите QEMU как обычно
 
 Однако в целях отладки драйвер также можно собирать отдельно от QEMU. Для этого:
@@ -60,7 +60,7 @@ cmake .. && make -j8 install
       * Для QEMU 2.0+: `<qemu>/qapi-types.h` &rarr; `<vitastor>/qemu/b/qemu/qapi-types.h`
    - `config-host.h` и `qapi` нужны, т.к. в них содержатся автогенерируемые заголовки
 - Сконфигурируйте cmake Vitastor с `WITH_QEMU=yes` (`cmake .. -DWITH_QEMU=yes`) и, если вы
-  используете RHEL-подобный дистрибутив, также с `QEMU_PLUGINDIR=qemu-kvm`.
+  используете RHEL-подобый дистрибутив, также с `QEMU_PLUGINDIR=qemu-kvm`.
 - После этого в процессе сборки Vitastor также будет собираться подходящий для вашей
   версии QEMU `block-vitastor.so`.
 - Таким образом можно использовать драйвер даже с немодифицированным QEMU, но в этом случае

docs/intro/architecture.en.md

@@ -44,7 +44,7 @@
   depends linearly on drive capacity and data store block size which is 128 KB by default.
   With 128 KB blocks metadata takes around 512 MB per 1 TB (which is still less than Ceph wants).
   Journal is also kept in memory by default, but in SSD-only clusters it's only 32 MB, and in SSD+HDD
-  clusters, where it's beneficial to increase it, [inmemory_journal](../config/osd.en.md#inmemory_journal) can be disabled.
+  clusters, where it's beneficial to increase it, [inmemory_journal](docs/config/osd.en.md#inmemory_journal) can be disabled.
 - Vitastor storage layer doesn't have internal copy-on-write or redirect-write. I know that maybe
   it's possible to create a good copy-on-write storage, but it's much harder and makes performance
   less deterministic, so CoW isn't used in Vitastor.

docs/intro/architecture.ru.md

@@ -156,7 +156,7 @@
   блока хранилища (block_size, по умолчанию 128 КБ). С 128 КБ блоком потребление памяти
   составляет примерно 512 МБ на 1 ТБ данных. Журналы по умолчанию тоже хранятся в памяти,
   но в SSD-кластерах нужный размер журнала составляет всего 32 МБ, а в гибридных (SSD+HDD)
-  кластерах, в которых есть смысл делать журналы больше, можно отключить [inmemory_journal](../config/osd.ru.md#inmemory_journal).
+  кластерах, в которых есть смысл делать журналы больше, можно отключить [inmemory_journal](../docs/config/osd.ru.md#inmemory_journal).
 - В Vitastor нет внутреннего copy-on-write. Я считаю, что реализация CoW-хранилища гораздо сложнее,
   поэтому сложнее добиться устойчиво хороших результатов. Возможно, в один прекрасный день
   я придумаю красивый алгоритм для CoW-хранилища, но пока нет — внутреннего CoW в Vitastor не будет.

docs/intro/features.en.md

@@ -29,14 +29,13 @@
 - Snapshots and copy-on-write image clones
 - [Write throttling to smooth random write workloads in SSD+HDD configurations](../config/osd.en.md#throttle_small_writes)
 - [RDMA/RoCEv2 support via libibverbs](../config/network.en.md#rdma_device)
-- [Scrubbing](../config/osd.en.md#auto_scrub) (verification of copies)
-- [Checksums](../config/layout-osd.en.md#data_csum_type)
+- [Scrubbing without checksums](../config/osd.en.md#auto_scrub) (verification of copies)
 
 ## Plugins and tools
 
 - [Debian and CentOS packages](../installation/packages.en.md)
 - [Image management CLI (vitastor-cli)](../usage/cli.en.md)
-- [Disk management CLI (vitastor-disk)](../usage/disk.en.md)
+- [Disk management CLI (vitastor-disk)](docs/usage/disk.en.md)
 - Generic user-space client library
 - [Native QEMU driver](../usage/qemu.en.md)
 - [Loadable fio engine for benchmarks](../usage/fio.en.md)
@@ -56,6 +55,7 @@ The following features are planned for the future:
 - iSCSI proxy
 - Multi-threaded client
 - Faster failover
+- Checksums
 - Tiered storage (SSD caching)
 - NVDIMM support
 - Compression (possibly)

docs/intro/features.ru.md

@@ -13,7 +13,7 @@
 ## Серверные функции
 
 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
-- [Производительность](../performance/comparison1.ru.md) ;-D
+- [Производительность](../comparison1.ru.md) ;-D
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
   Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd
@@ -31,14 +31,13 @@
 - Снапшоты и copy-on-write клоны
 - [Сглаживание производительности случайной записи в SSD+HDD конфигурациях](../config/osd.ru.md#throttle_small_writes)
 - [Поддержка RDMA/RoCEv2 через libibverbs](../config/network.ru.md#rdma_device)
-- [Фоновая проверка целостности](../config/osd.ru.md#auto_scrub) (сверка копий)
-- [Контрольные суммы](../config/layout-osd.ru.md#data_csum_type)
+- [Фоновая проверка целостности без контрольных сумм](../config/osd.ru.md#auto_scrub) (сверка копий)
 
 ## Драйверы и инструменты
 
 - [Пакеты для Debian и CentOS](../installation/packages.ru.md)
 - [Консольный интерфейс управления образами (vitastor-cli)](../usage/cli.ru.md)
-- [Инструмент управления дисками (vitastor-disk)](../usage/disk.ru.md)
+- [Инструмент управления дисками (vitastor-disk)](docs/usage/disk.ru.md)
 - Общая пользовательская клиентская библиотека для работы с кластером
 - [Драйвер диска для QEMU](../usage/qemu.ru.md)
 - [Драйвер диска для утилиты тестирования производительности fio](../usage/fio.ru.md)
@@ -56,6 +55,7 @@
 - iSCSI-прокси
 - Многопоточный клиент
 - Более быстрое переключение при отказах
+- Контрольные суммы
 - Поддержка SSD-кэширования (tiered storage)
 - Поддержка NVDIMM
 - Возможно, сжатие

docs/intro/quickstart.en.md

@@ -7,7 +7,6 @@
 # Quick Start
 
 - [Preparation](#preparation)
-- [Recommended drives](#recommended-drives)
 - [Configure monitors](#configure-monitors)
 - [Configure OSDs](#configure-osds)
 - [Create a pool](#create-a-pool)
@@ -20,20 +19,10 @@
 - Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
   with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
   [here](../config/layout-cluster.en.md#immediate_commit).
-- If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
-  Toshiba MG08, Seagate EXOS or something similar. If your drives don't have such cache then
-  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
 - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
 - [Install Vitastor packages](../installation/packages.en.md).
 
-## Recommended drives
-
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
-- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
-- HDD: HGST Ultrastar, Toshiba MG06/MG07/MG08, Seagate EXOS
-
 ## Configure monitors
 
 On the monitor hosts:
@@ -56,10 +45,9 @@ On the monitor hosts:
   }
   ```
 - Initialize OSDs:
-  - SSD-only or HDD-only: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`.
-    Add `--disable_data_fsync off` to leave disk write cache enabled if you use
-    desktop SSDs without capacitors. Do NOT add `--disable_data_fsync off` if you
-    use HDDs or SSD+HDD.
+  - SSD-only: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`. You can add
+    `--disable_data_fsync off` to leave disk cache enabled if you use desktop
+    SSDs without capacitors.
   - Hybrid, SSD+HDD: `vitastor-disk prepare --hybrid /dev/sdXXX [/dev/sdYYY ...]`.
     Pass all your devices (HDD and SSD) to this script — it will partition disks and initialize journals on its own.
     This script skips HDDs which are already partitioned so if you want to use non-empty disks for

docs/intro/quickstart.ru.md

@@ -7,7 +7,6 @@
 # Быстрый старт
 
 - [Подготовка](#подготовка)
-- [Рекомендуемые диски](#рекомендуемые-диски)
 - [Настройте мониторы](#настройте-мониторы)
 - [Настройте OSD](#настройте-osd)
 - [Создайте пул](#создайте-пул)
@@ -20,20 +19,10 @@
 - Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
   использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
   О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
-- Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
-  Toshiba MG08, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
-  обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места).
 - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2.
 - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
 - [Установите пакеты Vitastor](../installation/packages.ru.md).
 
-## Рекомендуемые диски
-
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
-- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
-- HDD: HGST Ultrastar, Toshiba MG06/MG07/MG08, Seagate EXOS
-
 ## Настройте мониторы
 
 На хостах, выделенных под мониторы:
@@ -56,10 +45,9 @@
   }
   ```
 - Инициализуйте OSD:
-  - Только SSD или только HDD: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`.
-    Если вы используете десктопные SSD без конденсаторов, добавьте опцию `--disable_data_fsync off`,
-    чтобы оставить кэш записи диска включённым. НЕ добавляйте эту опцию, если используете
-    жёсткие диски (HDD).
+  - SSD: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`. Если вы используете
+    десктопные SSD без конденсаторов, можете оставить кэш включённым, добавив
+    опцию `--disable_data_fsync off`.
   - Гибридные, SSD+HDD: `vitastor-disk prepare --hybrid /dev/sdXXX [/dev/sdYYY ...]`.
     Передайте все ваши SSD и HDD скрипту в командной строке подряд, скрипт автоматически выделит
     разделы под журналы на SSD и данные на HDD. Скрипт пропускает HDD, на которых уже есть разделы

docs/usage/disk.en.md

@@ -86,8 +86,6 @@ Options (both modes):
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity
---data_csum_type none      Set data checksum type (crc32c or none)
---csum_block_size 4k       Set data checksum block size
 --data_device_block 4k     Override data device block size
 --meta_device_block 4k     Override metadata device block size
 --journal_device_block 4k  Override journal device block size
@@ -102,9 +100,8 @@ checks the device cache status on start and tries to disable cache for SATA/SAS
 If it doesn't succeed it issues a warning in the system log.
 
 You can also pass other OSD options here as arguments and they'll be persisted
-in the superblock: cached_io_data, cached_io_meta, cached_io_journal,
-inmemory_metadata, inmemory_journal, max_write_iodepth,
-min_flusher_count, max_flusher_count, journal_sector_buffer_count,
+to the superblock: max_write_iodepth, max_write_iodepth, min_flusher_count,
+max_flusher_count, inmemory_metadata, inmemory_journal, journal_sector_buffer_count,
 journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,
 throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.
 See [Runtime OSD Parameters](../config/osd.en.md) for details.
@@ -252,9 +249,7 @@ Options (see also [Cluster-Wide Disk Layout Parameters](../config/layout-cluster
 ```
 --object_size 128k       Set blockstore block size
 --bitmap_granularity 4k  Set bitmap granularity
---journal_size 16M       Set journal size
---data_csum_type none    Set data checksum type (crc32c or none)
---csum_block_size 4k     Set data checksum block size
+--journal_size 32M       Set journal size
 --device_block_size 4k   Set device block size
 --journal_offset 0       Set journal offset
 --device_size 0          Set device size

docs/usage/disk.ru.md

@@ -87,8 +87,6 @@ vitastor-disk - инструмент командной строки для уп
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт
---data_csum_type none      Задать тип контрольных сумм (crc32c или none)
---csum_block_size 4k       Задать размер блока расчёта контрольных сумм
 --data_device_block 4k     Задать размер блока устройства данных
 --meta_device_block 4k     Задать размер блока метаданных
 --journal_device_block 4k  Задать размер блока журнала
@@ -103,9 +101,8 @@ vitastor-disk - инструмент командной строки для уп
 это не удаётся, в системный журнал выводится предупреждение.
 
 Вы можете передать данной команде и некоторые другие опции OSD в качестве аргументов
-и они тоже будут сохранены в суперблок: cached_io_data, cached_io_meta,
-cached_io_journal, inmemory_metadata, inmemory_journal, max_write_iodepth,
-min_flusher_count, max_flusher_count, journal_sector_buffer_count,
+и они тоже будут сохранены в суперблок: max_write_iodepth, max_write_iodepth, min_flusher_count,
+max_flusher_count, inmemory_metadata, inmemory_journal, journal_sector_buffer_count,
 journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,
 throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.
 Читайте об этих параметрах подробнее в разделе [Изменяемые параметры OSD](../config/osd.ru.md).
@@ -257,9 +254,7 @@ OSD отключены fsync-и.
 ```
 --object_size 128k       Размер блока хранилища
 --bitmap_granularity 4k  Гранулярность битовых карт
---journal_size 16M       Размер журнала
---data_csum_type none    Задать тип контрольных сумм (crc32c или none)
---csum_block_size 4k     Задать размер блока расчёта контрольных сумм
+--journal_size 32M       Размер журнала
 --device_block_size 4k   Размер блока устройства
 --journal_offset 0       Смещение журнала
 --device_size 0          Размер устройства

docs/usage/nbd.en.md

@@ -13,8 +13,6 @@ remains decent (see an example [here](../performance/comparison1.en.md#vitastor-
 
 Vitastor Kubernetes CSI driver is based on NBD.
 
-See also [VDUSE](qemu.en.md#vduse).
-
 ## Map image
 
 To create a local block device for a Vitastor image run:

docs/usage/nbd.ru.md

@@ -16,8 +16,6 @@ NBD немного снижает производительность из-за
 
 CSI-драйвер Kubernetes Vitastor основан на NBD.
 
-Смотрите также [VDUSE](qemu.ru.md#vduse).
-
 ## Подключить устройство
 
 Чтобы создать локальное блочное устройство для образа, выполните команду:

docs/usage/nfs.ru.md

@@ -29,7 +29,7 @@ vitastor-nfs [--etcd_address ADDR] [ДРУГИЕ ОПЦИИ]
 --bind <IP>       принимать соединения по адресу <IP> (по умолчанию 0.0.0.0 - на всех)
 --nfspath <PATH>  установить путь NFS-экспорта в <PATH> (по умолчанию /)
 --port <PORT>     использовать порт <PORT> для NFS-сервисов (по умолчанию 2049)
---pool <POOL>     использовать пул <POOL> для новых образов (обязательно, если пул в кластере не один)
+--pool <POOL>     исползовать пул <POOL> для новых образов (обязательно, если пул в кластере не один)
 --foreground 1    не уходить в фон после запуска
 ```
 

docs/usage/qemu.en.md

@@ -83,44 +83,3 @@ qemu-img rebase -u -b '' testimg.qcow2
 This can be used for backups. Just note that exporting an image that is currently being written to
 is of course unsafe and doesn't produce a consistent result, so only export snapshots if you do this
 on a live VM.
-
-## VDUSE
-
-Linux kernel, starting with version 5.15, supports a new interface for attaching virtual disks
-to the host - VDUSE (vDPA Device in Userspace). QEMU, starting with 7.2, has support for
-exporting QEMU block devices over this protocol using qemu-storage-daemon.
-
-VDUSE has the same problem as other FUSE-like interfaces in Linux: if a userspace process hangs,
-for example, if it loses connectivity with Vitastor cluster - active processes doing I/O may
-hang in the D state (uninterruptible sleep) and you won't be able to kill them even with kill -9.
-In this case reboot will be the only way to remove VDUSE devices from system.
-
-On the other hand, VDUSE is faster than [NBD](nbd.en.md), so you may prefer to use it if
-performance is important for you. Approximate performance numbers:
-direct fio benchmark - 115000 iops, NBD - 60000 iops, VDUSE - 90000 iops.
-
-To try VDUSE you need at least Linux 5.15, built with VDUSE support
-(CONFIG_VIRTIO_VDPA=m and CONFIG_VDPA_USER=m). Debian Linux kernels have these options
-disabled by now, so if you want to try it on Debian, use a kernel from Ubuntu
-[kernel-ppa/mainline](https://kernel.ubuntu.com/~kernel-ppa/mainline/) or Proxmox.
-
-Commands to attach Vitastor image as a VDUSE device:
-
-```
-modprobe vduse
-modprobe virtio-vdpa
-qemu-storage-daemon --daemonize --blockdev '{"node-name":"test1","driver":"vitastor",\
-  "etcd-host":"192.168.7.2:2379/v3","image":"testosd1","cache":{"direct":true,"no-flush":false},"discard":"unmap"}' \
-  --export vduse-blk,id=test1,node-name=test1,name=test1,num-queues=16,queue-size=128,writable=true
-vdpa dev add name test1 mgmtdev vduse
-```
-
-After running these commands /dev/vda device will appear in the system and you'll be able to
-use it as a normal disk.
-
-To remove the device:
-
-```
-vdpa dev del test1
-kill <qemu-storage-daemon_process_PID>
-```

docs/usage/qemu.ru.md

@@ -87,44 +87,3 @@ qemu-img rebase -u -b '' testimg.qcow2
 Это можно использовать для резервного копирования. Только помните, что экспортировать образ, в который
 в то же время идёт запись, небезопасно - результат чтения не будет целостным. Так что если вы работаете
 с активными виртуальными машинами, экспортируйте только их снимки, но не сам образ.
-
-## VDUSE
-
-В Linux, начиная с версии ядра 5.15, доступен новый интерфейс для подключения виртуальных дисков
-к системе - VDUSE (vDPA Device in Userspace), а в QEMU, начиная с версии 7.2, есть поддержка
-экспорта блочных устройств QEMU по этому протоколу через qemu-storage-daemon.
-
-VDUSE страдает общей проблемой FUSE-подобных интерфейсов в Linux: если пользовательский процесс
-подвиснет, например, если будет потеряна связь с кластером Vitastor - читающие/пишущие в кластер
-процессы могут "залипнуть" в состоянии D (непрерываемый сон) и их будет невозможно убить даже
-через kill -9. В этом случае удалить из системы устройство можно только перезагрузившись.
-
-С другой стороны, VDUSE быстрее по сравнению с [NBD](nbd.ru.md), поэтому его может
-быть предпочтительно использовать там, где производительность важнее. Порядок показателей:
-прямое тестирование через fio - 115000 iops, NBD - 60000 iops, VDUSE - 90000 iops.
-
-Чтобы использовать VDUSE, вам нужно ядро Linux версии хотя бы 5.15, собранное с поддержкой
-VDUSE (CONFIG_VIRTIO_VDPA=m и CONFIG_VDPA_USER=m). В ядрах в Debian Linux поддержка пока
-отключена - если хотите попробовать эту функцию на Debian, поставьте ядро из Ubuntu
-[kernel-ppa/mainline](https://kernel.ubuntu.com/~kernel-ppa/mainline/) или из Proxmox.
-
-Команды для подключения виртуального диска через VDUSE:
-
-```
-modprobe vduse
-modprobe virtio-vdpa
-qemu-storage-daemon --daemonize --blockdev '{"node-name":"test1","driver":"vitastor",\
-  "etcd-host":"192.168.7.2:2379/v3","image":"testosd1","cache":{"direct":true,"no-flush":false},"discard":"unmap"}' \
-  --export vduse-blk,id=test1,node-name=test1,name=test1,num-queues=16,queue-size=128,writable=true
-vdpa dev add name test1 mgmtdev vduse
-```
-
-После этого в системе появится устройство /dev/vda, которое можно будет использовать как
-обычный диск.
-
-Для удаления устройства из системы:
-
-```
-vdpa dev del test1
-kill <PID_процесса_qemu-storage-daemon>
-```

mon/make-etcd

@@ -63,9 +63,8 @@ Wants=network-online.target local-fs.target time-sync.target
 
 [Service]
 Restart=always
-Environment=GOGC=50
-ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
-    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
+ExecStart=/usr/local/bin/etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
+    --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
     --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
     --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
     --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\

mon/mon.js

@@ -391,7 +391,6 @@ class Mon
         this.etcd_start_timeout = (config.etcd_start_timeout || 5) * 1000;
         this.state = JSON.parse(JSON.stringify(this.constructor.etcd_tree));
         this.signals_set = false;
-        this.stat_time = Date.now();
         this.ws = null;
         this.ws_alive = false;
         this.ws_keepalive_timer = null;
@@ -539,18 +538,10 @@ class Mon
         {
             retries = 1;
         }
-        const tried = {};
         while (retries < 0 || retry < retries)
         {
             const cur_addr = this.pick_next_etcd();
             const base = 'ws'+cur_addr.substr(4);
-            let now = Date.now();
-            if (tried[base] && now-tried[base] < timeout)
-            {
-                await new Promise(ok => setTimeout(ok, timeout-(now-tried[base])));
-                now = Date.now();
-            }
-            tried[base] = now;
             const ok = await new Promise((ok, no) =>
             {
                 const timer_id = setTimeout(() =>
@@ -1419,75 +1410,65 @@ class Mon
         }
     }
 
-    derive_osd_stats(st, prev)
-    {
-        const zero_stats = { op: { bps: 0n, iops: 0n, lat: 0n }, subop: { iops: 0n, lat: 0n }, recovery: { bps: 0n, iops: 0n } };
-        const diff = { op_stats: {}, subop_stats: {}, recovery_stats: {} };
-        if (!st || !st.time || prev && (prev.time || this.stat_time/1000) >= st.time)
-        {
-            return diff;
-        }
-        const timediff = BigInt(st.time*1000 - (prev && prev.time*1000 || this.stat_time));
-        for (const op in st.op_stats||{})
-        {
-            const pr = prev && prev.op_stats && prev.op_stats[op];
-            let c = st.op_stats[op];
-            c = { bytes: BigInt(c.bytes||0), usec: BigInt(c.usec||0), count: BigInt(c.count||0) };
-            const b = c.bytes - BigInt(pr && pr.bytes||0);
-            const us = c.usec - BigInt(pr && pr.usec||0);
-            const n = c.count - BigInt(pr && pr.count||0);
-            if (n > 0)
-                diff.op_stats[op] = { ...c, bps: b*1000n/timediff, iops: n*1000n/timediff, lat: us/n };
-        }
-        for (const op in st.subop_stats||{})
-        {
-            const pr = prev && prev.subop_stats && prev.subop_stats[op];
-            let c = st.subop_stats[op];
-            c = { usec: BigInt(c.usec||0), count: BigInt(c.count||0) };
-            const us = c.usec - BigInt(pr && pr.usec||0);
-            const n = c.count - BigInt(pr && pr.count||0);
-            if (n > 0)
-                diff.subop_stats[op] = { ...c, iops: n*1000n/timediff, lat: us/n };
-        }
-        for (const op in st.recovery_stats||{})
-        {
-            const pr = prev && prev.recovery_stats && prev.recovery_stats[op];
-            let c = st.recovery_stats[op];
-            c = { bytes: BigInt(c.bytes||0), count: BigInt(c.count||0) };
-            const b = c.bytes - BigInt(pr && pr.bytes||0);
-            const n = c.count - BigInt(pr && pr.count||0);
-            if (n > 0)
-                diff.recovery_stats[op] = { ...c, bps: b*1000n/timediff, iops: n*1000n/timediff };
-        }
-        return diff;
-    }
-
     sum_op_stats(timestamp, prev_stats)
     {
-        const sum_diff = { op_stats: {}, subop_stats: {}, recovery_stats: {} };
-        if (!prev_stats || prev_stats.timestamp >= timestamp)
-        {
-            return sum_diff;
-        }
-        const tm = BigInt(timestamp - (prev_stats.timestamp || 0));
-        // Sum derived values instead of deriving summed
+        const op_stats = {}, subop_stats = {}, recovery_stats = {};
         for (const osd in this.state.osd.stats)
         {
-            const derived = this.derive_osd_stats(this.state.osd.stats[osd],
-                this.prev_stats && this.prev_stats.osd_stats && this.prev_stats.osd_stats[osd]);
-            for (const type in derived)
+            const st = this.state.osd.stats[osd]||{};
+            for (const op in st.op_stats||{})
             {
-                for (const op in derived[type])
-                {
-                    for (const k in derived[type][op])
-                    {
-                        sum_diff[type][op] = sum_diff[type][op] || {};
-                        sum_diff[type][op][k] = (sum_diff[type][op][k] || 0n) + derived[type][op][k];
-                    }
-                }
+                op_stats[op] = op_stats[op] || { count: 0n, usec: 0n, bytes: 0n };
+                op_stats[op].count += BigInt(st.op_stats[op].count||0);
+                op_stats[op].usec += BigInt(st.op_stats[op].usec||0);
+                op_stats[op].bytes += BigInt(st.op_stats[op].bytes||0);
+            }
+            for (const op in st.subop_stats||{})
+            {
+                subop_stats[op] = subop_stats[op] || { count: 0n, usec: 0n };
+                subop_stats[op].count += BigInt(st.subop_stats[op].count||0);
+                subop_stats[op].usec += BigInt(st.subop_stats[op].usec||0);
+            }
+            for (const op in st.recovery_stats||{})
+            {
+                recovery_stats[op] = recovery_stats[op] || { count: 0n, bytes: 0n };
+                recovery_stats[op].count += BigInt(st.recovery_stats[op].count||0);
+                recovery_stats[op].bytes += BigInt(st.recovery_stats[op].bytes||0);
             }
         }
-        return sum_diff;
+        if (prev_stats && prev_stats.timestamp >= timestamp)
+        {
+            prev_stats = null;
+        }
+        const tm = prev_stats ? BigInt(timestamp - prev_stats.timestamp) : 0;
+        for (const op in op_stats)
+        {
+            if (prev_stats && prev_stats.op_stats && prev_stats.op_stats[op])
+            {
+                op_stats[op].bps = (op_stats[op].bytes - prev_stats.op_stats[op].bytes) * 1000n / tm;
+                op_stats[op].iops = (op_stats[op].count - prev_stats.op_stats[op].count) * 1000n / tm;
+                op_stats[op].lat = (op_stats[op].usec - prev_stats.op_stats[op].usec)
+                    / ((op_stats[op].count - prev_stats.op_stats[op].count) || 1n);
+            }
+        }
+        for (const op in subop_stats)
+        {
+            if (prev_stats && prev_stats.subop_stats && prev_stats.subop_stats[op])
+            {
+                subop_stats[op].iops = (subop_stats[op].count - prev_stats.subop_stats[op].count) * 1000n / tm;
+                subop_stats[op].lat = (subop_stats[op].usec - prev_stats.subop_stats[op].usec)
+                    / ((subop_stats[op].count - prev_stats.subop_stats[op].count) || 1n);
+            }
+        }
+        for (const op in recovery_stats)
+        {
+            if (prev_stats && prev_stats.recovery_stats && prev_stats.recovery_stats[op])
+            {
+                recovery_stats[op].bps = (recovery_stats[op].bytes - prev_stats.recovery_stats[op].bytes) * 1000n / tm;
+                recovery_stats[op].iops = (recovery_stats[op].count - prev_stats.recovery_stats[op].count) * 1000n / tm;
+            }
+        }
+        return { op_stats, subop_stats, recovery_stats };
     }
 
     sum_object_counts()
@@ -1505,14 +1486,10 @@ class Mon
                     break;
                 }
             }
-            const pool_cfg = (this.state.config.pools[pool_id]||{});
             if (!object_size)
             {
-                object_size = pool_cfg.block_size || this.config.block_size || 131072;
-            }
-            if (pool_cfg.scheme !== 'replicated')
-            {
-                object_size *= ((pool_cfg.pg_size||0) - (pool_cfg.parity_chunks||0));
+                object_size = (this.state.config.pools[pool_id]||{}).block_size ||
+                    this.config.block_size || 131072;
             }
             object_size = BigInt(object_size);
             for (const pg_num in this.state.pg.stats[pool_id])
@@ -1620,7 +1597,7 @@ class Mon
                 }
             }
         }
-        return { inode_stats, seen_pools };
+        return inode_stats;
     }
 
     serialize_bigints(obj)
@@ -1646,12 +1623,11 @@ class Mon
         const timestamp = Date.now();
         const { object_counts, object_bytes } = this.sum_object_counts();
         let stats = this.sum_op_stats(timestamp, this.prev_stats);
-        let { inode_stats, seen_pools } = this.sum_inode_stats(
+        let inode_stats = this.sum_inode_stats(
             this.prev_stats ? this.prev_stats.inode_stats : null,
             timestamp, this.prev_stats ? this.prev_stats.timestamp : null
         );
-        this.prev_stats = { timestamp, inode_stats, osd_stats: { ...this.state.osd.stats } };
-        this.stat_time = Date.now();
+        this.prev_stats = { timestamp, ...stats, inode_stats };
         stats.object_counts = object_counts;
         stats.object_bytes = object_bytes;
         stats = this.serialize_bigints(stats);
@@ -1681,22 +1657,12 @@ class Mon
         }
         for (const pool_id in this.state.pool.stats)
         {
-            if (!seen_pools[pool_id])
-            {
-                txn.push({ requestDeleteRange: {
-                    key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
-                } });
-                delete this.state.pool.stats[pool_id];
-            }
-            else
-            {
-                const pool_stats = { ...this.state.pool.stats[pool_id] };
-                this.serialize_bigints(pool_stats);
-                txn.push({ requestPut: {
-                    key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
-                    value: b64(JSON.stringify(pool_stats)),
-                } });
-            }
+            const pool_stats = { ...this.state.pool.stats[pool_id] };
+            this.serialize_bigints(pool_stats);
+            txn.push({ requestPut: {
+                key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
+                value: b64(JSON.stringify(pool_stats)),
+            } });
         }
         if (txn.length)
         {
@@ -1777,14 +1743,13 @@ class Mon
         else if (key_parts[0] === 'osd' && key_parts[1] === 'stats')
         {
             // Recheck OSD tree on OSD addition/deletion
-            const osd_num = key_parts[2];
             if ((!old) != (!kv.value) || old && kv.value && old.size != kv.value.size)
             {
                 this.schedule_recheck();
             }
             // Recheck PGs <osd_out_time> after last OSD statistics report
             this.schedule_next_recheck_at(
-                !this.state.osd.stats[osd_num] ? 0 : this.state.osd.stats[osd_num].time+this.config.osd_out_time
+                !this.state.osd.stats[key[2]] ? 0 : this.state.osd.stats[key[2]].time+this.config.osd_out_time
             );
         }
     }
@@ -1796,18 +1761,10 @@ class Mon
         {
             retries = 1;
         }
-        const tried = {};
         while (retries < 0 || retry < retries)
         {
             retry++;
             const base = this.pick_next_etcd();
-            let now = Date.now();
-            if (tried[base] && now-tried[base] < timeout)
-            {
-                await new Promise(ok => setTimeout(ok, timeout-(now-tried[base])));
-                now = Date.now();
-            }
-            tried[base] = now;
             const res = await POST(base+path, body, timeout);
             if (res.error)
             {

patches/VitastorPlugin.pm

@@ -388,6 +388,8 @@ sub unmap_volume
     my ($class, $storeid, $scfg, $volname, $snapname) = @_;
     my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/';
 
+    return 1 if !$scfg->{vitastor_nbd};
+
     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
     $name .= '@'.$snapname if $snapname;
 
@@ -411,7 +413,7 @@ sub activate_volume
 sub deactivate_volume
 {
     my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
-    $class->unmap_volume($storeid, $scfg, $volname, $snapname) if $scfg->{vitastor_nbd};
+    $class->unmap_volume($storeid, $scfg, $volname, $snapname);
     return 1;
 }
 

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '1.0.0'
+VERSION = '0.9.0'
 
 LOG = logging.getLogger(__name__)
 

patches/libvirt-9.0-vitastor.diff

@@ -1,644 +0,0 @@
-commit e6f935157944279c2c0634915c3c00feeec748c9
-Author: Vitaliy Filippov <vitalif@yourcmc.ru>
-Date:   Mon Jun 19 00:58:19 2023 +0300
-
-    Add Vitastor support
-
-diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
-index aaad4a3..5f5daa8 100644
---- a/include/libvirt/libvirt-storage.h
-+++ b/include/libvirt/libvirt-storage.h
-@@ -326,6 +326,7 @@ typedef enum {
-     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
-+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
- } virConnectListAllStoragePoolsFlags;
- 
- int                     virConnectListAllStoragePools(virConnectPtr conn,
-diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
-index 45965fa..b7c23d3 100644
---- a/src/conf/domain_conf.c
-+++ b/src/conf/domain_conf.c
-@@ -7103,7 +7103,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
-     src->configFile = virXPathString("string(./config/@file)", ctxt);
- 
-     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
--        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
-         src->query = virXMLPropString(node, "query");
- 
-     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
-@@ -30121,6 +30122,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
- 
-     case VIR_STORAGE_POOL_MPATH:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_SHEEPDOG:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_LAST:
-diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
-index 5a9bf20..05058b8 100644
---- a/src/conf/domain_validate.c
-+++ b/src/conf/domain_validate.c
-@@ -494,6 +494,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-@@ -541,7 +542,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         }
-     }
- 
--    /* internal snapshots and config files are currently supported only with rbd: */
-+    /* internal snapshots are currently supported only with rbd: */
-     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
-         if (src->snapshot) {
-@@ -550,11 +551,15 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-                              "only with 'rbd' disks"));
-             return -1;
-         }
--
-+    }
-+    /* config files are currently supported only with rbd and vitastor: */
-+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
-         if (src->configFile) {
-             virReportError(VIR_ERR_XML_ERROR, "%s",
-                            _("<config> element is currently supported "
--                             "only with 'rbd' disks"));
-+                             "only with 'rbd' and 'vitastor' disks"));
-             return -1;
-         }
-     }
-diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
-index 6cb0a20..8bf7de9 100644
---- a/src/conf/schemas/domaincommon.rng
-+++ b/src/conf/schemas/domaincommon.rng
-@@ -1972,6 +1972,35 @@
-     </element>
-   </define>
- 
-+  <define name="diskSourceNetworkProtocolVitastor">
-+    <element name="source">
-+      <interleave>
-+        <attribute name="protocol">
-+          <value>vitastor</value>
-+        </attribute>
-+        <ref name="diskSourceCommon"/>
-+        <optional>
-+          <attribute name="name"/>
-+        </optional>
-+        <optional>
-+          <attribute name="query"/>
-+        </optional>
-+        <zeroOrMore>
-+          <ref name="diskSourceNetworkHost"/>
-+        </zeroOrMore>
-+        <optional>
-+          <element name="config">
-+            <attribute name="file">
-+              <ref name="absFilePath"/>
-+            </attribute>
-+            <empty/>
-+          </element>
-+        </optional>
-+        <empty/>
-+      </interleave>
-+    </element>
-+  </define>
-+
-   <define name="diskSourceNetworkProtocolISCSI">
-     <element name="source">
-       <attribute name="protocol">
-@@ -2264,6 +2293,7 @@
-       <ref name="diskSourceNetworkProtocolSimple"/>
-       <ref name="diskSourceNetworkProtocolVxHS"/>
-       <ref name="diskSourceNetworkProtocolNFS"/>
-+      <ref name="diskSourceNetworkProtocolVitastor"/>
-     </choice>
-   </define>
- 
-diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
-index f5a9636..8339bc4 100644
---- a/src/conf/storage_conf.c
-+++ b/src/conf/storage_conf.c
-@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
-               "logical", "disk", "iscsi",
-               "iscsi-direct", "scsi", "mpath",
-               "rbd", "sheepdog", "gluster",
--              "zfs", "vstorage",
-+              "zfs", "vstorage", "vitastor",
- );
- 
- VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
-@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
-           .formatToString = virStorageFileFormatTypeToString,
-       }
-     },
-+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
-+     .poolOptions = {
-+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
-+                   VIR_STORAGE_POOL_SOURCE_NAME),
-+      },
-+      .volOptions = {
-+          .defaultFormat = VIR_STORAGE_FILE_RAW,
-+          .formatFromString = virStorageVolumeFormatFromString,
-+          .formatToString = virStorageFileFormatTypeToString,
-+      }
-+    },
-     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
-      .poolOptions = {
-          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-@@ -542,6 +554,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
-                        _("element 'name' is mandatory for RBD pool"));
-         return -1;
-     }
-+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
-+        virReportError(VIR_ERR_XML_ERROR, "%s",
-+                       _("element 'name' is mandatory for Vitastor pool"));
-+        return -1;
-+    }
- 
-     if (options->formatFromString) {
-         g_autofree char *format = NULL;
-@@ -1132,6 +1149,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
-     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
-      * files, so they don't have a target */
-     if (def->type != VIR_STORAGE_POOL_RBD &&
-+        def->type != VIR_STORAGE_POOL_VITASTOR &&
-         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
-         def->type != VIR_STORAGE_POOL_GLUSTER &&
-         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
-diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
-index fc67957..720c07e 100644
---- a/src/conf/storage_conf.h
-+++ b/src/conf/storage_conf.h
-@@ -103,6 +103,7 @@ typedef enum {
-     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
-     VIR_STORAGE_POOL_ZFS,      /* ZFS */
-     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
-+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
- 
-     VIR_STORAGE_POOL_LAST,
- } virStoragePoolType;
-@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
-+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
-diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
-index cecd7e8..d7b79a4 100644
---- a/src/conf/storage_source_conf.c
-+++ b/src/conf/storage_source_conf.c
-@@ -87,6 +87,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
-               "ssh",
-               "vxhs",
-               "nfs",
-+              "vitastor",
- );
- 
- 
-@@ -1286,6 +1287,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-             return 24007;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             /* we don't provide a default for RBD */
-             return 0;
-diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
-index 14a6825..eb4acac 100644
---- a/src/conf/storage_source_conf.h
-+++ b/src/conf/storage_source_conf.h
-@@ -128,6 +128,7 @@ typedef enum {
-     VIR_STORAGE_NET_PROTOCOL_SSH,
-     VIR_STORAGE_NET_PROTOCOL_VXHS,
-     VIR_STORAGE_NET_PROTOCOL_NFS,
-+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
- 
-     VIR_STORAGE_NET_PROTOCOL_LAST
- } virStorageNetProtocol;
-diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
-index e6c187e..035b423 100644
---- a/src/conf/virstorageobj.c
-+++ b/src/conf/virstorageobj.c
-@@ -1433,6 +1433,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
-             return 1;
-         break;
- 
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_RBD:
-     case VIR_STORAGE_POOL_LAST:
-@@ -1918,6 +1919,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
-                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
-                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
-+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
-+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
-                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
-diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
-index 8490034..ab2cdaa 100644
---- a/src/libvirt-storage.c
-+++ b/src/libvirt-storage.c
-@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
-  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
-  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
-+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
-  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
-  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
-diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
-index 17ac880..59711b5 100644
---- a/src/libxl/libxl_conf.c
-+++ b/src/libxl/libxl_conf.c
-@@ -970,6 +970,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
-index 6919325..55ffc32 100644
---- a/src/libxl/xen_xl.c
-+++ b/src/libxl/xen_xl.c
-@@ -1445,6 +1445,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
-index e865aa1..40162af 100644
---- a/src/qemu/qemu_block.c
-+++ b/src/qemu/qemu_block.c
-@@ -604,6 +604,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
- }
- 
- 
-+static virJSONValue *
-+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
-+{
-+    virJSONValue *ret = NULL;
-+    virStorageNetHostDef *host;
-+    size_t i;
-+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-+    g_autofree char *etcd = NULL;
-+
-+    for (i = 0; i < src->nhosts; i++) {
-+        host = src->hosts + i;
-+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
-+            return NULL;
-+        }
-+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
-+    }
-+    if (src->nhosts > 0) {
-+        etcd = virBufferContentAndReset(&buf);
-+    }
-+
-+    if (virJSONValueObjectAdd(&ret,
-+                              "S:etcd-host", etcd,
-+                              "S:etcd-prefix", src->query,
-+                              "S:config-path", src->configFile,
-+                              "s:image", src->path,
-+                              NULL) < 0)
-+        return NULL;
-+
-+    return ret;
-+}
-+
-+
- static virJSONValue *
- qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src)
- {
-@@ -917,6 +949,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
-                 return NULL;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return NULL;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             driver = "sheepdog";
-             if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src)))
-@@ -1860,6 +1898,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
- 
-             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             case VIR_STORAGE_NET_PROTOCOL_RBD:
-+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-             case VIR_STORAGE_NET_PROTOCOL_VXHS:
-             case VIR_STORAGE_NET_PROTOCOL_NFS:
-             case VIR_STORAGE_NET_PROTOCOL_SSH:
-@@ -2242,6 +2281,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
-                 return -1;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return -1;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             driver = "sheepdog";
-             if (!(location = qemuBlockStorageSourceGetSheepdogProps(src)))
-diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
-index 2eb5653..60ee82d 100644
---- a/src/qemu/qemu_domain.c
-+++ b/src/qemu/qemu_domain.c
-@@ -4958,7 +4958,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
-     if (src->query &&
-         (actualType != VIR_STORAGE_TYPE_NETWORK ||
-          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
--          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
-         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                        _("query is supported only with HTTP(S) protocols"));
-         return -1;
-@@ -10129,6 +10130,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
-         break;
- 
-     case VIR_STORAGE_NET_PROTOCOL_RBD:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
-index b841680..a6be771 100644
---- a/src/qemu/qemu_snapshot.c
-+++ b/src/qemu/qemu_snapshot.c
-@@ -373,6 +373,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-@@ -578,6 +579,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
-index d90c1c9..e853457 100644
---- a/src/storage/storage_driver.c
-+++ b/src/storage/storage_driver.c
-@@ -1627,6 +1627,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
- 
-         case VIR_STORAGE_POOL_GLUSTER:
-         case VIR_STORAGE_POOL_RBD:
-+        case VIR_STORAGE_POOL_VITASTOR:
-         case VIR_STORAGE_POOL_SHEEPDOG:
-         case VIR_STORAGE_POOL_ZFS:
-         case VIR_STORAGE_POOL_LAST:
-diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
-index e48ae72..2017ccc 100644
---- a/src/storage_file/storage_source_backingstore.c
-+++ b/src/storage_file/storage_source_backingstore.c
-@@ -284,6 +284,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
- }
- 
- 
-+static int
-+virStorageSourceParseVitastorColonString(const char *colonstr,
-+                                         virStorageSource *src)
-+{
-+    char *p, *e, *next;
-+    g_autofree char *options = NULL;
-+
-+    /* optionally skip the "vitastor:" prefix if provided */
-+    if (STRPREFIX(colonstr, "vitastor:"))
-+        colonstr += strlen("vitastor:");
-+
-+    options = g_strdup(colonstr);
-+
-+    p = options;
-+    while (*p) {
-+        /* find : delimiter or end of string */
-+        for (e = p; *e && *e != ':'; ++e) {
-+            if (*e == '\\') {
-+                e++;
-+                if (*e == '\0')
-+                    break;
-+            }
-+        }
-+        if (*e == '\0') {
-+            next = e;    /* last kv pair */
-+        } else {
-+            next = e + 1;
-+            *e = '\0';
-+        }
-+
-+        if (STRPREFIX(p, "image=")) {
-+            src->path = g_strdup(p + strlen("image="));
-+        } else if (STRPREFIX(p, "etcd-prefix=")) {
-+            src->query = g_strdup(p + strlen("etcd-prefix="));
-+        } else if (STRPREFIX(p, "config-path=")) {
-+            src->configFile = g_strdup(p + strlen("config-path="));
-+        } else if (STRPREFIX(p, "etcd-host=")) {
-+            char *h, *sep;
-+
-+            h = p + strlen("etcd-host=");
-+            while (h < e) {
-+                for (sep = h; sep < e; ++sep) {
-+                    if (*sep == '\\' && (sep[1] == ',' ||
-+                                         sep[1] == ';' ||
-+                                         sep[1] == ' ')) {
-+                        *sep = '\0';
-+                        sep += 2;
-+                        break;
-+                    }
-+                }
-+
-+                if (virStorageSourceRBDAddHost(src, h) < 0)
-+                    return -1;
-+
-+                h = sep;
-+            }
-+        }
-+
-+        p = next;
-+    }
-+
-+    if (!src->path) {
-+        return -1;
-+    }
-+
-+    return 0;
-+}
-+
-+
- static int
- virStorageSourceParseNBDColonString(const char *nbdstr,
-                                     virStorageSource *src)
-@@ -396,6 +465,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
-             return -1;
-         break;
- 
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
-+            return -1;
-+        break;
-+
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-@@ -984,6 +1058,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
-     return 0;
- }
- 
-+static int
-+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
-+                                         virJSONValue *json,
-+                                         const char *jsonstr G_GNUC_UNUSED,
-+                                         int opaque G_GNUC_UNUSED)
-+{
-+    const char *filename;
-+    const char *image = virJSONValueObjectGetString(json, "image");
-+    const char *conf = virJSONValueObjectGetString(json, "config-path");
-+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
-+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
-+    size_t nservers;
-+    size_t i;
-+
-+    src->type = VIR_STORAGE_TYPE_NETWORK;
-+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
-+
-+    /* legacy syntax passed via 'filename' option */
-+    if ((filename = virJSONValueObjectGetString(json, "filename")))
-+        return virStorageSourceParseVitastorColonString(filename, src);
-+
-+    if (!image) {
-+        virReportError(VIR_ERR_INVALID_ARG, "%s",
-+                       _("missing image name in Vitastor backing volume "
-+                         "JSON specification"));
-+        return -1;
-+    }
-+
-+    src->path = g_strdup(image);
-+    src->configFile = g_strdup(conf);
-+    src->query = g_strdup(etcd_prefix);
-+
-+    if (servers) {
-+        nservers = virJSONValueArraySize(servers);
-+
-+        src->hosts = g_new0(virStorageNetHostDef, nservers);
-+        src->nhosts = nservers;
-+
-+        for (i = 0; i < nservers; i++) {
-+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
-+                                                                  virJSONValueArrayGet(servers, i)) < 0)
-+                return -1;
-+        }
-+    }
-+
-+    return 0;
-+}
-+
- static int
- virStorageSourceParseBackingJSONRaw(virStorageSource *src,
-                                     virJSONValue *json,
-@@ -1162,6 +1284,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
-     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
-     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
-     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
-+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
-     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
-     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
-     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
-diff --git a/src/test/test_driver.c b/src/test/test_driver.c
-index bd6f063..cce34e1 100644
---- a/src/test/test_driver.c
-+++ b/src/test/test_driver.c
-@@ -7338,6 +7338,7 @@ testStorageVolumeTypeForPool(int pooltype)
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-         return VIR_STORAGE_VOL_NETWORK;
-     case VIR_STORAGE_POOL_LOGICAL:
-     case VIR_STORAGE_POOL_DISK:
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-index eee75af..8bd0a57 100644
---- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='no'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-index 805950a..852df0d 100644
---- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='yes'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
-index e8e40d6..db55fe5 100644
---- a/tests/storagepoolxml2argvtest.c
-+++ b/tests/storagepoolxml2argvtest.c
-@@ -65,6 +65,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_ZFS:
-     case VIR_STORAGE_POOL_VSTORAGE:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_LAST:
-     default:
-         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
-diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
-index 8a98c6a..4b1bbd4 100644
---- a/tools/virsh-pool.c
-+++ b/tools/virsh-pool.c
-@@ -1221,6 +1221,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
-             case VIR_STORAGE_POOL_VSTORAGE:
-                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
-                 break;
-+            case VIR_STORAGE_POOL_VITASTOR:
-+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
-+                break;
-             case VIR_STORAGE_POOL_LAST:
-                 break;
-             }

patches/pve-qemu-8.0-vitastor.patch

@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 382bec0e7d..af6207dbce 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -114,6 +114,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index c44d05a13f..ebedb42843 100644
---- a/meson.build
-+++ b/meson.build
-@@ -1028,6 +1028,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1882,6 +1902,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
-@@ -4020,6 +4041,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index fc9447d267..c4ac55c283 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -173,6 +173,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index c05ad0c07e..f5eb701604 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3308,7 +3308,7 @@
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
-             'pbs',
--            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4338,6 +4338,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4787,6 +4809,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5187,6 +5210,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5385,6 +5419,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index 6e8983f39c..1b0b9fcf3e 100755
---- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -32,7 +32,7 @@
- --with-git=meson \
- --with-git-submodules=update \
- --target-list="x86_64-softmmu" \
----block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -179,6 +179,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 009fab1515..95914e6ebc 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -144,6 +144,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  sdl             SDL user interface'
-@@ -392,6 +393,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-replication) printf "%s" -Dreplication=enabled ;;

patches/qemu-2.12-vitastor.patch

@@ -1,176 +0,0 @@
-diff --git a/block/Makefile.objs b/block/Makefile.objs
-index d644bac60a..e404236291 100644
---- a/block/Makefile.objs
-+++ b/block/Makefile.objs
-@@ -19,6 +19,7 @@ block-obj-$(if $(CONFIG_LIBISCSI),y,n) += iscsi-opts.o
- block-obj-$(CONFIG_LIBNFS) += nfs.o
- block-obj-$(CONFIG_CURL) += curl.o
- block-obj-$(CONFIG_RBD) += rbd.o
-+block-obj-$(CONFIG_VITASTOR) += vitastor.o
- block-obj-$(CONFIG_GLUSTERFS) += gluster.o
- block-obj-$(CONFIG_VXHS) += vxhs.o
- block-obj-$(CONFIG_LIBSSH2) += ssh.o
-@@ -39,6 +40,8 @@ curl.o-cflags      := $(CURL_CFLAGS)
- curl.o-libs        := $(CURL_LIBS)
- rbd.o-cflags       := $(RBD_CFLAGS)
- rbd.o-libs         := $(RBD_LIBS)
-+vitastor.o-cflags  := $(VITASTOR_CFLAGS)
-+vitastor.o-libs    := $(VITASTOR_LIBS)
- gluster.o-cflags   := $(GLUSTERFS_CFLAGS)
- gluster.o-libs     := $(GLUSTERFS_LIBS)
- vxhs.o-libs        := $(VXHS_LIBS)
-diff --git a/configure b/configure
-index 0a19b033bc..58b7fbf24c 100755
---- a/configure
-+++ b/configure
-@@ -398,6 +398,7 @@ trace_backends="log"
- trace_file="trace"
- spice=""
- rbd=""
-+vitastor=""
- smartcard=""
- libusb=""
- usb_redir=""
-@@ -1213,6 +1214,10 @@ for opt do
-   ;;
-   --enable-rbd) rbd="yes"
-   ;;
-+  --disable-vitastor) vitastor="no"
-+  ;;
-+  --enable-vitastor) vitastor="yes"
-+  ;;
-   --disable-xfsctl) xfs="no"
-   ;;
-   --enable-xfsctl) xfs="yes"
-@@ -1601,6 +1606,7 @@ disabled with --disable-FEATURE, default is enabled if available:
-   vhost-crypto    vhost-crypto acceleration support
-   spice           spice
-   rbd             rados block device (rbd)
-+  vitastor        vitastor block device
-   libiscsi        iscsi support
-   libnfs          nfs support
-   smartcard       smartcard support (libcacard)
-@@ -3594,6 +3600,27 @@ EOF
-   fi
- fi
- 
-+##########################################
-+# vitastor probe
-+if test "$vitastor" != "no" ; then
-+  cat > $TMPC <<EOF
-+#include <vitastor_c.h>
-+int main(void) {
-+  vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+  return 0;
-+}
-+EOF
-+  vitastor_libs="-lvitastor_client"
-+  if compile_prog "" "$vitastor_libs" ; then
-+    vitastor=yes
-+  else
-+    if test "$vitastor" = "yes" ; then
-+      feature_not_found "vitastor block device" "Install vitastor-client-dev"
-+    fi
-+    vitastor=no
-+  fi
-+fi
-+
- ##########################################
- # libssh2 probe
- min_libssh2_version=1.2.8
-@@ -5837,6 +5864,7 @@ echo "Trace output file $trace_file-<pid>"
- fi
- echo "spice support     $spice $(echo_version $spice $spice_protocol_version/$spice_server_version)"
- echo "rbd support       $rbd"
-+echo "vitastor support  $vitastor"
- echo "xfsctl support    $xfs"
- echo "smartcard support $smartcard"
- echo "libusb            $libusb"
-@@ -6416,6 +6444,11 @@ if test "$rbd" = "yes" ; then
-   echo "RBD_CFLAGS=$rbd_cflags" >> $config_host_mak
-   echo "RBD_LIBS=$rbd_libs" >> $config_host_mak
- fi
-+if test "$vitastor" = "yes" ; then
-+  echo "CONFIG_VITASTOR=m" >> $config_host_mak
-+  echo "VITASTOR_CFLAGS=$vitastor_cflags" >> $config_host_mak
-+  echo "VITASTOR_LIBS=$vitastor_libs" >> $config_host_mak
-+fi
- 
- echo "CONFIG_COROUTINE_BACKEND=$coroutine" >> $config_host_mak
- if test "$coroutine_pool" = "yes" ; then
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index c50517bff3..c780bb2c1c 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -2514,7 +2514,7 @@
-             'dmg', 'file', 'ftp', 'ftps', 'gluster', 'host_cdrom',
-             'host_device', 'http', 'https', 'iscsi', 'luks', 'nbd', 'nfs',
-             'null-aio', 'null-co', 'nvme', 'parallels', 'qcow', 'qcow2', 'qed',
--            'quorum', 'raw', 'rbd', 'replication', 'sheepdog', 'ssh',
-+            'quorum', 'raw', 'rbd', 'vitastor', 'replication', 'sheepdog', 'ssh',
-             'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
- 
- ##
-@@ -3217,6 +3217,28 @@
-             '*snap-id': 'uint32',
-             '*tag': 'str' } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -3547,6 +3569,7 @@
-       'rbd':        'BlockdevOptionsRbd',
-       'replication':'BlockdevOptionsReplication',
-       'sheepdog':   'BlockdevOptionsSheepdog',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'ssh':        'BlockdevOptionsSsh',
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-@@ -3991,6 +4014,17 @@
-             '*subformat':           'BlockdevVhdxSubformat',
-             '*block-state-zero':    'bool' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVpcSubformat:
- #
-@@ -4074,6 +4108,7 @@
-       'rbd':            'BlockdevCreateOptionsRbd',
-       'replication':    'BlockdevCreateNotSupported',
-       'sheepdog':       'BlockdevCreateOptionsSheepdog',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'throttle':       'BlockdevCreateNotSupported',
-       'vdi':            'BlockdevCreateOptionsVdi',

patches/qemu-5.2-vitastor.patch

@@ -1,181 +0,0 @@
-Index: qemu-5.2+dfsg/qapi/block-core.json
-===================================================================
---- qemu-5.2+dfsg.orig/qapi/block-core.json
-+++ qemu-5.2+dfsg/qapi/block-core.json
-@@ -2831,7 +2831,7 @@
-             'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
-             'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
-             { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
--            'sheepdog',
-+            'sheepdog', 'vitastor',
-             'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
- 
- ##
-@@ -3668,6 +3668,28 @@
-             '*tag': 'str' } }
- 
- ##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
-+##
- # @ReplicationMode:
- #
- # An enumeration of replication modes.
-@@ -4015,6 +4037,7 @@
-       'replication': { 'type': 'BlockdevOptionsReplication',
-                        'if': 'defined(CONFIG_REPLICATION)' },
-       'sheepdog':   'BlockdevOptionsSheepdog',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'ssh':        'BlockdevOptionsSsh',
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-@@ -4404,6 +4427,17 @@
-             '*cluster-size' :   'size' } }
- 
- ##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
-+##
- # @BlockdevVmdkSubformat:
- #
- # Subformat options for VMDK images
-@@ -4665,6 +4699,7 @@
-       'qed':            'BlockdevCreateOptionsQed',
-       'rbd':            'BlockdevCreateOptionsRbd',
-       'sheepdog':       'BlockdevCreateOptionsSheepdog',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-Index: qemu-5.2+dfsg/block/meson.build
-===================================================================
---- qemu-5.2+dfsg.orig/block/meson.build
-+++ qemu-5.2+dfsg/block/meson.build
-@@ -76,6 +76,7 @@ foreach m : [
-   ['CONFIG_LIBNFS', 'nfs', libnfs, 'nfs.c'],
-   ['CONFIG_LIBSSH', 'ssh', libssh, 'ssh.c'],
-   ['CONFIG_RBD', 'rbd', rbd, 'rbd.c'],
-+  ['CONFIG_VITASTOR', 'vitastor', vitastor, 'vitastor.c'],
- ]
-   if config_host.has_key(m[0])
-     if enable_modules
-Index: qemu-5.2+dfsg/configure
-===================================================================
---- qemu-5.2+dfsg.orig/configure
-+++ qemu-5.2+dfsg/configure
-@@ -372,6 +372,7 @@ trace_backends="log"
- trace_file="trace"
- spice=""
- rbd=""
-+vitastor=""
- smartcard=""
- u2f="auto"
- libusb=""
-@@ -1263,6 +1264,10 @@ for opt do
-   ;;
-   --enable-rbd) rbd="yes"
-   ;;
-+  --disable-vitastor) vitastor="no"
-+  ;;
-+  --enable-vitastor) vitastor="yes"
-+  ;;
-   --disable-xfsctl) xfs="no"
-   ;;
-   --enable-xfsctl) xfs="yes"
-@@ -1827,6 +1832,7 @@ disabled with --disable-FEATURE, default
-   vhost-vdpa      vhost-vdpa kernel backend support
-   spice           spice
-   rbd             rados block device (rbd)
-+  vitastor        vitastor block device
-   libiscsi        iscsi support
-   libnfs          nfs support
-   smartcard       smartcard support (libcacard)
-@@ -3719,6 +3725,27 @@ EOF
- fi
- 
- ##########################################
-+# vitastor probe
-+if test "$vitastor" != "no" ; then
-+  cat > $TMPC <<EOF
-+#include <vitastor_c.h>
-+int main(void) {
-+  vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+  return 0;
-+}
-+EOF
-+  vitastor_libs="-lvitastor_client"
-+  if compile_prog "" "$vitastor_libs" ; then
-+    vitastor=yes
-+  else
-+    if test "$vitastor" = "yes" ; then
-+      feature_not_found "vitastor block device" "Install vitastor-client-dev"
-+    fi
-+    vitastor=no
-+  fi
-+fi
-+
-+##########################################
- # libssh probe
- if test "$libssh" != "no" ; then
-   if $pkg_config --exists libssh; then
-@@ -6456,6 +6483,10 @@ if test "$rbd" = "yes" ; then
-   echo "CONFIG_RBD=y" >> $config_host_mak
-   echo "RBD_LIBS=$rbd_libs" >> $config_host_mak
- fi
-+if test "$vitastor" = "yes" ; then
-+  echo "CONFIG_VITASTOR=y" >> $config_host_mak
-+  echo "VITASTOR_LIBS=$vitastor_libs" >> $config_host_mak
-+fi
- 
- echo "CONFIG_COROUTINE_BACKEND=$coroutine" >> $config_host_mak
- if test "$coroutine_pool" = "yes" ; then
-Index: qemu-5.2+dfsg/meson.build
-===================================================================
---- qemu-5.2+dfsg.orig/meson.build
-+++ qemu-5.2+dfsg/meson.build
-@@ -596,6 +596,10 @@ rbd = not_found
- if 'CONFIG_RBD' in config_host
-   rbd = declare_dependency(link_args: config_host['RBD_LIBS'].split())
- endif
-+vitastor = not_found
-+if 'CONFIG_VITASTOR' in config_host
-+  vitastor = declare_dependency(link_args: config_host['VITASTOR_LIBS'].split())
-+endif
- glusterfs = not_found
- if 'CONFIG_GLUSTERFS' in config_host
-   glusterfs = declare_dependency(compile_args: config_host['GLUSTERFS_CFLAGS'].split(),
-@@ -2145,6 +2149,7 @@ endif
- # TODO: add back protocol and server version
- summary_info += {'spice support':     config_host.has_key('CONFIG_SPICE')}
- summary_info += {'rbd support':       config_host.has_key('CONFIG_RBD')}
-+summary_info += {'vitastor support':  config_host.has_key('CONFIG_VITASTOR')}
- summary_info += {'xfsctl support':    config_host.has_key('CONFIG_XFS')}
- summary_info += {'smartcard support': config_host.has_key('CONFIG_SMARTCARD')}
- summary_info += {'U2F support':       u2f.found()}

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-1.0.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.0.0$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.9.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.0$(rpm --eval '%dist').tar.gz *

rpm/qemu-kvm-4.2-el7.spec.patch

@@ -22,7 +22,7 @@
  Name: qemu-kvm
  Version: 4.2.0
 -Release: 29.vitastor%{?dist}.6
-+Release: 34.vitastor%{?dist}.6
++Release: 32.vitastor%{?dist}.6
  # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
  Epoch: 15
  License: GPLv2 and GPLv2+ and CC-BY

rpm/qemu-kvm-4.2-el8.spec.patch

@@ -13,7 +13,7 @@
  Name: qemu-kvm
  Version: 4.2.0
 -Release: 29%{?dist}.6
-+Release: 33.vitastor%{?dist}.6
++Release: 32.vitastor%{?dist}.6
  # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
  Epoch: 15
  License: GPLv2 and GPLv2+ and CC-BY

rpm/qemu-kvm-6.2-el8.spec.patch

@@ -1,103 +0,0 @@
---- qemu-kvm-6.2.spec.orig	2023-07-18 13:52:57.636625440 +0000
-+++ qemu-kvm-6.2.spec	2023-07-18 13:52:19.011683886 +0000
-@@ -73,6 +73,7 @@ Requires: %{name}-hw-usbredir = %{epoch}
- %endif                                                           \
- Requires: %{name}-block-iscsi = %{epoch}:%{version}-%{release}   \
- Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release}     \
-+Requires: %{name}-block-vitastor = %{epoch}:%{version}-%{release}\
- Requires: %{name}-block-ssh = %{epoch}:%{version}-%{release}
- 
- # Macro to properly setup RHEL/RHEV conflict handling
-@@ -83,7 +84,7 @@ Obsoletes: %1-rhev <= %{epoch}:%{version
- Summary: QEMU is a machine emulator and virtualizer
- Name: qemu-kvm
- Version: 6.2.0
--Release: 32%{?rcrel}%{?dist}
-+Release: 32.vitastor%{?rcrel}%{?dist}
- # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
- Epoch: 15
- License: GPLv2 and GPLv2+ and CC-BY
-@@ -122,6 +123,7 @@ Source37: tests_data_acpi_pc_SSDT.dimmpx
- Source38: tests_data_acpi_q35_FACP.slic
- Source39: tests_data_acpi_q35_SSDT.dimmpxm
- Source40: tests_data_acpi_virt_SSDT.memhp
-+Source41: qemu-vitastor.c
- 
- Patch0001: 0001-redhat-Adding-slirp-to-the-exploded-tree.patch
- Patch0005: 0005-Initial-redhat-build.patch
-@@ -652,6 +654,7 @@ Patch255: kvm-scsi-protect-req-aiocb-wit
- Patch256: kvm-dma-helpers-prevent-dma_blk_cb-vs-dma_aio_cancel-rac.patch
- # For bz#2090990 - qemu crash with error scsi_req_unref(SCSIRequest *): Assertion `req->refcount > 0' failed or scsi_dma_complete(void *, int): Assertion `r->req.aiocb != NULL' failed [8.7.0]
- Patch257: kvm-virtio-scsi-reset-SCSI-devices-from-main-loop-thread.patch
-+Patch258: qemu-6.2-vitastor.patch
- 
- BuildRequires: wget
- BuildRequires: rpm-build
-@@ -689,6 +692,7 @@ BuildRequires: libcurl-devel
- BuildRequires: libssh-devel
- BuildRequires: librados-devel
- BuildRequires: librbd-devel
-+BuildRequires: vitastor-client-devel
- %if %{have_gluster}
- # For gluster block driver
- BuildRequires: glusterfs-api-devel
-@@ -926,6 +930,14 @@ Install this package if you want to acce
- using the rbd protocol.
- 
- 
-+%package  block-vitastor
-+Summary: QEMU Vitastor block driver
-+Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
-+
-+%description block-vitastor
-+This package provides the additional Vitastor block driver for QEMU.
-+
-+
- %package  block-ssh
- Summary: QEMU SSH block driver
- Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
-@@ -979,6 +991,7 @@ This package provides usbredir support.
- rm -fr slirp
- mkdir slirp
- %autopatch -p1
-+cp %{SOURCE41} ./block/vitastor.c
- 
- %global qemu_kvm_build qemu_kvm_build
- mkdir -p %{qemu_kvm_build}
-@@ -994,7 +1007,7 @@ cp -f %{SOURCE40} tests/data/acpi/virt/S
- # --build-id option is used for giving info to the debug packages.
- buildldflags="VL_LDFLAGS=-Wl,--build-id"
- 
--%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle
-+%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle
- 
- %if 0%{have_gluster}
-     %global block_drivers_list %{block_drivers_list},gluster
-@@ -1149,9 +1162,7 @@ pushd %{qemu_kvm_build}
-   --firmwarepath=%{_prefix}/share/qemu-firmware \
-   --meson="git" \
-   --target-list="%{buildarch}" \
--  --block-drv-rw-whitelist=%{block_drivers_list} \
-   --audio-drv-list= \
--  --block-drv-ro-whitelist=vmdk,vhdx,vpc,https,ssh \
-   --with-coroutine=ucontext \
-   --with-git=git \
-   --tls-priority=@QEMU,SYSTEM \
-@@ -1197,6 +1208,7 @@ pushd %{qemu_kvm_build}
- %endif
-   --enable-pie \
-   --enable-rbd \
-+  --enable-vitastor \
- %if 0%{have_librdma}
-   --enable-rdma \
- %endif
-@@ -1794,6 +1806,9 @@ sh %{_sysconfdir}/sysconfig/modules/kvm.
- %files block-rbd
- %{_libdir}/qemu-kvm/block-rbd.so
- 
-+%files block-vitastor
-+%{_libdir}/qemu-kvm/block-vitastor.so
-+
- %files block-ssh
- %{_libdir}/qemu-kvm/block-ssh.so
- 

rpm/qemu-kvm-7.2-el9.spec.patch

@@ -1,93 +0,0 @@
---- qemu-kvm-7.2.spec.orig	2023-06-22 13:56:19.000000000 +0000
-+++ qemu-kvm-7.2.spec	2023-07-18 07:55:22.347090196 +0000
-@@ -100,8 +100,6 @@
- %endif
- 
- %global target_list %{kvm_target}-softmmu
--%global block_drivers_rw_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,compress
--%global block_drivers_ro_list vdi,vmdk,vhdx,vpc,https
- %define qemudocdir %{_docdir}/%{name}
- %global firmwaredirs "%{_datadir}/qemu-firmware:%{_datadir}/ipxe/qemu:%{_datadir}/seavgabios:%{_datadir}/seabios"
- 
-@@ -126,6 +124,7 @@ Requires: %{name}-device-usb-host = %{ep
- Requires: %{name}-device-usb-redirect = %{epoch}:%{version}-%{release}   \
- %endif                                                           \
- Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release}     \
-+Requires: %{name}-block-vitastor = %{epoch}:%{version}-%{release}\
- Requires: %{name}-audio-pa = %{epoch}:%{version}-%{release}
- 
- # Since SPICE is removed from RHEL-9, the following Obsoletes:
-@@ -148,7 +147,7 @@ Obsoletes: %{name}-block-ssh <= %{epoch}
- Summary: QEMU is a machine emulator and virtualizer
- Name: qemu-kvm
- Version: 7.2.0
--Release: 14%{?rcrel}%{?dist}%{?cc_suffix}.1
-+Release: 14.vitastor%{?rcrel}%{?dist}%{?cc_suffix}.1
- # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
- # Epoch 15 used for RHEL 8
- # Epoch 17 used for RHEL 9 (due to release versioning offset in RHEL 8.5)
-@@ -171,6 +170,7 @@ Source28: 95-kvm-memlock.conf
- Source30: kvm-s390x.conf
- Source31: kvm-x86.conf
- Source36: README.tests
-+Source37: qemu-vitastor.c
- 
- 
- Patch0004: 0004-Initial-redhat-build.patch
-@@ -418,6 +418,7 @@ Patch134: kvm-target-i386-Fix-BZHI-instr
- Patch135: kvm-intel-iommu-fail-DEVIOTLB_UNMAP-without-dt-mode.patch
- # For bz#2203745 - Disk detach is unsuccessful while the guest is still booting [rhel-9.2.0.z]
- Patch136: kvm-acpi-pcihp-allow-repeating-hot-unplug-requests.patch
-+Patch137: qemu-7.2-vitastor.patch
- 
- %if %{have_clang}
- BuildRequires: clang
-@@ -449,6 +450,7 @@ BuildRequires: libcurl-devel
- %if %{have_block_rbd}
- BuildRequires: librbd-devel
- %endif
-+BuildRequires: vitastor-client-devel
- # We need both because the 'stap' binary is probed for by configure
- BuildRequires: systemtap
- BuildRequires: systemtap-sdt-devel
-@@ -642,6 +644,14 @@ using the rbd protocol.
- %endif
- 
- 
-+%package  block-vitastor
-+Summary: QEMU Vitastor block driver
-+Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
-+
-+%description block-vitastor
-+This package provides the additional Vitastor block driver for QEMU.
-+
-+
- %package  audio-pa
- Summary: QEMU PulseAudio audio driver
- Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
-@@ -719,6 +729,7 @@ This package provides usbredir support.
- %prep
- %setup -q -n qemu-%{version}%{?rcstr}
- %autopatch -p1
-+cp %{SOURCE37} ./block/vitastor.c
- 
- %global qemu_kvm_build qemu_kvm_build
- mkdir -p %{qemu_kvm_build}
-@@ -946,6 +957,7 @@ run_configure \
- %if %{have_block_rbd}
-   --enable-rbd \
- %endif
-+  --enable-vitastor \
- %if %{have_librdma}
-   --enable-rdma \
- %endif
-@@ -1426,6 +1438,9 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
- %files block-rbd
- %{_libdir}/%{name}/block-rbd.so
- %endif
-+%files block-vitastor
-+%{_libdir}/%{name}/block-vitastor.so
-+
- %files audio-pa
- %{_libdir}/%{name}/audio-pa.so
- 

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.0.0.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.0.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.0.0
+Version:        0.9.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.0.0.el7.tar.gz
+Source0:        vitastor-0.9.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.0.0.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.0.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.0.0
+Version:        0.9.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.0.0.el8.tar.gz
+Source0:        vitastor-0.9.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.0.0.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.0.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.0.0
+Version:        0.9.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.0.0.el9.tar.gz
+Source0:        vitastor-0.9.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="1.0.0")
+add_definitions(-DVERSION="0.9.0")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/addr_util.cpp

@@ -19,8 +19,8 @@ bool string_to_addr(std::string str, bool parse_port, int default_port, struct s
         if (p != std::string::npos && !(str.length() > 0 && str[p-1] == ']')) // "[ipv6]" which contains ':'
         {
             char null_byte = 0;
-            int scanned = sscanf(str.c_str()+p+1, "%d%c", &default_port, &null_byte);
-            if (scanned != 1 || default_port >= 0x10000)
+            int n = sscanf(str.c_str()+p+1, "%d%c", &default_port, &null_byte);
+            if (n != 1 || default_port >= 0x10000)
                 return false;
             str = str.substr(0, p);
         }

src/allocator.cpp

@@ -143,83 +143,34 @@ uint64_t allocator::get_free_count()
     return free;
 }
 
-// FIXME: Move to utils?
 void bitmap_set(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
 {
-    if (start == 0 && len == 32*bitmap_granularity)
-        *((uint32_t*)bitmap) = UINT32_MAX;
-    else if (start == 0 && len == 64*bitmap_granularity)
-        *((uint64_t*)bitmap) = UINT64_MAX;
-    else
+    if (start == 0)
     {
-        unsigned bit_start = start / bitmap_granularity;
-        unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
-        while (bit_start < bit_end)
+        if (len == 32*bitmap_granularity)
         {
-            if (!(bit_start & 7) && bit_end >= bit_start+8)
-            {
-                ((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
-                bit_start += 8;
-            }
-            else
-            {
-                ((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
-                bit_start++;
-            }
+            *((uint32_t*)bitmap) = UINT32_MAX;
+            return;
+        }
+        else if (len == 64*bitmap_granularity)
+        {
+            *((uint64_t*)bitmap) = UINT64_MAX;
+            return;
+        }
+    }
+    unsigned bit_start = start / bitmap_granularity;
+    unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
+    while (bit_start < bit_end)
+    {
+        if (!(bit_start & 7) && bit_end >= bit_start+8)
+        {
+            ((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
+            bit_start += 8;
+        }
+        else
+        {
+            ((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
+            bit_start++;
         }
     }
 }
-
-void bitmap_clear(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
-{
-    if (start == 0 && len == 32*bitmap_granularity)
-        *((uint32_t*)bitmap) = 0;
-    else if (start == 0 && len == 64*bitmap_granularity)
-        *((uint64_t*)bitmap) = 0;
-    else
-    {
-        unsigned bit_start = start / bitmap_granularity;
-        unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
-        while (bit_start < bit_end)
-        {
-            if (!(bit_start & 7) && bit_end >= bit_start+8)
-            {
-                ((uint8_t*)bitmap)[bit_start / 8] = 0;
-                bit_start += 8;
-            }
-            else
-            {
-                ((uint8_t*)bitmap)[bit_start / 8] &= (0xFF ^ (1 << (bit_start % 8)));
-                bit_start++;
-            }
-        }
-    }
-}
-
-bool bitmap_check(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
-{
-    bool r = false;
-    if (start == 0 && len == 32*bitmap_granularity)
-        r = !!*((uint32_t*)bitmap);
-    else if (start == 0 && len == 64*bitmap_granularity)
-        r = !!*((uint64_t*)bitmap);
-    else
-    {
-        unsigned bit_start = start / bitmap_granularity;
-        unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
-        while (bit_start < bit_end)
-        {
-            if (!(bit_start & 7) && bit_end >= bit_start+8)
-            {
-                r = r || !!((uint8_t*)bitmap)[bit_start / 8];
-                bit_start += 8;
-            }
-            else
-            {
-                r = r || (((uint8_t*)bitmap)[bit_start / 8] & (1 << (bit_start % 8)));
-                bit_start++;
-            }
-        }
-    }
-    return r;
-}

src/allocator.h

@@ -23,5 +23,3 @@ public:
 };
 
 void bitmap_set(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);
-void bitmap_clear(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);
-bool bitmap_check(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);

src/blockstore.h

@@ -77,7 +77,6 @@ Output:
   -EINVAL = invalid input parameters
   -ENOENT = requested object/version does not exist for reads
   -ENOSPC = no space left in the store for writes
-  -EDOM = checksum error.
 - version = the version actually read or written
 
 ## BS_OP_DELETE

src/blockstore_disk.cpp

@@ -40,31 +40,10 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     data_block_size = parse_size(config["block_size"]);
     journal_device = config["journal_device"];
     journal_offset = parse_size(config["journal_offset"]);
-    disk_alignment = parse_size(config["disk_alignment"]);
-    journal_block_size = parse_size(config["journal_block_size"]);
-    meta_block_size = parse_size(config["meta_block_size"]);
-    bitmap_granularity = parse_size(config["bitmap_granularity"]);
-    meta_format = stoull_full(config["meta_format"]);
-    cached_io_data = config["cached_io_data"] == "true" || config["cached_io_data"] == "yes" || config["cached_io_data"] == "1";
-    cached_io_meta = cached_io_data && (meta_device == data_device || meta_device == "") &&
-        config.find("cached_io_meta") == config.end() ||
-        config["cached_io_meta"] == "true" || config["cached_io_meta"] == "yes" || config["cached_io_meta"] == "1";
-    cached_io_journal = cached_io_meta && (journal_device == meta_device || journal_device == "") &&
-        config.find("cached_io_journal") == config.end() ||
-        config["cached_io_journal"] == "true" || config["cached_io_journal"] == "yes" || config["cached_io_journal"] == "1";
-    if (config["data_csum_type"] == "crc32c")
-    {
-        data_csum_type = BLOCKSTORE_CSUM_CRC32C;
-    }
-    else if (config["data_csum_type"] == "" || config["data_csum_type"] == "none")
-    {
-        data_csum_type = BLOCKSTORE_CSUM_NONE;
-    }
-    else
-    {
-        throw std::runtime_error("data_csum_type="+config["data_csum_type"]+" is unsupported, only \"crc32c\" and \"none\" are supported");
-    }
-    csum_block_size = parse_size(config["csum_block_size"]);
+    disk_alignment = strtoull(config["disk_alignment"].c_str(), NULL, 10);
+    journal_block_size = strtoull(config["journal_block_size"].c_str(), NULL, 10);
+    meta_block_size = strtoull(config["meta_block_size"].c_str(), NULL, 10);
+    bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
     // Validate
     if (!data_block_size)
     {
@@ -112,23 +91,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
     if (data_block_size % bitmap_granularity)
     {
-        throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
-    }
-    if (!data_csum_type)
-    {
-        csum_block_size = 0;
-    }
-    else if (!csum_block_size)
-    {
-        csum_block_size = bitmap_granularity;
-    }
-    if (csum_block_size && (csum_block_size % bitmap_granularity))
-    {
-        throw std::runtime_error("Checksum block size must be a multiple of sparse write tracking granularity");
-    }
-    if (csum_block_size && (data_block_size % csum_block_size))
-    {
-        throw std::runtime_error("Checksum block size must be a divisor of data block size");
+        throw std::runtime_error("Block size must be a multiple of sparse write tracking granularity");
     }
     if (meta_device == "")
     {
@@ -147,9 +110,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
         throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
     }
     clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
-    clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
-        ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
-    clean_entry_size = sizeof(clean_disk_entry) + clean_dyn_size + 4 /*entry_csum*/;
+    clean_entry_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
 }
 
 void blockstore_disk_t::calc_lengths(bool skip_meta_check)
@@ -199,25 +160,6 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
     // required metadata size
     block_count = data_len / data_block_size;
     meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
-    if (meta_format == BLOCKSTORE_META_FORMAT_V1 ||
-        !meta_format && !skip_meta_check && meta_area_size < meta_len && !data_csum_type)
-    {
-        uint64_t clean_entry_v0_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
-        uint64_t meta_v0_len = (1 + (block_count - 1 + meta_block_size / clean_entry_v0_size)
-            / (meta_block_size / clean_entry_v0_size)) * meta_block_size;
-        if (meta_format == BLOCKSTORE_META_FORMAT_V1 || meta_area_size >= meta_v0_len)
-        {
-            // Old metadata fits.
-            printf("Warning: Using old metadata format without checksums because the new format doesn't fit into provided area\n");
-            clean_entry_size = clean_entry_v0_size;
-            meta_len = meta_v0_len;
-            meta_format = BLOCKSTORE_META_FORMAT_V1;
-        }
-        else
-            meta_format = BLOCKSTORE_META_FORMAT_V2;
-    }
-    else
-        meta_format = BLOCKSTORE_META_FORMAT_V2;
     if (!skip_meta_check && meta_area_size < meta_len)
     {
         throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(meta_len)+" bytes");
@@ -274,7 +216,7 @@ static void check_size(int fd, uint64_t *size, uint64_t *sectsize, std::string n
 
 void blockstore_disk_t::open_data()
 {
-    data_fd = open(data_device.c_str(), (cached_io_data ? O_SYNC : O_DIRECT) | O_RDWR);
+    data_fd = open(data_device.c_str(), O_DIRECT|O_RDWR);
     if (data_fd == -1)
     {
         throw std::runtime_error("Failed to open data device "+data_device+": "+std::string(strerror(errno)));
@@ -299,9 +241,9 @@ void blockstore_disk_t::open_data()
 
 void blockstore_disk_t::open_meta()
 {
-    if (meta_device != data_device || cached_io_meta != cached_io_data)
+    if (meta_device != data_device)
     {
-        meta_fd = open(meta_device.c_str(), (cached_io_meta ? O_SYNC : O_DIRECT) | O_RDWR);
+        meta_fd = open(meta_device.c_str(), O_DIRECT|O_RDWR);
         if (meta_fd == -1)
         {
             throw std::runtime_error("Failed to open metadata device "+meta_device+": "+std::string(strerror(errno)));
@@ -311,7 +253,7 @@ void blockstore_disk_t::open_meta()
         {
             throw std::runtime_error("meta_offset exceeds device size = "+std::to_string(meta_device_size));
         }
-        if (!disable_flock && meta_device != data_device && flock(meta_fd, LOCK_EX|LOCK_NB) != 0)
+        if (!disable_flock && flock(meta_fd, LOCK_EX|LOCK_NB) != 0)
         {
             throw std::runtime_error(std::string("Failed to lock metadata device: ") + strerror(errno));
         }
@@ -337,15 +279,15 @@ void blockstore_disk_t::open_meta()
 
 void blockstore_disk_t::open_journal()
 {
-    if (journal_device != meta_device || cached_io_journal != cached_io_meta)
+    if (journal_device != meta_device)
     {
-        journal_fd = open(journal_device.c_str(), (cached_io_journal ? O_SYNC : O_DIRECT) | O_RDWR);
+        journal_fd = open(journal_device.c_str(), O_DIRECT|O_RDWR);
         if (journal_fd == -1)
         {
             throw std::runtime_error("Failed to open journal device "+journal_device+": "+std::string(strerror(errno)));
         }
         check_size(journal_fd, &journal_device_size, &journal_device_sect, "journal device");
-        if (!disable_flock && journal_device != meta_device && flock(journal_fd, LOCK_EX|LOCK_NB) != 0)
+        if (!disable_flock && flock(journal_fd, LOCK_EX|LOCK_NB) != 0)
         {
             throw std::runtime_error(std::string("Failed to lock journal device: ") + strerror(errno));
         }

src/blockstore_disk.h

@@ -8,10 +8,6 @@
 #include <string>
 #include <map>
 
-#define BLOCKSTORE_CSUM_NONE 0
-// Lower byte of checksum type is its length
-#define BLOCKSTORE_CSUM_CRC32C 0x104
-
 struct blockstore_disk_t
 {
     std::string data_device, meta_device, journal_device;
@@ -25,23 +21,17 @@ struct blockstore_disk_t
     uint64_t meta_block_size = 4096;
     // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
     uint64_t bitmap_granularity = 4096;
-    // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
-    uint32_t data_csum_type = BLOCKSTORE_CSUM_NONE;
-    // Checksum block size, must be a multiple of bitmap_granularity
-    uint32_t csum_block_size = 4096;
     // By default, Blockstore locks all opened devices exclusively. This option can be used to disable locking
     bool disable_flock = false;
-    // Use Linux page cache for reads and writes, i.e. open FDs with O_SYNC instead of O_DIRECT
-    bool cached_io_data = false, cached_io_meta = false, cached_io_journal = false;
 
     int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_len, meta_format = 0;
+    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_len;
     uint64_t data_offset, data_device_sect, data_device_size, data_len;
     uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
 
     uint32_t block_order;
     uint64_t block_count;
-    uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0, clean_dyn_size = 0;
+    uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0;
 
     void parse_config(std::map<std::string, std::string> & config);
     void open_data();
@@ -49,13 +39,4 @@ struct blockstore_disk_t
     void open_journal();
     void calc_lengths(bool skip_meta_check = false);
     void close_all();
-
-    inline uint64_t dirty_dyn_size(uint64_t offset, uint64_t len)
-    {
-        // Checksums may be partial if write is not aligned with csum_block_size
-        return clean_entry_bitmap_size + (csum_block_size && len > 0
-            ? ((offset+len+csum_block_size-1)/csum_block_size - offset/csum_block_size)
-                * (data_csum_type & 0xFF)
-            : 0);
-    }
 };

src/blockstore_flush.h

@@ -1,22 +1,10 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
-#define COPY_BUF_JOURNAL 1
-#define COPY_BUF_DATA 2
-#define COPY_BUF_ZERO 4
-#define COPY_BUF_CSUM_FILL 8
-#define COPY_BUF_COALESCED 16
-#define COPY_BUF_META_BLOCK 32
-#define COPY_BUF_JOURNALED_BIG 64
-
 struct copy_buffer_t
 {
-    int copy_flags;
-    uint64_t offset, len, disk_offset;
-    uint64_t journal_sector; // only for reads: sector+1 if used and !journal.inmemory, otherwise 0
+    uint64_t offset, len;
     void *buf;
-    uint8_t *csum_buf;
-    int *dyn_data;
 };
 
 struct meta_sector_t
@@ -49,7 +37,7 @@ class journal_flusher_co
 {
     blockstore_impl_t *bs;
     journal_flusher_t *flusher;
-    int wait_state, wait_count, wait_journal_count;
+    int wait_state, wait_count;
     struct io_uring_sqe *sqe;
     struct ring_data_t *data;
 
@@ -58,39 +46,28 @@ class journal_flusher_co
     obj_ver_id cur;
     std::map<obj_ver_id, dirty_entry>::iterator dirty_it, dirty_start, dirty_end;
     std::map<object_id, uint64_t>::iterator repeat_it;
-    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_rj, simple_callback_w;
+    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
 
     bool skip_copy, has_delete, has_writes;
     std::vector<copy_buffer_t> v;
     std::vector<copy_buffer_t>::iterator it;
-    int i;
-    bool fill_incomplete, cleared_incomplete;
-    int read_to_fill_incomplete;
     int copy_count;
-    uint64_t clean_loc, clean_ver, old_clean_loc, old_clean_ver;
+    uint64_t clean_loc, old_clean_loc;
     flusher_meta_write_t meta_old, meta_new;
     bool clean_init_bitmap;
     uint64_t clean_bitmap_offset, clean_bitmap_len;
-    uint8_t *clean_init_dyn_ptr;
-    uint8_t *new_clean_bitmap;
+    void *new_clean_bitmap;
 
     uint64_t new_trim_pos;
 
+    // local: scan_dirty()
+    uint64_t offset, end_offset, submit_offset, submit_len;
+
     friend class journal_flusher_t;
-    void scan_dirty();
-    bool read_dirty(int wait_base);
-    bool modify_meta_do_reads(int wait_base);
-    bool wait_meta_reads(int wait_base);
+    bool scan_dirty(int wait_base);
     bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base);
-    bool clear_incomplete_csum_block_bits(int wait_base);
-    void calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites);
-    void update_metadata_entry();
-    bool write_meta_block(flusher_meta_write_t & meta_block, int wait_base);
     void update_clean_db();
-    void free_data_blocks();
     bool fsync_batch(bool fsync_meta, int wait_base);
-    bool trim_journal(int wait_base);
-    void free_buffers();
 public:
     journal_flusher_co();
     bool loop();
@@ -118,16 +95,14 @@ class journal_flusher_t
 
     std::map<uint64_t, meta_sector_t> meta_sectors;
     std::deque<object_id> flush_queue;
-    std::map<object_id, uint64_t> flush_versions; // FIXME: consider unordered_map?
+    std::map<object_id, uint64_t> flush_versions;
 
     bool try_find_older(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
-    bool try_find_other(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
 
 public:
     journal_flusher_t(blockstore_impl_t *bs);
     ~journal_flusher_t();
     void loop();
-    bool is_trim_wanted() { return trim_wanted; }
     bool is_active();
     void mark_trim_possible();
     void request_trim();
@@ -136,5 +111,4 @@ public:
     void unshift_flush(obj_ver_id oid, bool force);
     void remove_flush(object_id oid);
     void dump_diagnostics();
-    bool is_mutated(uint64_t clean_loc);
 };

src/blockstore_impl.cpp

@@ -13,7 +13,6 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *
     initialized = 0;
     parse_config(config, true);
     zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
-    alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0;
     try
     {
         dsk.open_data();
@@ -39,8 +38,8 @@ blockstore_impl_t::~blockstore_impl_t()
     dsk.close_all();
     if (metadata_buffer)
         free(metadata_buffer);
-    if (clean_bitmaps)
-        free(clean_bitmaps);
+    if (clean_bitmap)
+        free(clean_bitmap);
 }
 
 bool blockstore_impl_t::is_started()
@@ -393,7 +392,6 @@ void blockstore_impl_t::init_op(blockstore_op_t *op)
 {
     // Call constructor without allocating memory. We'll call destructor before returning op back
     new ((void*)op->private_data) blockstore_op_private_t;
-    PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
     PRIV(op)->wait_for = 0;
     PRIV(op)->op_state = 0;
     PRIV(op)->pending_ops = 0;

src/blockstore_impl.h

@@ -93,10 +93,11 @@
 
 // "VITAstor"
 #define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
-#define BLOCKSTORE_META_FORMAT_V1 1
-#define BLOCKSTORE_META_FORMAT_V2 2
+#define BLOCKSTORE_META_VERSION_V1 1
 
 // metadata header (superblock)
+// FIXME: After adding the OSD superblock, add a key to metadata
+// and journal headers to check if they belong to the same OSD
 struct __attribute__((__packed__)) blockstore_meta_header_v1_t
 {
     uint64_t zero;
@@ -107,29 +108,14 @@ struct __attribute__((__packed__)) blockstore_meta_header_v1_t
     uint32_t bitmap_granularity;
 };
 
-struct __attribute__((__packed__)) blockstore_meta_header_v2_t
-{
-    uint64_t zero;
-    uint64_t magic;
-    uint64_t version;
-    uint32_t meta_block_size;
-    uint32_t data_block_size;
-    uint32_t bitmap_granularity;
-    uint32_t data_csum_type;
-    uint32_t csum_block_size;
-    uint32_t header_csum;
-};
-
 // 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
 // per "clean" entry on disk with fixed metadata tables
+// FIXME: maybe add crc32's to metadata
 struct __attribute__((__packed__)) clean_disk_entry
 {
     object_id oid;
     uint64_t version;
     uint8_t bitmap[];
-    // Two more fields come after bitmap in metadata version 2:
-    // uint32_t data_csum[];
-    // uint32_t entry_csum;
 };
 
 // 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
@@ -139,7 +125,7 @@ struct __attribute__((__packed__)) clean_entry
     uint64_t location;
 };
 
-// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums
+// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry)
 struct __attribute__((__packed__)) dirty_entry
 {
     uint32_t state;
@@ -148,7 +134,7 @@ struct __attribute__((__packed__)) dirty_entry
     uint32_t offset;   // data offset within object (stripe)
     uint32_t len;      // data length
     uint64_t journal_sector; // journal sector used for this entry
-    void* dyn_data;    // dynamic data: external bitmap and data block checksums. may be a pointer to the in-memory journal
+    void* bitmap;   // either external bitmap itself when it fits, or a pointer to it when it doesn't
 };
 
 // - Sync must be submitted after previous writes/deletes (not before!)
@@ -177,23 +163,12 @@ struct __attribute__((__packed__)) dirty_entry
 // Suspend operation until there is some free space on the data device
 #define WAIT_FREE 5
 
-struct used_clean_obj_t
+struct fulfill_read_t
 {
-    int refs;
-    bool was_freed; // was freed by a parallel flush?
-    bool was_changed; // was changed by a parallel flush?
+    uint64_t offset, len;
+    uint64_t journal_sector; // sector+1 if used and !journal.inmemory, otherwise 0
 };
 
-// https://github.com/algorithm-ninja/cpp-btree
-// https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing
-// with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB
-typedef btree::btree_map<object_id, clean_entry> blockstore_clean_db_t;
-typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
-
-#include "blockstore_init.h"
-
-#include "blockstore_flush.h"
-
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
 #define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
 
@@ -206,11 +181,10 @@ struct blockstore_op_private_t
     int op_state;
 
     // Read
-    uint64_t clean_block_used;
-    std::vector<copy_buffer_t> read_vec;
+    std::vector<fulfill_read_t> read_vec;
 
     // Sync, write
-    uint64_t min_flushed_journal_sector, max_flushed_journal_sector;
+    int min_flushed_journal_sector, max_flushed_journal_sector;
 
     // Write
     struct iovec iov_zerofill[3];
@@ -220,8 +194,19 @@ struct blockstore_op_private_t
 
     // Sync
     std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
+    int sync_small_checked, sync_big_checked;
 };
 
+// https://github.com/algorithm-ninja/cpp-btree
+// https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing
+// with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB
+typedef btree::btree_map<object_id, clean_entry> blockstore_clean_db_t;
+typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
+
+#include "blockstore_init.h"
+
+#include "blockstore_flush.h"
+
 typedef uint32_t pool_id_t;
 typedef uint64_t pool_pg_id_t;
 
@@ -268,7 +253,7 @@ class blockstore_impl_t
 
     std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
     std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
-    uint8_t *clean_bitmaps = NULL;
+    uint8_t *clean_bitmap = NULL;
     blockstore_dirty_db_t dirty_db;
     std::vector<blockstore_op_t*> submit_queue;
     std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
@@ -282,10 +267,6 @@ class blockstore_impl_t
     journal_flusher_t *flusher;
     int big_to_flush = 0;
     int write_iodepth = 0;
-    bool alloc_dyn_data = false;
-
-    // clean data blocks referenced by read operations
-    std::map<uint64_t, used_clean_obj_t> used_clean_objects;
 
     bool live = false, queue_stall = false;
     ring_loop_t *ringloop;
@@ -329,30 +310,8 @@ class blockstore_impl_t
 
     // Read
     int dequeue_read(blockstore_op_t *read_op);
-    void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
-        std::function<int(int, bool, uint32_t, uint32_t)> callback);
-    int fulfill_read(blockstore_op_t *read_op,
-        uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-        uint32_t item_state, uint64_t item_version, uint64_t item_location,
-        uint64_t journal_sector, uint8_t *csum, int *dyn_data);
-    bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
-        uint8_t *clean_entry_bitmap, int *dyn_data,
-        uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
-    int fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
-        uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end);
-    int pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
-        uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
-        uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf);
-    bool read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
-        uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end);
-    bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc);
-    uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos);
-    bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
-        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
-    bool verify_journal_checksums(uint8_t *csums, uint32_t offset,
-        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
-    bool verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal,
-        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
+    int fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
+        uint32_t item_state, uint64_t item_version, uint64_t item_location, uint64_t journal_sector);
     int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
         uint32_t item_state, uint64_t item_version);
     void handle_read_event(ring_data_t *data, blockstore_op_t *op);
@@ -383,7 +342,6 @@ class blockstore_impl_t
     int continue_rollback(blockstore_op_t *op);
     void mark_rolled_back(const obj_ver_id & ov);
     void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc);
-    void free_dirty_dyn_data(dirty_entry & e);
 
     // List
     void process_list(blockstore_op_t *op);

src/blockstore_init.cpp

@@ -77,20 +77,13 @@ resume_1:
     if (iszero((uint64_t*)metadata_buffer, bs->dsk.meta_block_size / sizeof(uint64_t)))
     {
         {
-            blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
+            blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)metadata_buffer;
             hdr->zero = 0;
             hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-            hdr->version = bs->dsk.meta_format;
+            hdr->version = BLOCKSTORE_META_VERSION_V1;
             hdr->meta_block_size = bs->dsk.meta_block_size;
             hdr->data_block_size = bs->dsk.data_block_size;
             hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
-            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
-            {
-                hdr->data_csum_type = bs->dsk.data_csum_type;
-                hdr->csum_block_size = bs->dsk.csum_block_size;
-                hdr->header_csum = 0;
-                hdr->header_csum = crc32c(0, hdr, sizeof(*hdr));
-            }
         }
         if (bs->readonly)
         {
@@ -116,62 +109,28 @@ resume_1:
     }
     else
     {
-        blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
-        if (hdr->zero != 0 || hdr->magic != BLOCKSTORE_META_MAGIC_V1 || hdr->version < BLOCKSTORE_META_FORMAT_V1)
+        blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)metadata_buffer;
+        if (hdr->zero != 0 ||
+            hdr->magic != BLOCKSTORE_META_MAGIC_V1 ||
+            hdr->version != BLOCKSTORE_META_VERSION_V1)
         {
             printf(
-                "Metadata is corrupt or too old (pre-0.6.x).\n"
-                " If this is a new OSD, please zero out the metadata area before starting it.\n"
-                " If you need to upgrade from 0.5.x, convert metadata with vitastor-disk.\n"
-            );
-            exit(1);
-        }
-        if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
-        {
-            uint32_t csum = hdr->header_csum;
-            hdr->header_csum = 0;
-            if (crc32c(0, hdr, sizeof(*hdr)) != csum)
-            {
-                printf("Metadata header is corrupt (checksum mismatch).\n");
-                exit(1);
-            }
-            hdr->header_csum = csum;
-            bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V2;
-        }
-        else if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
-        {
-            hdr->data_csum_type = 0;
-            hdr->csum_block_size = 0;
-            hdr->header_csum = 0;
-            // Enable compatibility mode - entries without checksums
-            bs->dsk.clean_entry_size = sizeof(clean_disk_entry) + bs->dsk.clean_entry_bitmap_size*2;
-            bs->dsk.meta_len = (1 + (bs->dsk.block_count - 1 + bs->dsk.meta_block_size / bs->dsk.clean_entry_size)
-                / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size;
-            bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V1;
-            printf("Warning: Starting with metadata in the old format without checksums, as stored on disk\n");
-        }
-        else if (hdr->version > BLOCKSTORE_META_FORMAT_V2)
-        {
-            printf(
-                "Metadata format is too new for me (stored version is %lu, max supported %u).\n",
-                hdr->version, BLOCKSTORE_META_FORMAT_V2
+                "Metadata is corrupt or old version.\n"
+                " If this is a new OSD please zero out the metadata area before starting it.\n"
+                " If you need to upgrade from 0.5.x please request it via the issue tracker.\n"
             );
             exit(1);
         }
         if (hdr->meta_block_size != bs->dsk.meta_block_size ||
             hdr->data_block_size != bs->dsk.data_block_size ||
-            hdr->bitmap_granularity != bs->dsk.bitmap_granularity ||
-            hdr->data_csum_type != bs->dsk.data_csum_type ||
-            hdr->csum_block_size != bs->dsk.csum_block_size)
+            hdr->bitmap_granularity != bs->dsk.bitmap_granularity)
         {
             printf(
                 "Configuration stored in metadata superblock"
-                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
-                " differs from OSD configuration (%lu/%u/%lu, %u/%u).\n",
+                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u)"
+                " differs from OSD configuration (%lu/%u/%lu).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
-                hdr->data_csum_type, hdr->csum_block_size,
-                bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
-                bs->dsk.data_csum_type, bs->dsk.csum_block_size
+                bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity
             );
             exit(1);
         }
@@ -320,22 +279,12 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
     for (uint64_t i = 0; i < max_i; i++)
     {
         clean_disk_entry *entry = (clean_disk_entry*)(buf + i*bs->dsk.clean_entry_size);
+        if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size)
+        {
+            memcpy(bs->clean_bitmap + (done_cnt+i)*2*bs->dsk.clean_entry_bitmap_size, &entry->bitmap, 2*bs->dsk.clean_entry_bitmap_size);
+        }
         if (entry->oid.inode > 0)
         {
-            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
-            {
-                // Check entry crc32
-                uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + bs->dsk.clean_entry_size - 4);
-                if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4))
-                {
-                    printf("Metadata entry %lu is corrupt (checksum mismatch), skipping\n", done_cnt+i);
-                    continue;
-                }
-            }
-            if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size)
-            {
-                memcpy(bs->clean_bitmaps + (done_cnt+i) * 2 * bs->dsk.clean_entry_bitmap_size, &entry->bitmap, 2 * bs->dsk.clean_entry_bitmap_size);
-            }
             auto & clean_db = bs->clean_db_shard(entry->oid);
             auto clean_it = clean_db.find(entry->oid);
             if (clean_it == clean_db.end() || clean_it->second.version < entry->version)
@@ -491,9 +440,7 @@ resume_1:
             .size = sizeof(journal_entry_start),
             .reserved = 0,
             .journal_start = bs->journal.block_size,
-            .version = JOURNAL_VERSION_V2,
-            .data_csum_type = bs->dsk.data_csum_type,
-            .csum_block_size = bs->dsk.csum_block_size,
+            .version = JOURNAL_VERSION,
         };
         ((journal_entry_start*)submitted_buf)->crc32 = je_crc32((journal_entry*)submitted_buf);
         if (bs->readonly)
@@ -545,36 +492,18 @@ resume_1:
         if (je_start->magic != JOURNAL_MAGIC ||
             je_start->type != JE_START ||
             je_crc32((journal_entry*)je_start) != je_start->crc32 ||
-            je_start->size != JE_START_V0_SIZE && je_start->size != JE_START_V1_SIZE && je_start->size != JE_START_V2_SIZE)
+            je_start->size != sizeof(journal_entry_start) && je_start->size != JE_START_LEGACY_SIZE)
         {
             // Entry is corrupt
-            fprintf(stderr, "First entry of the journal is corrupt or unsupported\n");
+            fprintf(stderr, "First entry of the journal is corrupt\n");
             exit(1);
         }
-        if (je_start->size == JE_START_V0_SIZE ||
-            (je_start->version != JOURNAL_VERSION_V1 || je_start->size != JE_START_V1_SIZE) &&
-            (je_start->version != JOURNAL_VERSION_V2 || je_start->size != JE_START_V2_SIZE))
+        if (je_start->size == JE_START_LEGACY_SIZE || je_start->version != JOURNAL_VERSION)
         {
             fprintf(
-                stderr, "The code only supports journal versions 2 and 1, but it is %lu on disk."
-                    " Please use vitastor-disk to rewrite the journal\n",
-                je_start->size == JE_START_V0_SIZE ? 0 : je_start->version
-            );
-            exit(1);
-        }
-        if (je_start->version == JOURNAL_VERSION_V1)
-        {
-            je_start->data_csum_type = 0;
-            je_start->csum_block_size = 0;
-        }
-        if (je_start->data_csum_type != bs->dsk.data_csum_type ||
-            je_start->csum_block_size != bs->dsk.csum_block_size)
-        {
-            printf(
-                "Configuration stored in journal superblock (data_csum_type=%u, csum_block_size=%u)"
-                " differs from OSD configuration (%u/%u).\n",
-                je_start->data_csum_type, je_start->csum_block_size,
-                bs->dsk.data_csum_type, bs->dsk.csum_block_size
+                stderr, "The code only supports journal version %d, but it is %lu on disk."
+                    " Please use the previous version to flush the journal before upgrading OSD\n",
+                JOURNAL_VERSION, je_start->size == JE_START_LEGACY_SIZE ? 0 : je_start->version
             );
             exit(1);
         }
@@ -776,14 +705,11 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     snprintf(err, 1024, "BUG: calculated journal data offset (%08lx) != stored journal data offset (%08lx)", location, je->small_write.data_offset);
                     throw std::runtime_error(err);
                 }
-                small_write_data.clear();
+                uint32_t data_crc32 = 0;
                 if (location >= done_pos && location+je->small_write.len <= done_pos+len)
                 {
                     // data is within this buffer
-                    small_write_data.push_back((iovec){
-                        .iov_base = (uint8_t*)buf + location - done_pos,
-                        .iov_len = je->small_write.len,
-                    });
+                    data_crc32 = crc32c(0, (uint8_t*)buf + location - done_pos, je->small_write.len);
                 }
                 else
                 {
@@ -798,10 +724,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                                 ? location+je->small_write.len : done[i].pos+done[i].len);
                             uint64_t part_begin = (location < done[i].pos ? done[i].pos : location);
                             covered += part_end - part_begin;
-                            small_write_data.push_back((iovec){
-                                .iov_base = (uint8_t*)done[i].buf + part_begin - done[i].pos,
-                                .iov_len = part_end - part_begin,
-                            });
+                            data_crc32 = crc32c(data_crc32, (uint8_t*)done[i].buf + part_begin - done[i].pos, part_end - part_begin);
                         }
                     }
                     if (covered < je->small_write.len)
@@ -811,102 +734,12 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         return 2;
                     }
                 }
-                bool data_csum_valid = true;
-                if (!bs->dsk.csum_block_size)
-                {
-                    uint32_t data_crc32 = 0;
-                    for (auto & sd: small_write_data)
-                    {
-                        data_crc32 = crc32c(data_crc32, sd.iov_base, sd.iov_len);
-                    }
-                    data_csum_valid = data_crc32 == je->small_write.crc32_data;
-                    if (!data_csum_valid)
-                    {
-                        printf(
-                            "Journal entry data is corrupt for small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u - data crc32 %x != %x\n",
-                            je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
-                            je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version,
-                            je->small_write.offset, je->small_write.len,
-                            data_crc32, je->small_write.crc32_data
-                        );
-                    }
-                }
-                else if (je->small_write.len > 0)
-                {
-                    // FIXME: deduplicate with disk_tool_journal.cpp
-                    // like in enqueue_write()
-                    uint32_t start = je->small_write.offset / bs->dsk.csum_block_size;
-                    uint32_t end = (je->small_write.offset+je->small_write.len-1) / bs->dsk.csum_block_size;
-                    uint32_t data_csum_size = (end-start+1) * (bs->dsk.data_csum_type & 0xFF);
-                    uint32_t required_size = sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size + data_csum_size;
-                    if (je->size != required_size)
-                    {
-                        printf(
-                            "Journal entry data has invalid size for small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u - should be %u bytes but is %u bytes\n",
-                            je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
-                            je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version,
-                            je->small_write.offset, je->small_write.len,
-                            required_size, je->size
-                        );
-                        data_csum_valid = false;
-                    }
-                    else
-                    {
-                        int sd_num = 0;
-                        size_t sd_pos = 0;
-                        uint32_t *block_csums = (uint32_t*)((uint8_t*)je + sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size);
-                        for (uint32_t pos = start; pos <= end; pos++, block_csums++)
-                        {
-                            size_t block_left = (pos == start
-                                ? (start == end
-                                    ? je->small_write.len
-                                    : bs->dsk.csum_block_size - je->small_write.offset%bs->dsk.csum_block_size)
-                                : (pos < end
-                                    ? bs->dsk.csum_block_size
-                                    : (je->small_write.offset + je->small_write.len)%bs->dsk.csum_block_size));
-                            if (pos > start && pos == end && block_left == 0)
-                            {
-                                // full last block
-                                block_left = bs->dsk.csum_block_size;
-                            }
-                            uint32_t block_crc32 = 0;
-                            while (block_left > 0)
-                            {
-                                assert(sd_num < small_write_data.size());
-                                if (small_write_data[sd_num].iov_len >= sd_pos+block_left)
-                                {
-                                    block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, block_left);
-                                    sd_pos += block_left;
-                                    break;
-                                }
-                                else
-                                {
-                                    block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, small_write_data[sd_num].iov_len-sd_pos);
-                                    block_left -= (small_write_data[sd_num].iov_len-sd_pos);
-                                    sd_pos = 0;
-                                    sd_num++;
-                                }
-                            }
-                            if (block_crc32 != *block_csums)
-                            {
-                                printf(
-                                    "Journal entry data is corrupt for small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u - block %u crc32 %x != %x\n",
-                                    je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
-                                    je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version,
-                                    je->small_write.offset, je->small_write.len,
-                                    pos, block_crc32, *block_csums
-                                );
-                                data_csum_valid = false;
-                                break;
-                            }
-                        }
-                    }
-                }
-                if (!data_csum_valid)
+                if (data_crc32 != je->small_write.crc32_data)
                 {
                     // journal entry is corrupt, stop here
                     // interesting thing is that we must clear the corrupt entry if we're not readonly,
                     // because we don't write next entries in the same journal block
+                    printf("Journal entry data is corrupt (data crc32 %x != %x)\n", data_crc32, je->small_write.crc32_data);
                     memset((uint8_t*)buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos);
                     bs->journal.next_free = prev_free;
                     init_write_buf = (uint8_t*)buf + proc_pos - done_pos;
@@ -922,14 +755,11 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .oid = je->small_write.oid,
                         .version = je->small_write.version,
                     };
-                    uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->small_write.offset, je->small_write.len);
-                    void *dyn = NULL;
-                    void *dyn_from = (uint8_t*)je + sizeof(journal_entry_small_write);
-                    if (!bs->alloc_dyn_data)
+                    void *bmp = NULL;
+                    void *bmp_from = (uint8_t*)je + sizeof(journal_entry_small_write);
+                    if (bs->dsk.clean_entry_bitmap_size <= sizeof(void*))
                     {
-                        // Bitmap without checksum is only 4 bytes for 128k objects, save it inline
-                        // It can even contain 4 byte bitmap + 4 byte CRC32 for 4 kb writes :)
-                        memcpy(&dyn, dyn_from, dyn_size);
+                        memcpy(&bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
                     }
                     else
                     {
@@ -937,9 +767,8 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         // allocations for entry bitmaps. This can only be fixed by using
                         // a patched map with dynamic entry size, but not the btree_map,
                         // because it doesn't keep iterators valid all the time.
-                        dyn = malloc_or_die(dyn_size+sizeof(int));
-                        *((int*)dyn) = 1;
-                        memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size);
+                        bmp = malloc_or_die(bs->dsk.clean_entry_bitmap_size);
+                        memcpy(bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
                     }
                     bs->dirty_db.emplace(ov, (dirty_entry){
                         .state = (BS_ST_SMALL_WRITE | BS_ST_SYNCED),
@@ -948,7 +777,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .offset = je->small_write.offset,
                         .len = je->small_write.len,
                         .journal_sector = proc_pos,
-                        .dyn_data = dyn,
+                        .bitmap = bmp,
                     });
                     bs->journal.used_sectors[proc_pos]++;
 #ifdef BLOCKSTORE_DEBUG
@@ -961,7 +790,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     unstab = unstab < ov.version ? ov.version : unstab;
                     if (je->type == JE_SMALL_WRITE_INSTANT)
                     {
-                        bs->mark_stable(ov, true);
+                        bs->mark_stable(ov);
                     }
                 }
             }
@@ -1007,13 +836,11 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .oid = je->big_write.oid,
                         .version = je->big_write.version,
                     };
-                    uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->big_write.offset, je->big_write.len);
-                    void *dyn = NULL;
-                    void *dyn_from = (uint8_t*)je + sizeof(journal_entry_big_write);
-                    if (!bs->alloc_dyn_data)
+                    void *bmp = NULL;
+                    void *bmp_from = (uint8_t*)je + sizeof(journal_entry_big_write);
+                    if (bs->dsk.clean_entry_bitmap_size <= sizeof(void*))
                     {
-                        // Bitmap without checksum is only 4 bytes for 128k objects, save it inline
-                        memcpy(&dyn, dyn_from, dyn_size);
+                        memcpy(&bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
                     }
                     else
                     {
@@ -1021,9 +848,8 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         // allocations for entry bitmaps. This can only be fixed by using
                         // a patched map with dynamic entry size, but not the btree_map,
                         // because it doesn't keep iterators valid all the time.
-                        dyn = malloc_or_die(dyn_size+sizeof(int));
-                        *((int*)dyn) = 1;
-                        memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size);
+                        bmp = malloc_or_die(bs->dsk.clean_entry_bitmap_size);
+                        memcpy(bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
                     }
                     auto dirty_it = bs->dirty_db.emplace(ov, (dirty_entry){
                         .state = (BS_ST_BIG_WRITE | BS_ST_SYNCED),
@@ -1032,7 +858,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .offset = je->big_write.offset,
                         .len = je->big_write.len,
                         .journal_sector = proc_pos,
-                        .dyn_data = dyn,
+                        .bitmap = bmp,
                     }).first;
                     if (bs->data_alloc->get(je->big_write.location >> bs->dsk.block_order))
                     {
@@ -1064,7 +890,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     unstab = unstab < ov.version ? ov.version : unstab;
                     if (je->type == JE_BIG_WRITE_INSTANT)
                     {
-                        bs->mark_stable(ov, true);
+                        bs->mark_stable(ov);
                     }
                 }
             }
@@ -1078,7 +904,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     .oid = je->stable.oid,
                     .version = je->stable.version,
                 };
-                bs->mark_stable(ov, true);
+                bs->mark_stable(ov);
             }
             else if (je->type == JE_ROLLBACK)
             {
@@ -1135,7 +961,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     bs->journal.used_sectors[proc_pos]++;
                     // Deletions are treated as immediately stable, because
                     // "2-phase commit" (write->stabilize) isn't sufficient for them anyway
-                    bs->mark_stable(ov, true);
+                    bs->mark_stable(ov);
                 }
                 // Ignore delete if neither preceding dirty entries nor the clean one are present
             }

src/blockstore_init.h

@@ -50,7 +50,6 @@ class blockstore_init_journal
     uint64_t next_free;
     std::vector<bs_init_journal_done> done;
     std::vector<obj_ver_id> double_allocs;
-    std::vector<iovec> small_write_data;
     uint64_t journal_pos = 0;
     uint64_t continue_pos = 0;
     void *init_write_buf = NULL;

src/blockstore_journal.cpp

@@ -17,7 +17,6 @@ blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
 // Check if we can write <required> entries of <size> bytes and <data_after> data bytes after them to the journal
 int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries_required, int size, int data_after)
 {
-    uint64_t prev_next = next_sector;
     int required = entries_required;
     while (1)
     {
@@ -36,19 +35,11 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
             }
             required -= fits;
             next_in_pos += fits * size;
-            if (next_sector != prev_next || !sectors_to_write)
-            {
-                // Except the previous call to this function
-                sectors_to_write++;
-            }
+            sectors_to_write++;
         }
         else if (bs->journal.sector_info[next_sector].dirty)
         {
-            if (next_sector != prev_next || !sectors_to_write)
-            {
-                // Except the previous call to this function
-                sectors_to_write++;
-            }
+            sectors_to_write++;
         }
         if (required <= 0)
         {
@@ -198,7 +189,6 @@ void blockstore_impl_t::prepare_journal_sector_write(int cur_sector, blockstore_
     priv->pending_ops++;
     if (!priv->min_flushed_journal_sector)
         priv->min_flushed_journal_sector = 1+cur_sector;
-    assert(priv->min_flushed_journal_sector <= journal.sector_count);
     priv->max_flushed_journal_sector = 1+cur_sector;
 }
 
@@ -246,6 +236,14 @@ journal_t::~journal_t()
 uint64_t journal_t::get_trim_pos()
 {
     auto journal_used_it = used_sectors.lower_bound(used_start);
+#ifdef BLOCKSTORE_DEBUG
+    printf(
+        "Trimming journal (used_start=%08lx, next_free=%08lx, dirty_start=%08lx, new_start=%08lx, new_refcount=%ld)\n",
+        used_start, next_free, dirty_start,
+        journal_used_it == used_sectors.end() ? 0 : journal_used_it->first,
+        journal_used_it == used_sectors.end() ? 0 : journal_used_it->second
+    );
+#endif
     if (journal_used_it == used_sectors.end())
     {
         // Journal is cleared to its end, restart from the beginning
@@ -258,26 +256,12 @@ uint64_t journal_t::get_trim_pos()
         else
         {
             // next_free does not need updating during trim
-#ifdef BLOCKSTORE_DEBUG
-            printf(
-                "Trimming journal (used_start=%08lx, next_free=%08lx, dirty_start=%08lx, new_start=%08lx, new_refcount=%ld)\n",
-                used_start, next_free, dirty_start,
-                journal_used_it->first, journal_used_it->second
-            );
-#endif
             return journal_used_it->first;
         }
     }
     else if (journal_used_it->first > used_start)
     {
         // Journal is cleared up to <journal_used_it>
-#ifdef BLOCKSTORE_DEBUG
-        printf(
-            "Trimming journal (used_start=%08lx, next_free=%08lx, dirty_start=%08lx, new_start=%08lx, new_refcount=%ld)\n",
-            used_start, next_free, dirty_start,
-            journal_used_it->first, journal_used_it->second
-        );
-#endif
         return journal_used_it->first;
     }
     // Can't trim journal
@@ -299,31 +283,3 @@ void journal_t::dump_diagnostics()
         journal_used_it == used_sectors.end() ? 0 : journal_used_it->second
     );
 }
-
-static uint64_t zero_page[4096];
-
-uint32_t crc32c_pad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad)
-{
-    uint32_t r = prev_crc;
-    while (left_pad >= 4096)
-    {
-        r = crc32c(r, zero_page, 4096);
-        left_pad -= 4096;
-    }
-    if (left_pad > 0)
-        r = crc32c(r, zero_page, left_pad);
-    r = crc32c(r, buf, len);
-    while (right_pad >= 4096)
-    {
-        r = crc32c(r, zero_page, 4096);
-        right_pad -= 4096;
-    }
-    if (left_pad > 0)
-        r = crc32c(r, zero_page, right_pad);
-    return r;
-}
-
-uint32_t crc32c_nopad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad)
-{
-    return crc32c(0, buf, len);
-}

src/blockstore_journal.h

@@ -8,8 +8,7 @@
 
 #define MIN_JOURNAL_SIZE 4*1024*1024
 #define JOURNAL_MAGIC 0x4A33
-#define JOURNAL_VERSION_V1 1
-#define JOURNAL_VERSION_V2 2
+#define JOURNAL_VERSION 1
 #define JOURNAL_BUFFER_SIZE 4*1024*1024
 #define JOURNAL_ENTRY_HEADER_SIZE 16
 
@@ -33,7 +32,7 @@
 #define JE_BIG_WRITE_INSTANT   0x08
 #define JE_MAX         0x08
 
-// crc32c comes first to ease calculation
+// crc32c comes first to ease calculation and is equal to crc32()
 struct __attribute__((__packed__)) journal_entry_start
 {
     uint32_t crc32;
@@ -43,12 +42,8 @@ struct __attribute__((__packed__)) journal_entry_start
     uint32_t reserved;
     uint64_t journal_start;
     uint64_t version;
-    uint32_t data_csum_type;
-    uint32_t csum_block_size;
 };
-#define JE_START_V0_SIZE 24
-#define JE_START_V1_SIZE 32
-#define JE_START_V2_SIZE 40
+#define JE_START_LEGACY_SIZE 24
 
 struct __attribute__((__packed__)) journal_entry_small_write
 {
@@ -64,12 +59,10 @@ struct __attribute__((__packed__)) journal_entry_small_write
     // small_write entries contain <len> bytes of data which is stored in next sectors
     // data_offset is its offset within journal
     uint64_t data_offset;
-    uint32_t crc32_data; // zero when data_csum_type != 0
+    uint32_t crc32_data;
     // small_write and big_write entries are followed by the "external" bitmap
     // its size is dynamic and included in journal entry's <size> field
     uint8_t bitmap[];
-    // and then data checksums if data_csum_type != 0
-    // uint32_t data_crc32c[];
 };
 
 struct __attribute__((__packed__)) journal_entry_big_write
@@ -87,8 +80,6 @@ struct __attribute__((__packed__)) journal_entry_big_write
     // small_write and big_write entries are followed by the "external" bitmap
     // its size is dynamic and included in journal entry's <size> field
     uint8_t bitmap[];
-    // and then data checksums if data_csum_type != 0
-    // uint32_t data_crc32c[];
 };
 
 struct __attribute__((__packed__)) journal_entry_stable
@@ -227,6 +218,3 @@ struct blockstore_journal_check_t
 };
 
 journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size);
-
-uint32_t crc32c_pad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad);
-uint32_t crc32c_nopad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad);

src/blockstore_open.cpp

@@ -85,13 +85,11 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
         immediate_commit = IMMEDIATE_SMALL;
     }
     metadata_buf_size = strtoull(config["meta_buf_size"].c_str(), NULL, 10);
-    inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" &&
-        config["inmemory_metadata"] != "no";
+    inmemory_meta = config["inmemory_metadata"] != "false";
     journal.sector_count = strtoull(config["journal_sector_buffer_count"].c_str(), NULL, 10);
     journal.no_same_sector_overwrites = config["journal_no_same_sector_overwrites"] == "true" ||
         config["journal_no_same_sector_overwrites"] == "1" || config["journal_no_same_sector_overwrites"] == "yes";
-    journal.inmemory = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" &&
-        config["inmemory_journal"] != "no";
+    journal.inmemory = config["inmemory_journal"] != "false";
     // Validate
     if (journal.sector_count < 2)
     {
@@ -135,24 +133,19 @@ void blockstore_impl_t::calc_lengths()
     {
         metadata_buffer = memalign(MEM_ALIGNMENT, dsk.meta_len);
         if (!metadata_buffer)
-            throw std::runtime_error("Failed to allocate memory for the metadata ("+std::to_string(dsk.meta_len/1024/1024)+" MB)");
+            throw std::runtime_error("Failed to allocate memory for the metadata");
     }
-    else if (dsk.clean_entry_bitmap_size || dsk.data_csum_type)
+    else if (dsk.clean_entry_bitmap_size)
     {
-        clean_bitmaps = (uint8_t*)malloc(dsk.block_count * 2 * dsk.clean_entry_bitmap_size);
-        if (!clean_bitmaps)
-        {
-            throw std::runtime_error(
-                "Failed to allocate memory for the metadata sparse write bitmap ("+
-                std::to_string(dsk.block_count * 2 * dsk.clean_entry_bitmap_size / 1024 / 1024)+" MB)"
-            );
-        }
+        clean_bitmap = (uint8_t*)malloc(dsk.block_count * 2*dsk.clean_entry_bitmap_size);
+        if (!clean_bitmap)
+            throw std::runtime_error("Failed to allocate memory for the metadata sparse write bitmap");
     }
     if (journal.inmemory)
     {
         journal.buffer = memalign(MEM_ALIGNMENT, journal.len);
         if (!journal.buffer)
-            throw std::runtime_error("Failed to allocate memory for journal ("+std::to_string(journal.len/1024/1024)+" MB)");
+            throw std::runtime_error("Failed to allocate memory for journal");
     }
     else
     {

src/blockstore_read.cpp

@@ -1,7 +1,6 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
-#include <limits.h>
 #include "blockstore_impl.h"
 
 int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
@@ -9,7 +8,12 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
 {
     if (!len)
     {
-        // Zero-length read
+        // Zero-length version - skip
+        return 1;
+    }
+    else if (IS_IN_FLIGHT(item_state))
+    {
+        // Write not finished yet - skip
         return 1;
     }
     else if (IS_DELETE(item_state))
@@ -18,7 +22,6 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
         memset(buf, 0, len);
         return 1;
     }
-    assert(!IS_IN_FLIGHT(item_state));
     if (journal.inmemory && IS_JOURNAL(item_state))
     {
         memcpy(buf, (uint8_t*)journal.buffer + offset, len);
@@ -37,115 +40,59 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
     return 1;
 }
 
-void blockstore_impl_t::find_holes(std::vector<copy_buffer_t> & read_vec,
-    uint32_t item_start, uint32_t item_end,
-    std::function<int(int, bool, uint32_t, uint32_t)> callback)
+// FIXME I've seen a bug here so I want some tests
+int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
+    uint32_t item_state, uint64_t item_version, uint64_t item_location, uint64_t journal_sector)
 {
-    auto cur_start = item_start;
-    int i = 0;
-    while (cur_start < item_end)
+    uint32_t cur_start = item_start;
+    if (cur_start < read_op->offset + read_op->len && item_end > read_op->offset)
     {
-        // COPY_BUF_CSUM_FILL items are fake items inserted in the end, their offsets aren't in order
-        if (i >= read_vec.size() || read_vec[i].copy_flags & COPY_BUF_CSUM_FILL || read_vec[i].offset >= item_end)
+        cur_start = cur_start < read_op->offset ? read_op->offset : cur_start;
+        item_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end;
+        auto it = PRIV(read_op)->read_vec.begin();
+        while (1)
         {
-            // Hole (at end): cur_start .. item_end
-            i += callback(i, false, cur_start, item_end);
-            break;
-        }
-        else if (read_vec[i].offset > cur_start)
-        {
-            // Hole: cur_start .. min(read_vec[i].offset, item_end)
-            auto cur_end = read_vec[i].offset > item_end ? item_end : read_vec[i].offset;
-            i += callback(i, false, cur_start, cur_end);
-            cur_start = cur_end;
-        }
-        else if (read_vec[i].offset + read_vec[i].len > cur_start)
-        {
-            // Allocated: cur_start .. min(read_vec[i].offset + read_vec[i].len, item_end)
-            auto cur_end = read_vec[i].offset + read_vec[i].len;
-            cur_end = cur_end > item_end ? item_end : cur_end;
-            i += callback(i, true, cur_start, cur_end);
-            cur_start = cur_end;
-            i++;
-        }
-        else
-            i++;
-    }
-}
-
-int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op,
-    uint64_t &fulfilled, uint32_t item_start, uint32_t item_end, // FIXME: Rename item_* to dirty_*
-    uint32_t item_state, uint64_t item_version, uint64_t item_location,
-    uint64_t journal_sector, uint8_t *csum, int *dyn_data)
-{
-    int r = 1;
-    if (item_start < read_op->offset + read_op->len && item_end > read_op->offset)
-    {
-        auto & rv = PRIV(read_op)->read_vec;
-        auto rd_start = item_start < read_op->offset ? read_op->offset : item_start;
-        auto rd_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end;
-        find_holes(rv, rd_start, rd_end, [&](int pos, bool alloc, uint32_t start, uint32_t end)
-        {
-            if (!r || alloc)
-                return 0;
-            if (!journal.inmemory && dsk.csum_block_size > dsk.bitmap_granularity && IS_JOURNAL(item_state) && !IS_DELETE(item_state))
+            for (; it != PRIV(read_op)->read_vec.end(); it++)
             {
-                uint32_t blk_begin = (start/dsk.csum_block_size) * dsk.csum_block_size;
-                blk_begin = blk_begin < item_start ? item_start : blk_begin;
-                uint32_t blk_end = ((end-1) / dsk.csum_block_size + 1) * dsk.csum_block_size;
-                blk_end = blk_end > item_end ? item_end : blk_end;
-                rv.push_back((copy_buffer_t){
-                    .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL,
-                    .offset = blk_begin,
-                    .len = blk_end-blk_begin,
-                    .csum_buf = (csum + (blk_begin/dsk.csum_block_size -
-                        item_start/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)),
-                    .dyn_data = dyn_data,
-                });
-                if (dyn_data)
+                if (it->offset >= cur_start)
                 {
-                    (*dyn_data)++;
+                    break;
                 }
-                // Submit the journal checksum block read
-                if (!read_checksum_block(read_op, 1, fulfilled, item_location - item_start))
+                else if (it->offset + it->len > cur_start)
                 {
-                    r = 0;
+                    cur_start = it->offset + it->len;
+                    if (cur_start >= item_end)
+                    {
+                        goto endwhile;
+                    }
                 }
-                return 0;
             }
-            copy_buffer_t el = {
-                .copy_flags = (IS_JOURNAL(item_state) ? COPY_BUF_JOURNAL : COPY_BUF_DATA),
-                .offset = start,
-                .len = end-start,
-                .disk_offset = item_location + start - item_start,
-                .journal_sector = (IS_JOURNAL(item_state) ? journal_sector : 0),
-                .csum_buf = !csum ? NULL : (csum + (start - item_start) / dsk.csum_block_size * (dsk.data_csum_type & 0xFF)),
-                .dyn_data = dyn_data,
-            };
-            if (dyn_data)
+            if (it == PRIV(read_op)->read_vec.end() || it->offset > cur_start)
             {
-                (*dyn_data)++;
+                fulfill_read_t el = {
+                    .offset = cur_start,
+                    .len = it == PRIV(read_op)->read_vec.end() || it->offset >= item_end ? item_end-cur_start : it->offset-cur_start,
+                    .journal_sector = journal_sector,
+                };
+                it = PRIV(read_op)->read_vec.insert(it, el);
+                if (!fulfill_read_push(read_op,
+                    (uint8_t*)read_op->buf + el.offset - read_op->offset,
+                    item_location + el.offset - item_start,
+                    el.len, item_state, item_version))
+                {
+                    return 0;
+                }
+                fulfilled += el.len;
             }
-            if (IS_BIG_WRITE(item_state))
+            cur_start = it->offset + it->len;
+            if (it == PRIV(read_op)->read_vec.end() || cur_start >= item_end)
             {
-                // If we don't track it then we may IN THEORY read another object's data:
-                // submit read -> remove the object -> flush remove -> overwrite with another object -> finish read
-                // Very improbable, but possible
-                PRIV(read_op)->clean_block_used = 1;
+                break;
             }
-            rv.insert(rv.begin() + pos, el);
-            fulfilled += el.len;
-            if (!fulfill_read_push(read_op,
-                (uint8_t*)read_op->buf + el.offset - read_op->offset,
-                item_location + el.offset - item_start,
-                el.len, item_state, item_version))
-            {
-                r = 0;
-            }
-            return 1;
-        });
+        }
     }
-    return r;
+endwhile:
+    return 1;
 }
 
 uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset)
@@ -159,225 +106,10 @@ uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offse
         clean_entry_bitmap = ((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size + sizeof(clean_disk_entry) + offset);
     }
     else
-        clean_entry_bitmap = (uint8_t*)(clean_bitmaps + meta_loc*2*dsk.clean_entry_bitmap_size + offset);
+        clean_entry_bitmap = (uint8_t*)(clean_bitmap + meta_loc*2*dsk.clean_entry_bitmap_size + offset);
     return clean_entry_bitmap;
 }
 
-int blockstore_impl_t::fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
-    uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end)
-{
-    if (read_end == read_offset)
-        return 0;
-    int required = 0;
-    read_buf -= read_offset;
-    uint32_t last_block = (read_end-1)/dsk.csum_block_size;
-    uint32_t start_block = read_offset/dsk.csum_block_size;
-    uint32_t end_block = 0;
-    while (start_block <= last_block)
-    {
-        if (read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap,
-            start_block*dsk.csum_block_size < read_offset ? read_offset : start_block*dsk.csum_block_size,
-            (start_block+1)*dsk.csum_block_size > read_end ? read_end : (start_block+1)*dsk.csum_block_size))
-        {
-            // read_range_fulfilled() also adds zero-filled areas
-            start_block++;
-        }
-        else
-        {
-            // Find a sequence of checksum blocks required to be read
-            end_block = start_block;
-            while ((end_block+1)*dsk.csum_block_size < read_end &&
-                !read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap,
-                    (end_block+1)*dsk.csum_block_size < read_offset ? read_offset : (end_block+1)*dsk.csum_block_size,
-                    (end_block+2)*dsk.csum_block_size > read_end ? read_end : (end_block+2)*dsk.csum_block_size))
-            {
-                end_block++;
-            }
-            end_block++;
-            // OK, mark this range as required
-            rv.push_back((copy_buffer_t){
-                .copy_flags = COPY_BUF_CSUM_FILL | (from_journal ? COPY_BUF_JOURNALED_BIG : 0),
-                .offset = start_block*dsk.csum_block_size,
-                .len = (end_block-start_block)*dsk.csum_block_size,
-                // save clean_entry_bitmap if we're reading clean data from the journal
-                .csum_buf = from_journal ? clean_entry_bitmap : NULL,
-                .dyn_data = dyn_data,
-            });
-            if (dyn_data)
-            {
-                (*dyn_data)++;
-            }
-            start_block = end_block;
-            required++;
-        }
-    }
-    return required;
-}
-
-// read_buf should be == op->buf - op->offset
-bool blockstore_impl_t::read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
-    uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end)
-{
-    bool all_done = true;
-    find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
-    {
-        if (alloc)
-            return 0;
-        int diff = 0;
-        uint32_t bmp_start = cur_start/dsk.bitmap_granularity;
-        uint32_t bmp_end = cur_end/dsk.bitmap_granularity;
-        uint32_t bmp_pos = bmp_start;
-        while (bmp_pos < bmp_end)
-        {
-            while (bmp_pos < bmp_end && !(clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7))))
-                bmp_pos++;
-            if (bmp_pos > bmp_start)
-            {
-                // zero fill
-                copy_buffer_t el = {
-                    .copy_flags = COPY_BUF_ZERO,
-                    .offset = bmp_start*dsk.bitmap_granularity,
-                    .len = (bmp_pos-bmp_start)*dsk.bitmap_granularity,
-                };
-                rv.insert(rv.begin() + pos, el);
-                if (read_buf)
-                    memset(read_buf + el.offset, 0, el.len);
-                fulfilled += el.len;
-                diff++;
-            }
-            bmp_start = bmp_pos;
-            while (bmp_pos < bmp_end && (clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7))))
-                bmp_pos++;
-            if (bmp_pos > bmp_start)
-            {
-                // something is to be read
-                all_done = false;
-            }
-            bmp_start = bmp_pos;
-        }
-        return diff;
-    });
-    return all_done;
-}
-
-bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc)
-{
-    auto & rv = PRIV(op)->read_vec;
-    auto *vi = &rv[rv.size()-rv_pos];
-    uint32_t item_start = vi->offset, item_end = vi->offset+vi->len;
-    uint32_t fill_size = 0;
-    int n_iov = 0;
-    find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
-    {
-        if (alloc)
-        {
-            fill_size += cur_end-cur_start;
-            n_iov++;
-        }
-        else
-        {
-            if (cur_start < op->offset)
-            {
-                fill_size += op->offset-cur_start;
-                n_iov++;
-                cur_start = op->offset;
-            }
-            if (cur_end > op->offset+op->len)
-            {
-                fill_size += cur_end-(op->offset+op->len);
-                n_iov++;
-                cur_end = op->offset+op->len;
-            }
-            if (cur_end > cur_start)
-            {
-                n_iov++;
-            }
-        }
-        return 0;
-    });
-    void *buf = memalign_or_die(MEM_ALIGNMENT, fill_size + n_iov*sizeof(struct iovec));
-    iovec *iov = (struct iovec*)((uint8_t*)buf+fill_size);
-    n_iov = 0;
-    fill_size = 0;
-    find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
-    {
-        int res = 0;
-        if (alloc)
-        {
-            iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end-cur_start };
-            fill_size += cur_end-cur_start;
-        }
-        else
-        {
-            if (cur_start < op->offset)
-            {
-                iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, op->offset-cur_start };
-                fill_size += op->offset-cur_start;
-                cur_start = op->offset;
-            }
-            auto lim_end = cur_end > op->offset+op->len ? op->offset+op->len : cur_end;
-            if (lim_end > cur_start)
-            {
-                iov[n_iov++] = (struct iovec){ (uint8_t*)op->buf+cur_start-op->offset, lim_end-cur_start };
-                rv.insert(rv.begin() + pos, (copy_buffer_t){
-                    .copy_flags = COPY_BUF_DATA,
-                    .offset = cur_start,
-                    .len = lim_end-cur_start,
-                });
-                fulfilled += lim_end-cur_start;
-                res++;
-            }
-            if (cur_end > op->offset+op->len)
-            {
-                iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end - (op->offset+op->len) };
-                fill_size += cur_end - (op->offset+op->len);
-                cur_end = op->offset+op->len;
-            }
-        }
-        return res;
-    });
-    vi = &rv[rv.size()-rv_pos];
-    // Save buf into read_vec too but in a creepy way
-    // FIXME: Shit, something else should be invented %)
-    *vi = (copy_buffer_t){
-        .copy_flags = vi->copy_flags,
-        .offset = vi->offset,
-        .len = ((uint64_t)n_iov << 32) | fill_size,
-        .disk_offset = clean_loc + item_start,
-        .buf = (uint8_t*)buf,
-        .csum_buf = vi->csum_buf,
-        .dyn_data = vi->dyn_data,
-    };
-    int submit_fd = (vi->copy_flags & COPY_BUF_JOURNAL ? dsk.journal_fd : dsk.data_fd);
-    uint64_t submit_offset = (vi->copy_flags & COPY_BUF_JOURNAL ? journal.offset : dsk.data_offset);
-    uint32_t d_pos = 0;
-    for (int n_pos = 0; n_pos < n_iov; n_pos += IOV_MAX)
-    {
-        int n_cur = n_iov-n_pos < IOV_MAX ? n_iov-n_pos : IOV_MAX;
-        BS_SUBMIT_GET_SQE(sqe, data);
-        PRIV(op)->pending_ops++;
-        my_uring_prep_readv(sqe, submit_fd, iov + n_pos, n_cur, submit_offset + clean_loc + item_start + d_pos);
-        data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
-        if (n_pos > 0 || n_pos + IOV_MAX < n_iov)
-        {
-            uint32_t d_len = 0;
-            for (int i = 0; i < IOV_MAX; i++)
-                d_len += iov[n_pos+i].iov_len;
-            data->iov.iov_len = d_len;
-            d_pos += d_len;
-        }
-        else
-            data->iov.iov_len = item_end-item_start;
-    }
-    if (!(vi->copy_flags & COPY_BUF_JOURNAL))
-    {
-        // Reads running parallel to flushes of the same clean block may read
-        // a mixture of old and new data. So we don't verify checksums for such blocks.
-        PRIV(op)->clean_block_used = 1;
-    }
-    return true;
-}
-
 int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
 {
     auto & clean_db = clean_db_shard(read_op->oid);
@@ -399,8 +131,6 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     }
     uint64_t fulfilled = 0;
     PRIV(read_op)->pending_ops = 0;
-    PRIV(read_op)->clean_block_used = 0;
-    auto & rv = PRIV(read_op)->read_vec;
     uint64_t result_version = 0;
     if (dirty_found)
     {
@@ -408,6 +138,10 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         {
             dirty_entry& dirty = dirty_it->second;
             bool version_ok = !IS_IN_FLIGHT(dirty.state) && read_op->version >= dirty_it->first.version;
+            if (IS_SYNCED(dirty.state))
+            {
+                version_ok = true;
+            }
             if (version_ok)
             {
                 if (IS_DELETE(dirty.state))
@@ -418,36 +152,23 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
                     FINISH_OP(read_op);
                     return 2;
                 }
-                int *dyn_data = (int*)(dsk.csum_block_size > 0 && alloc_dyn_data ? dirty.dyn_data : NULL);
-                uint8_t *bmp_ptr = (alloc_dyn_data
-                    ? (uint8_t*)dirty.dyn_data + sizeof(int) : (uint8_t*)&dirty.dyn_data);
                 if (!result_version)
                 {
                     result_version = dirty_it->first.version;
                     if (read_op->bitmap)
                     {
+                        void *bmp_ptr = (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap);
                         memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
                     }
                 }
                 // If inmemory_journal is false, journal trim will have to wait until the read is completed
-                if (!IS_JOURNAL(dirty.state))
+                if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
+                    dirty.state, dirty_it->first.version, dirty.location + (IS_JOURNAL(dirty.state) ? 0 : dirty.offset),
+                    (IS_JOURNAL(dirty.state) ? dirty.journal_sector+1 : 0)))
                 {
-                    // Read from data disk, possibly checking checksums
-                    if (!fulfill_clean_read(read_op, fulfilled, bmp_ptr, dyn_data,
-                        dirty.offset, dirty.offset+dirty.len, dirty.location, dirty_it->first.version))
-                    {
-                        goto undo_read;
-                    }
-                }
-                else
-                {
-                    // Copy from memory or read from journal, possibly checking checksums
-                    if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
-                        dirty.state, dirty_it->first.version, dirty.location, dirty.journal_sector+1,
-                        journal.inmemory ? NULL : bmp_ptr+dsk.clean_entry_bitmap_size, dyn_data))
-                    {
-                        goto undo_read;
-                    }
+                    // need to wait. undo added requests, don't dequeue op
+                    PRIV(read_op)->read_vec.clear();
+                    return 0;
                 }
             }
             if (fulfilled == read_op->len || dirty_it == dirty_db.begin())
@@ -470,10 +191,50 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         }
         if (fulfilled < read_op->len)
         {
-            if (!fulfill_clean_read(read_op, fulfilled, NULL, NULL, 0, dsk.data_block_size,
-                clean_it->second.location, clean_it->second.version))
+            if (!dsk.clean_entry_bitmap_size)
             {
-                goto undo_read;
+                if (!fulfill_read(read_op, fulfilled, 0, dsk.data_block_size,
+                    (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_it->second.location, 0))
+                {
+                    // need to wait. undo added requests, don't dequeue op
+                    PRIV(read_op)->read_vec.clear();
+                    return 0;
+                }
+            }
+            else
+            {
+                uint8_t *clean_entry_bitmap = get_clean_entry_bitmap(clean_it->second.location, 0);
+                uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity;
+                while (bmp_start < bmp_size)
+                {
+                    while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size)
+                    {
+                        bmp_end++;
+                    }
+                    if (bmp_end > bmp_start)
+                    {
+                        // fill with zeroes
+                        assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
+                            bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0));
+                    }
+                    bmp_start = bmp_end;
+                    while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
+                    {
+                        bmp_end++;
+                    }
+                    if (bmp_end > bmp_start)
+                    {
+                        if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
+                            bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0,
+                            clean_it->second.location + bmp_start * dsk.bitmap_granularity, 0))
+                        {
+                            // need to wait. undo added requests, don't dequeue op
+                            PRIV(read_op)->read_vec.clear();
+                            return 0;
+                        }
+                        bmp_start = bmp_end;
+                    }
+                }
             }
         }
     }
@@ -485,7 +246,11 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         FINISH_OP(read_op);
         return 2;
     }
-    assert(fulfilled == read_op->len);
+    if (fulfilled < read_op->len)
+    {
+        assert(fulfill_read(read_op, fulfilled, 0, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0));
+        assert(fulfilled == read_op->len);
+    }
     read_op->version = result_version;
     if (!PRIV(read_op)->pending_ops)
     {
@@ -510,309 +275,6 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     }
     read_op->retval = 0;
     return 2;
-undo_read:
-    // need to wait. undo added requests, don't dequeue op
-    if (dsk.csum_block_size > dsk.bitmap_granularity)
-    {
-        for (auto & vec: rv)
-        {
-            if ((vec.copy_flags & COPY_BUF_CSUM_FILL) && vec.buf)
-            {
-                free(vec.buf);
-                vec.buf = NULL;
-            }
-            if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount
-            {
-                free(vec.dyn_data);
-                vec.dyn_data = NULL;
-            }
-        }
-    }
-    rv.clear();
-    return 0;
-}
-
-int blockstore_impl_t::pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
-    // FIXME Passing dirty_entry& would be nicer
-    uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
-    uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf)
-{
-    if (offset % dsk.csum_block_size || submit_len % dsk.csum_block_size)
-    {
-        if (offset < blk_end)
-        {
-            // Already being read as a part of the previous checksum block series
-            cp.buf = blk_buf + offset - blk_begin;
-            cp.copy_flags |= COPY_BUF_COALESCED;
-            if (offset+submit_len > blk_end)
-                cp.len = blk_end-offset;
-            return 2;
-        }
-        else
-        {
-            // We don't use fill_partial_checksum_blocks for journal because journal writes never have holes (internal bitmap)
-            blk_begin = (offset/dsk.csum_block_size) * dsk.csum_block_size;
-            blk_begin = blk_begin < dirty_offset ? dirty_offset : blk_begin;
-            blk_end = ((offset+submit_len-1)/dsk.csum_block_size + 1) * dsk.csum_block_size;
-            blk_end = blk_end > dirty_end ? dirty_end : blk_end;
-            if (blk_begin < offset || blk_end > offset+submit_len)
-            {
-                blk_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end-blk_begin);
-                cp.buf = blk_buf + offset - blk_begin;
-                cp.copy_flags |= COPY_BUF_COALESCED;
-                rv.push_back((copy_buffer_t){
-                    .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL,
-                    .offset = blk_begin,
-                    .len = blk_end-blk_begin,
-                    .disk_offset = dirty_loc + blk_begin - dirty_offset,
-                    .buf = blk_buf,
-                    .csum_buf = (csum_ptr + (blk_begin/dsk.csum_block_size -
-                        dirty_offset/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)),
-                    .dyn_data = dyn_data,
-                });
-                if (dyn_data)
-                {
-                    (*dyn_data)++;
-                }
-                return 1;
-            }
-        }
-    }
-    return 0;
-}
-
-bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
-    uint8_t *clean_entry_bitmap, int *dyn_data, uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver)
-{
-    bool from_journal = clean_entry_bitmap != NULL;
-    if (!clean_entry_bitmap)
-    {
-        // NULL clean_entry_bitmap means we're reading from data, not from the journal,
-        // and the bitmap location is obvious
-        clean_entry_bitmap = get_clean_entry_bitmap(clean_loc, 0);
-    }
-    if (dsk.csum_block_size > dsk.bitmap_granularity)
-    {
-        auto & rv = PRIV(read_op)->read_vec;
-        int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, from_journal,
-            (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len);
-        if (!inmemory_meta && !from_journal && req > 0)
-        {
-            // Read checksums from disk
-            uint8_t *csum_buf = read_clean_meta_block(read_op, clean_loc, rv.size()-req);
-            for (int i = req; i > 0; i--)
-            {
-                rv[rv.size()-i].csum_buf = csum_buf;
-            }
-        }
-        for (int i = req; i > 0; i--)
-        {
-            if (!read_checksum_block(read_op, i, fulfilled, clean_loc))
-            {
-                return false;
-            }
-        }
-        PRIV(read_op)->clean_block_used = req > 0;
-    }
-    else if (from_journal)
-    {
-        // Don't scan bitmap - journal writes don't have holes (internal bitmap)!
-        uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size +
-            item_start/dsk.csum_block_size*(dsk.data_csum_type & 0xFF));
-        if (!fulfill_read(read_op, fulfilled, item_start, item_end,
-            (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_loc + item_start, 0, csum, dyn_data))
-        {
-            return false;
-        }
-        if (item_start > 0 && fulfilled < read_op->len)
-        {
-            // fill with zeroes
-            assert(fulfill_read(read_op, fulfilled, 0, item_start, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
-        }
-        if (item_end < dsk.data_block_size && fulfilled < read_op->len)
-        {
-            // fill with zeroes
-            assert(fulfill_read(read_op, fulfilled, item_end, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
-        }
-    }
-    else
-    {
-        bool csum_done = !dsk.csum_block_size || inmemory_meta;
-        uint8_t *csum_buf = clean_entry_bitmap;
-        uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity;
-        while (bmp_start < bmp_size)
-        {
-            while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size)
-            {
-                bmp_end++;
-            }
-            if (bmp_end > bmp_start)
-            {
-                // fill with zeroes
-                assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
-                    bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
-            }
-            bmp_start = bmp_end;
-            while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
-            {
-                bmp_end++;
-            }
-            if (bmp_end > bmp_start)
-            {
-                if (!csum_done)
-                {
-                    // Read checksums from disk
-                    csum_buf = read_clean_meta_block(read_op, clean_loc, PRIV(read_op)->read_vec.size());
-                    csum_done = true;
-                }
-                uint8_t *csum = !dsk.csum_block_size ? 0 : (csum_buf + 2*dsk.clean_entry_bitmap_size + bmp_start*(dsk.data_csum_type & 0xFF));
-                if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
-                    bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0,
-                    clean_loc + bmp_start * dsk.bitmap_granularity, 0, csum, dyn_data))
-                {
-                    return false;
-                }
-                bmp_start = bmp_end;
-            }
-        }
-    }
-    // Increment reference counter if clean data is being read from the disk
-    if (PRIV(read_op)->clean_block_used)
-    {
-        auto & uo = used_clean_objects[clean_loc];
-        uo.refs++;
-        if (dsk.csum_block_size && flusher->is_mutated(clean_loc))
-            uo.was_changed = true;
-        PRIV(read_op)->clean_block_used = clean_loc;
-    }
-    return true;
-}
-
-uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, uint64_t clean_loc, int rv_pos)
-{
-    auto & rv = PRIV(op)->read_vec;
-    auto sector = ((clean_loc >> dsk.block_order) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
-    auto pos = ((clean_loc >> dsk.block_order) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size;
-    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
-    rv.insert(rv.begin()+rv_pos, (copy_buffer_t){
-        .copy_flags = COPY_BUF_META_BLOCK|COPY_BUF_CSUM_FILL,
-        .offset = pos,
-        .buf = buf,
-    });
-    BS_SUBMIT_GET_SQE(sqe, data);
-    data->iov = (struct iovec){ buf, dsk.meta_block_size };
-    PRIV(op)->pending_ops++;
-    my_uring_prep_readv(sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + dsk.meta_block_size + sector);
-    data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
-    // return pointer to checksums + bitmap
-    return buf + pos + sizeof(clean_disk_entry);
-}
-
-bool blockstore_impl_t::verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
-    iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb)
-{
-    assert(!(offset % dsk.csum_block_size));
-    uint32_t *csums = (uint32_t*)csum_buf;
-    uint32_t block_csum = 0;
-    uint32_t block_done = 0;
-    uint32_t block_num = clean_entry_bitmap ? offset/dsk.csum_block_size : 0;
-    uint32_t bmp_pos = offset/dsk.bitmap_granularity;
-    for (int i = 0; i < n_iov; i++)
-    {
-        uint32_t pos = 0;
-        while (pos < iov[i].iov_len)
-        {
-            uint32_t start = pos;
-            uint8_t bit = (clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1;
-            while (pos < iov[i].iov_len && ((clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1) == bit)
-            {
-                pos += dsk.bitmap_granularity;
-                bmp_pos++;
-            }
-            uint32_t len = pos-start;
-            auto buf = (uint8_t*)iov[i].iov_base+start;
-            while (block_done+len >= dsk.csum_block_size)
-            {
-                auto cur_len = dsk.csum_block_size-block_done;
-                block_csum = crc32c_pad(block_csum, buf, bit ? cur_len : 0, bit ? 0 : cur_len, 0);
-                if (block_csum != csums[block_num])
-                {
-                    if (bad_block_cb)
-                        bad_block_cb(block_num*dsk.csum_block_size, block_csum, csums[block_num]);
-                    else
-                        return false;
-                }
-                block_num++;
-                buf += cur_len;
-                len -= cur_len;
-                block_done = block_csum = 0;
-            }
-            if (len > 0)
-            {
-                block_csum = crc32c_pad(block_csum, buf, bit ? len : 0, bit ? 0 : len, 0);
-                block_done += len;
-            }
-        }
-    }
-    assert(!block_done);
-    return true;
-}
-
-bool blockstore_impl_t::verify_journal_checksums(uint8_t *csums, uint32_t offset,
-    iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb)
-{
-    uint32_t block_csum = 0;
-    uint32_t block_num = 0;
-    uint32_t block_done = offset%dsk.csum_block_size;
-    for (int i = 0; i < n_iov; i++)
-    {
-        uint32_t len = iov[i].iov_len;
-        auto buf = (uint8_t*)iov[i].iov_base;
-        while (block_done+len >= dsk.csum_block_size)
-        {
-            auto cur_len = dsk.csum_block_size-block_done;
-            block_csum = crc32c(block_csum, buf, cur_len);
-            if (block_csum != ((uint32_t*)csums)[block_num])
-            {
-                if (bad_block_cb)
-                    bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]);
-                else
-                    return false;
-            }
-            block_num++;
-            buf += cur_len;
-            len -= cur_len;
-            block_done = block_csum = 0;
-        }
-        if (len > 0)
-        {
-            block_csum = crc32c(block_csum, buf, len);
-            block_done += len;
-        }
-    }
-    if (block_done > 0 && block_csum != ((uint32_t*)csums)[block_num])
-    {
-        if (bad_block_cb)
-            bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]);
-        else
-            return false;
-    }
-    return true;
-}
-
-bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal,
-    iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb)
-{
-    uint32_t offset = clean_loc % dsk.data_block_size;
-    if (from_journal)
-        return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
-    clean_loc = (clean_loc >> dsk.block_order) << dsk.block_order;
-    if (!dyn_data)
-    {
-        assert(inmemory_meta);
-        dyn_data = get_clean_entry_bitmap(clean_loc, 0);
-    }
-    return verify_padded_checksums(dyn_data, dyn_data + 2*dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
 }
 
 void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op)
@@ -826,139 +288,6 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
     }
     if (PRIV(op)->pending_ops == 0)
     {
-        if (dsk.csum_block_size)
-        {
-            // verify checksums if required
-            auto & rv = PRIV(op)->read_vec;
-            void *meta_block = NULL;
-            if (dsk.csum_block_size > dsk.bitmap_granularity)
-            {
-                for (int i = rv.size()-1; i >= 0 && (rv[i].copy_flags & COPY_BUF_CSUM_FILL); i--)
-                {
-                    if (rv[i].copy_flags & COPY_BUF_META_BLOCK)
-                    {
-                        // Metadata read. Skip
-                        assert(!meta_block);
-                        meta_block = rv[i].buf;
-                        rv[i].buf = NULL;
-                        continue;
-                    }
-                    struct iovec *iov = (struct iovec*)((uint8_t*)rv[i].buf + (rv[i].len & 0xFFFFFFFF));
-                    int n_iov = rv[i].len >> 32;
-                    bool ok = true;
-                    if (rv[i].copy_flags & COPY_BUF_JOURNAL)
-                    {
-                        // SMALL_WRITE from journal
-                        verify_journal_checksums(
-                            rv[i].csum_buf, rv[i].offset, iov, n_iov,
-                            [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
-                            {
-                                ok = false;
-                                printf(
-                                    "Checksum mismatch in object %lx:%lx v%lu in journal at 0x%lx, checksum block #%u: got %08x, expected %08x\n",
-                                    op->oid.inode, op->oid.stripe, op->version,
-                                    rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum
-                                );
-                            }
-                        );
-                    }
-                    else
-                    {
-                        // BIG_WRITE from journal or clean data
-                        // Do not verify checksums if the data location is/was mutated by flushers
-                        auto & uo = used_clean_objects.at((rv[i].disk_offset >> dsk.block_order) << dsk.block_order);
-                        if (!uo.was_changed)
-                        {
-                            verify_clean_padded_checksums(
-                                op, rv[i].disk_offset, rv[i].csum_buf, (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG), iov, n_iov,
-                                [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
-                                {
-                                    ok = false;
-                                    printf(
-                                        "Checksum mismatch in object %lx:%lx v%lu in %s data at 0x%lx, checksum block #%u: got %08x, expected %08x\n",
-                                        op->oid.inode, op->oid.stripe, op->version,
-                                        (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG ? "redirect-write" : "clean"),
-                                        rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum
-                                    );
-                                }
-                            );
-                        }
-                    }
-                    if (!ok)
-                    {
-                        op->retval = -EDOM;
-                    }
-                    free(rv[i].buf);
-                    rv[i].buf = NULL;
-                    if (rv[i].dyn_data && --(*rv[i].dyn_data) == 0) // refcount
-                    {
-                        free(rv[i].dyn_data);
-                        rv[i].dyn_data = NULL;
-                    }
-                }
-            }
-            else
-            {
-                for (auto & vec: rv)
-                {
-                    if (vec.copy_flags & COPY_BUF_META_BLOCK)
-                    {
-                        // Metadata read. Skip
-                        assert(!meta_block);
-                        meta_block = vec.buf;
-                        vec.buf = NULL;
-                        continue;
-                    }
-                    if (vec.csum_buf)
-                    {
-                        uint32_t *csum = (uint32_t*)vec.csum_buf;
-                        for (size_t p = 0; p < vec.len; p += dsk.csum_block_size, csum++)
-                        {
-                            if (crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size) != *csum)
-                            {
-                                // checksum error
-                                printf(
-                                    "Checksum mismatch in object %lx:%lx v%lu in %s area at offset 0x%lx+0x%lx: %08x vs %08x\n",
-                                    op->oid.inode, op->oid.stripe, op->version,
-                                    (vec.copy_flags & COPY_BUF_JOURNAL) ? "journal" : "data", vec.disk_offset, p,
-                                    crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size), *csum
-                                );
-                                op->retval = -EDOM;
-                                break;
-                            }
-                        }
-                    }
-                    if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount
-                    {
-                        free(vec.dyn_data);
-                        vec.dyn_data = NULL;
-                    }
-                }
-            }
-            if (meta_block)
-            {
-                // Free after checking
-                free(meta_block);
-                meta_block = NULL;
-            }
-        }
-        if (PRIV(op)->clean_block_used)
-        {
-            // Release clean data block
-            auto uo_it = used_clean_objects.find(PRIV(op)->clean_block_used);
-            if (uo_it != used_clean_objects.end())
-            {
-                uo_it->second.refs--;
-                if (uo_it->second.refs <= 0)
-                {
-                    if (uo_it->second.was_freed)
-                    {
-                        data_alloc->set(PRIV(op)->clean_block_used, false);
-                    }
-                    used_clean_objects.erase(uo_it);
-                }
-            }
-        }
         if (!journal.inmemory)
         {
             // Release journal sector usage
@@ -999,9 +328,8 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
                     *result_version = dirty_it->first.version;
                 if (bitmap)
                 {
-                    void *dyn_ptr = (alloc_dyn_data
-                        ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data);
-                    memcpy(bitmap, dyn_ptr, dsk.clean_entry_bitmap_size);
+                    void *bmp_ptr = (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap);
+                    memcpy(bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
                 }
                 return 0;
             }

src/blockstore_rollback.cpp

@@ -218,7 +218,7 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
         auto used = --journal.used_sectors[dirty_it->second.journal_sector];
 #ifdef BLOCKSTORE_DEBUG
         printf(
-            "remove usage of journal offset %08lx by %lx:%lx v%lu (%lu refs)\n", dirty_it->second.journal_sector,
+            "remove usage of journal offset %08lx by %lx:%lx v%lu (%d refs)\n", dirty_it->second.journal_sector,
             dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, used
         );
 #endif
@@ -227,7 +227,11 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
             journal.used_sectors.erase(dirty_it->second.journal_sector);
             flusher->mark_trim_possible();
         }
-        free_dirty_dyn_data(dirty_it->second);
+        if (dsk.clean_entry_bitmap_size > sizeof(void*))
+        {
+            free(dirty_it->second.bitmap);
+            dirty_it->second.bitmap = NULL;
+        }
         if (dirty_it == dirty_start)
         {
             break;
@@ -236,18 +240,3 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
     }
     dirty_db.erase(dirty_start, dirty_end);
 }
-
-void blockstore_impl_t::free_dirty_dyn_data(dirty_entry & e)
-{
-    if (e.dyn_data)
-    {
-        if (alloc_dyn_data &&
-            --*((int*)e.dyn_data) == 0) // refcount
-        {
-            // dyn_data contains the bitmap and checksums
-            // free it if it doesn't refer to the in-memory journal
-            free(e.dyn_data);
-        }
-        e.dyn_data = NULL;
-    }
-}

src/blockstore_stable.cpp

@@ -458,16 +458,6 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
                     big_to_flush++;
                 }
             }
-            else if (IS_IN_FLIGHT(dirty_it->second.state))
-            {
-                // mark_stable should never be called for in-flight or submitted writes
-                printf(
-                    "BUG: Attempt to mark_stable object %lx:%lx v%lu state of which is %x\n",
-                    dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
-                    dirty_it->second.state
-                );
-                exit(1);
-            }
             if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) ||
                 IS_DELETE(dirty_it->second.state)))
             {

src/blockstore_sync.cpp

@@ -27,6 +27,8 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
         unsynced_big_write_count -= unsynced_big_writes.size();
         PRIV(op)->sync_big_writes.swap(unsynced_big_writes);
         PRIV(op)->sync_small_writes.swap(unsynced_small_writes);
+        PRIV(op)->sync_small_checked = 0;
+        PRIV(op)->sync_big_checked = 0;
         unsynced_big_writes.clear();
         unsynced_small_writes.clear();
         if (PRIV(op)->sync_big_writes.size() > 0)
@@ -76,23 +78,7 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
         // 2nd step: Data device is synced, prepare & write journal entries
         // Check space in the journal and journal memory buffers
         blockstore_journal_check_t space_check(this);
-        if (dsk.csum_block_size)
-        {
-            // More complex check because all journal entries have different lengths
-            int left = PRIV(op)->sync_big_writes.size();
-            for (auto & sbw: PRIV(op)->sync_big_writes)
-            {
-                left--;
-                auto & dirty_entry = dirty_db.at(sbw);
-                uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
-                if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size,
-                    left == 0 ? JOURNAL_STABILIZE_RESERVATION : 0))
-                {
-                    return 0;
-                }
-            }
-        }
-        else if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(),
+        if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(),
             sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, JOURNAL_STABILIZE_RESERVATION))
         {
             return 0;
@@ -104,17 +90,16 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
         int s = 0;
         while (it != PRIV(op)->sync_big_writes.end())
         {
-            auto & dirty_entry = dirty_db.at(*it);
-            uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
-            if (!journal.entry_fits(sizeof(journal_entry_big_write) + dyn_size) &&
+            if (!journal.entry_fits(sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size) &&
                 journal.sector_info[journal.cur_sector].dirty)
             {
                 prepare_journal_sector_write(journal.cur_sector, op);
                 s++;
             }
+            auto & dirty_entry = dirty_db.at(*it);
             journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
                 journal, (dirty_entry.state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
-                sizeof(journal_entry_big_write) + dyn_size
+                sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
             );
             dirty_entry.journal_sector = journal.sector_info[journal.cur_sector].offset;
             journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
@@ -130,8 +115,8 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
             je->offset = dirty_entry.offset;
             je->len = dirty_entry.len;
             je->location = dirty_entry.location;
-            memcpy((void*)(je+1), (alloc_dyn_data
-                ? (uint8_t*)dirty_entry.dyn_data+sizeof(int) : (uint8_t*)&dirty_entry.dyn_data), dyn_size);
+            memcpy((void*)(je+1), (dsk.clean_entry_bitmap_size > sizeof(void*)
+                ? dirty_entry.bitmap : &dirty_entry.bitmap), dsk.clean_entry_bitmap_size);
             je->crc32 = je_crc32((journal_entry*)je);
             journal.crc32_last = je->crc32;
             it++;

src/blockstore_write.cpp

@@ -8,21 +8,12 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
     // Check or assign version number
     bool found = false, deleted = false, unsynced = false, is_del = (op->opcode == BS_OP_DELETE);
     bool wait_big = false, wait_del = false;
-    void *dyn = NULL;
-    if (is_del)
-    {
-        op->len = 0;
-    }
-    size_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
-    if (!is_del && alloc_dyn_data)
-    {
-        // FIXME: Working with `dyn_data` has to be refactored somehow but I first have to decide how :)
-        // +sizeof(int) = refcount
-        dyn = calloc_or_die(1, dyn_size+sizeof(int));
-        *((int*)dyn) = 1;
-    }
-    uint8_t *dyn_ptr = (uint8_t*)(alloc_dyn_data ? dyn+sizeof(int) : &dyn);
+    void *bmp = NULL;
     uint64_t version = 1;
+    if (!is_del && dsk.clean_entry_bitmap_size > sizeof(void*))
+    {
+        bmp = calloc_or_die(1, dsk.clean_entry_bitmap_size);
+    }
     if (dirty_db.size() > 0)
     {
         auto dirty_it = dirty_db.upper_bound((obj_ver_id){
@@ -42,9 +33,10 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
                 : ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG);
             if (!is_del && !deleted)
             {
-                void *dyn_from = alloc_dyn_data
-                    ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data;
-                memcpy(dyn_ptr, dyn_from, dsk.clean_entry_bitmap_size);
+                if (dsk.clean_entry_bitmap_size > sizeof(void*))
+                    memcpy(bmp, dirty_it->second.bitmap, dsk.clean_entry_bitmap_size);
+                else
+                    bmp = dirty_it->second.bitmap;
             }
         }
     }
@@ -58,7 +50,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             if (!is_del)
             {
                 void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size);
-                memcpy(dyn_ptr, bmp_ptr, dsk.clean_entry_bitmap_size);
+                memcpy((dsk.clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp), bmp_ptr, dsk.clean_entry_bitmap_size);
             }
         }
         else
@@ -120,9 +112,9 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             printf("Write %lx:%lx v%lu requested, but we already have v%lu\n", op->oid.inode, op->oid.stripe, op->version, version);
 #endif
             op->retval = -EEXIST;
-            if (!is_del && alloc_dyn_data)
+            if (!is_del && dsk.clean_entry_bitmap_size > sizeof(void*))
             {
-                free(dyn);
+                free(bmp);
             }
             return false;
         }
@@ -166,50 +158,26 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         if (op->bitmap)
         {
             // Only allow to overwrite part of the object bitmap respective to the write's offset/len
+            uint8_t *bmp_ptr = (uint8_t*)(dsk.clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp);
             uint32_t bit = op->offset/dsk.bitmap_granularity;
             uint32_t bits_left = op->len/dsk.bitmap_granularity;
             while (!(bit % 8) && bits_left >= 8)
             {
                 // Copy bytes
-                dyn_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];
+                bmp_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];
                 bit += 8;
                 bits_left -= 8;
             }
             while (bits_left > 0)
             {
                 // Copy bits
-                dyn_ptr[bit/8] = (dyn_ptr[bit/8] & ~(1 << (bit%8)))
+                bmp_ptr[bit/8] = (bmp_ptr[bit/8] & ~(1 << (bit%8)))
                     | (((uint8_t*)op->bitmap)[bit/8] & (1 << bit%8));
                 bit++;
                 bits_left--;
             }
         }
     }
-    // Calculate checksums
-    // FIXME: Allow to receive checksums from outside?
-    if (!is_del && dsk.data_csum_type && op->len > 0)
-    {
-        uint32_t *data_csums = (uint32_t*)(dyn_ptr + dsk.clean_entry_bitmap_size);
-        uint32_t start = op->offset / dsk.csum_block_size;
-        uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
-        auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
-        if (start == end)
-            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
-        else
-        {
-            // First block
-            data_csums[0] = fn(0, op->buf, dsk.csum_block_size*(start+1)-op->offset, op->offset - start*dsk.csum_block_size, 0);
-            // Intermediate blocks
-            for (uint32_t i = start+1; i < end; i++)
-                data_csums[i-start] = crc32c(0, (uint8_t*)op->buf + dsk.csum_block_size*i-op->offset, dsk.csum_block_size);
-            // Last block
-            data_csums[end-start] = fn(
-                0, (uint8_t*)op->buf + end*dsk.csum_block_size - op->offset,
-                op->offset+op->len - end*dsk.csum_block_size,
-                0, (end+1)*dsk.csum_block_size - (op->offset+op->len)
-            );
-        }
-    }
     dirty_db.emplace((obj_ver_id){
         .oid = op->oid,
         .version = op->version,
@@ -220,7 +188,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         .offset = is_del ? 0 : op->offset,
         .len = is_del ? 0 : op->len,
         .journal_sector = 0,
-        .dyn_data = dyn,
+        .bitmap = bmp,
     });
     return true;
 }
@@ -229,7 +197,8 @@ void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_
 {
     while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
     {
-        free_dirty_dyn_data(dirty_it->second);
+        if (dsk.clean_entry_bitmap_size > sizeof(void*))
+            free(dirty_it->second.bitmap);
         dirty_db.erase(dirty_it++);
     }
     bool found = false;
@@ -286,18 +255,13 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         printf("Restoring %lx:%lx version: v%lu -> v%lu\n", op->oid.inode, op->oid.stripe, op->version, PRIV(op)->real_version);
 #endif
         auto prev_it = dirty_it;
-        if (prev_it != dirty_db.begin())
+        prev_it--;
+        if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version)
         {
-            prev_it--;
-            if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version)
-            {
-                // Original version is still invalid
-                // All subsequent writes to the same object must be canceled too
-                printf("Tried to write %lx:%lx v%lu after delete (old version v%lu), but already have v%lu\n",
-                    op->oid.inode, op->oid.stripe, PRIV(op)->real_version, op->version, prev_it->first.version);
-                cancel_all_writes(op, dirty_it, -EEXIST);
-                return 2;
-            }
+            // Original version is still invalid
+            // All subsequent writes to the same object must be canceled too
+            cancel_all_writes(op, dirty_it, -EEXIST);
+            return 2;
         }
         op->version = PRIV(op)->real_version;
         PRIV(op)->real_version = 0;
@@ -316,7 +280,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         blockstore_journal_check_t space_check(this);
         if (!space_check.check_available(op, unsynced_big_write_count + 1,
-            sizeof(journal_entry_big_write) + dsk.clean_dyn_size,
+            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size,
             (dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION))
         {
             return 0;
@@ -383,6 +347,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc << dsk.block_order) + op->offset - stripe_offset
         );
         PRIV(op)->pending_ops = 1;
+        PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
         if (immediate_commit != IMMEDIATE_ALL)
         {
             // Increase the counter, but don't save into unsynced_writes yet (can't sync until the write is finished)
@@ -398,13 +363,12 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         // Small (journaled) write
         // First check if the journal has sufficient space
-        uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
         blockstore_journal_check_t space_check(this);
         if (unsynced_big_write_count &&
             !space_check.check_available(op, unsynced_big_write_count,
-                sizeof(journal_entry_big_write) + dsk.clean_dyn_size, 0)
+                sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0)
             || !space_check.check_available(op, 1,
-                sizeof(journal_entry_small_write) + dyn_size,
+                sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size,
                 op->len + ((dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION)))
         {
             return 0;
@@ -413,21 +377,27 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         BS_SUBMIT_CHECK_SQES(
             // Write current journal sector only if it's dirty and full, or in the immediate_commit mode
             (immediate_commit != IMMEDIATE_NONE ||
-                !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size) ? 1 : 0) +
+                !journal.entry_fits(sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size) ? 1 : 0) +
             (op->len > 0 ? 1 : 0)
         );
         write_iodepth++;
         // Got SQEs. Prepare previous journal sector write if required
         auto cb = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        if (immediate_commit == IMMEDIATE_NONE &&
-            !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size))
+        if (immediate_commit == IMMEDIATE_NONE)
         {
-            prepare_journal_sector_write(journal.cur_sector, op);
+            if (!journal.entry_fits(sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size))
+            {
+                prepare_journal_sector_write(journal.cur_sector, op);
+            }
+            else
+            {
+                PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
+            }
         }
         // Then pre-fill journal entry
         journal_entry_small_write *je = (journal_entry_small_write*)prefill_single_journal_entry(
             journal, op->opcode == BS_OP_WRITE_STABLE ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE,
-            sizeof(journal_entry_small_write) + dyn_size
+            sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size
         );
         dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
         journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
@@ -439,31 +409,14 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         );
 #endif
         // Figure out where data will be
-        auto next_next_free = (journal.next_free + op->len) <= journal.len ? journal.next_free : dsk.journal_block_size;
-        if (op->len > 0)
-        {
-            auto journal_used_it = journal.used_sectors.lower_bound(next_next_free);
-            if (journal_used_it != journal.used_sectors.end() &&
-                journal_used_it->first < next_next_free + op->len)
-            {
-                printf(
-                    "BUG: Attempt to overwrite used offset (%lx, %lu refs) of the journal with the object %lx:%lx v%lu: data at %lx, len %x!"
-                    " Journal used_start=%08lx (%lu refs), next_free=%08lx, dirty_start=%08lx\n",
-                    journal_used_it->first, journal_used_it->second, op->oid.inode, op->oid.stripe, op->version, next_next_free, op->len,
-                    journal.used_start, journal.used_sectors[journal.used_start], journal.next_free, journal.dirty_start
-                );
-                exit(1);
-            }
-        }
-        journal.next_free = next_next_free;
+        journal.next_free = (journal.next_free + op->len) <= journal.len ? journal.next_free : dsk.journal_block_size;
         je->oid = op->oid;
         je->version = op->version;
         je->offset = op->offset;
         je->len = op->len;
         je->data_offset = journal.next_free;
-        je->crc32_data = dsk.csum_block_size ? 0 : crc32c(0, op->buf, op->len);
-        memcpy((void*)(je+1), (alloc_dyn_data
-            ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size);
+        je->crc32_data = crc32c(0, op->buf, op->len);
+        memcpy((void*)(je+1), (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap), dsk.clean_entry_bitmap_size);
         je->crc32 = je_crc32((journal_entry*)je);
         journal.crc32_last = je->crc32;
         if (immediate_commit != IMMEDIATE_NONE)
@@ -532,9 +485,9 @@ resume_2:
             .version = op->version,
         });
         assert(dirty_it != dirty_db.end());
-        uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
         blockstore_journal_check_t space_check(this);
-        if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size,
+        if (!space_check.check_available(op, 1,
+            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size,
             ((dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION)))
         {
             return 0;
@@ -542,7 +495,7 @@ resume_2:
         BS_SUBMIT_CHECK_SQES(1);
         journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
             journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
-            sizeof(journal_entry_big_write) + dyn_size
+            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
         );
         dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
         journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
@@ -558,8 +511,7 @@ resume_2:
         je->offset = op->offset;
         je->len = op->len;
         je->location = dirty_it->second.location;
-        memcpy((void*)(je+1), (alloc_dyn_data
-            ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size);
+        memcpy((void*)(je+1), (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap), dsk.clean_entry_bitmap_size);
         je->crc32 = je_crc32((journal_entry*)je);
         journal.crc32_last = je->crc32;
         prepare_journal_sector_write(journal.cur_sector, op);
@@ -693,13 +645,8 @@ void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
         uint64_t s = PRIV(op)->min_flushed_journal_sector;
         while (1)
         {
-            if (!journal.sector_info[s-1].dirty && journal.sector_info[s-1].flush_count == 0)
+            if (s != (1+journal.cur_sector) && journal.sector_info[s-1].flush_count == 0)
             {
-                if (s == (1+journal.cur_sector))
-                {
-                    // Forcibly move to the next sector and move dirty position
-                    journal.in_sector_pos = journal.block_size;
-                }
                 // We know for sure that we won't write into this sector anymore
                 uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size;
                 if (new_ds >= journal.len)
@@ -748,11 +695,17 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
     }
     write_iodepth++;
     // Prepare journal sector write
-    if (immediate_commit == IMMEDIATE_NONE &&
-        (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
-        journal.sector_info[journal.cur_sector].dirty)
+    if (immediate_commit == IMMEDIATE_NONE)
     {
-        prepare_journal_sector_write(journal.cur_sector, op);
+        if ((dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
+            journal.sector_info[journal.cur_sector].dirty)
+        {
+            prepare_journal_sector_write(journal.cur_sector, op);
+        }
+        else
+        {
+            PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
+        }
     }
     // Pre-fill journal entry
     journal_entry_del *je = (journal_entry_del*)prefill_single_journal_entry(

src/cli.cpp

@@ -357,8 +357,6 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
         p->ringloop = NULL;
     }
     // Print result
-    fflush(stderr);
-    fflush(stdout);
     if (p->json_output && !result.data.is_null())
     {
         printf("%s\n", result.data.dump().c_str());

src/cli_alloc_osd.cpp

@@ -77,8 +77,8 @@ struct alloc_osd_t
                     std::string key = base64_decode(kv["key"].string_value());
                     osd_num_t cur_osd;
                     char null_byte = 0;
-                    int scanned = sscanf(key.c_str() + parent->cli->st_cli.etcd_prefix.length(), "/osd/stats/%lu%c", &cur_osd, &null_byte);
-                    if (scanned != 1 || !cur_osd)
+                    sscanf(key.c_str() + parent->cli->st_cli.etcd_prefix.length(), "/osd/stats/%lu%c", &cur_osd, &null_byte);
+                    if (!cur_osd || null_byte != 0)
                     {
                         fprintf(stderr, "Invalid key in etcd: %s\n", key.c_str());
                         continue;

src/cli_df.cpp

@@ -67,8 +67,8 @@ resume_1:
             // pool ID
             pool_id_t pool_id;
             char null_byte = 0;
-            int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
-            if (scanned != 1 || !pool_id || pool_id >= POOL_ID_MAX)
+            sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
+            if (!pool_id || pool_id >= POOL_ID_MAX || null_byte != 0)
             {
                 fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
                 continue;
@@ -82,8 +82,8 @@ resume_1:
             // osd ID
             osd_num_t osd_num;
             char null_byte = 0;
-            int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/osd/stats/%lu%c", &osd_num, &null_byte);
-            if (scanned != 1 || !osd_num || osd_num >= POOL_ID_MAX)
+            sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/osd/stats/%lu%c", &osd_num, &null_byte);
+            if (!osd_num || osd_num >= POOL_ID_MAX || null_byte != 0)
             {
                 fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
                 continue;

src/cli_ls.cpp

@@ -56,15 +56,14 @@ struct image_lister_t
             {
                 continue;
             }
-            auto pool_it = parent->cli->st_cli.pool_config.find(INODE_POOL(ic.second.num));
-            bool good_pool = pool_it != parent->cli->st_cli.pool_config.end();
+            auto & pool_cfg = parent->cli->st_cli.pool_config.at(INODE_POOL(ic.second.num));
             auto item = json11::Json::object {
                 { "name", ic.second.name },
                 { "size", ic.second.size },
                 { "used_size", 0 },
                 { "readonly", ic.second.readonly },
                 { "pool_id", (uint64_t)INODE_POOL(ic.second.num) },
-                { "pool_name", good_pool ? pool_it->second.name : "? (ID:"+std::to_string(INODE_POOL(ic.second.num))+")" },
+                { "pool_name", pool_cfg.name },
                 { "inode_num", INODE_NO_POOL(ic.second.num) },
                 { "inode_id", ic.second.num },
             };
@@ -133,8 +132,8 @@ resume_1:
             // pool ID
             pool_id_t pool_id;
             char null_byte = 0;
-            int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
-            if (scanned != 1 || !pool_id || pool_id >= POOL_ID_MAX)
+            sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
+            if (!pool_id || pool_id >= POOL_ID_MAX || null_byte != 0)
             {
                 fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
                 continue;
@@ -149,9 +148,9 @@ resume_1:
             pool_id_t pool_id;
             inode_t only_inode_num;
             char null_byte = 0;
-            int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(),
+            sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(),
                 "/inode/stats/%u/%lu%c", &pool_id, &only_inode_num, &null_byte);
-            if (scanned != 2 || !pool_id || pool_id >= POOL_ID_MAX || INODE_POOL(only_inode_num) != 0)
+            if (!pool_id || pool_id >= POOL_ID_MAX || INODE_POOL(only_inode_num) != 0 || null_byte != 0)
             {
                 fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
                 continue;
@@ -174,7 +173,7 @@ resume_1:
                     { "size", 0 },
                     { "readonly", false },
                     { "pool_id", (uint64_t)INODE_POOL(inode_num) },
-                    { "pool_name", pool_it != parent->cli->st_cli.pool_config.end()
+                    { "pool_name", pool_it == parent->cli->st_cli.pool_config.end()
                         ? (pool_it->second.name == "" ? "<Unnamed>" : pool_it->second.name) : "?" },
                     { "inode_num", INODE_NO_POOL(inode_num) },
                     { "inode_id", inode_num },
@@ -248,8 +247,6 @@ resume_1:
         if (state == 1)
             goto resume_1;
         get_list();
-        if (state == 100)
-            return;
         if (show_stats)
         {
 resume_1:
@@ -272,7 +269,7 @@ resume_1:
             { "key", "name" },
             { "title", "NAME" },
         });
-        if (list_pool_name == "")
+        if (!list_pool_id)
         {
             cols.push_back(json11::Json::object{
                 { "key", "pool_name" },
@@ -379,18 +376,16 @@ resume_1:
 
 std::string print_table(json11::Json items, json11::Json header, bool use_esc)
 {
-    int header_sizes[header.array_items().size()];
     std::vector<int> sizes;
     for (int i = 0; i < header.array_items().size(); i++)
     {
-        header_sizes[i] = utf8_length(header[i]["title"].string_value());
-        sizes.push_back(header_sizes[i]);
+        sizes.push_back(header[i]["title"].string_value().length());
     }
     for (auto & item: items.array_items())
     {
         for (int i = 0; i < header.array_items().size(); i++)
         {
-            int l = utf8_length(item[header[i]["key"].string_value()].as_string());
+            int l = item[header[i]["key"].string_value()].as_string().length();
             sizes[i] = sizes[i] < l ? l : sizes[i];
         }
     }
@@ -402,7 +397,7 @@ std::string print_table(json11::Json items, json11::Json header, bool use_esc)
             // Separator
             str += "  ";
         }
-        int pad = sizes[i]-header_sizes[i];
+        int pad = sizes[i]-header[i]["title"].string_value().length();
         if (header[i]["right"].bool_value())
         {
             // Align right
@@ -430,7 +425,7 @@ std::string print_table(json11::Json items, json11::Json header, bool use_esc)
                 // Separator
                 str += "  ";
             }
-            int pad = sizes[i] - utf8_length(item[header[i]["key"].string_value()].as_string());
+            int pad = sizes[i] - item[header[i]["key"].string_value()].as_string().length();
             if (header[i]["right"].bool_value())
             {
                 // Align right

src/cli_merge.cpp

@@ -41,7 +41,7 @@ struct snap_merger_t
     int fsync_interval = 128;
 
     // -- STATE --
-    inode_t target, to_num;
+    inode_t target;
     int target_rank;
     bool inside_continue = false;
     int state = 0;
@@ -98,7 +98,6 @@ struct snap_merger_t
             state = 100;
             return;
         }
-        to_num = to_cfg->num;
         // Check that to_cfg is actually a child of from_cfg and target_cfg is somewhere between them
         std::vector<inode_t> chain_list;
         inode_config_t *cur = to_cfg;
@@ -452,7 +451,7 @@ struct snap_merger_t
     {
         cluster_op_t *op = &rwo->op;
         op->opcode = OSD_OP_READ;
-        op->inode = to_num;
+        op->inode = target;
         op->offset = rwo->offset;
         op->len = target_block_size;
         op->iov.push_back(rwo->buf, target_block_size);
@@ -484,7 +483,7 @@ struct snap_merger_t
                 {
                     // write start->end
                     rwo->todo++;
-                    write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas && to_num == target ? 1+rwo->op.version : 0);
+                    write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas ? 1+rwo->op.version : 0);
                     rwo->start = rwo->end;
                     if (use_cas)
                     {
@@ -503,7 +502,7 @@ struct snap_merger_t
         {
             // write start->end
             rwo->todo++;
-            write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas && to_num == target ? 1+rwo->op.version : 0);
+            write_subop(rwo, rwo->start*gran, rwo->end*gran, use_cas ? 1+rwo->op.version : 0);
             rwo->start = rwo->end;
             if (use_cas)
             {
@@ -533,7 +532,7 @@ struct snap_merger_t
                 if (use_cas && subop->retval == -EINTR)
                 {
                     // CAS failure - reread and repeat optimistically
-                    rwo->start = rwo->end = 0;
+                    rwo->start = subop->offset - rwo->offset;
                     rwo_read(rwo);
                     delete subop;
                     return;
@@ -543,7 +542,7 @@ struct snap_merger_t
                 rwo->error_read = false;
             }
             // Increment CAS version
-            rwo->op.version = subop->version;
+            rwo->op.version++;
             if (use_cas)
                 next_write(rwo);
             else

src/cli_modify.cpp

@@ -13,7 +13,7 @@ struct image_changer_t
     std::string image_name;
     std::string new_name;
     uint64_t new_size = 0;
-    bool force_size = false, inc_size = false;
+    bool force_size = false;
     bool set_readonly = false, set_readwrite = false, force = false;
     // interval between fsyncs
     int fsync_interval = 128;
@@ -81,14 +81,14 @@ struct image_changer_t
         }
         if ((!set_readwrite || !cfg.readonly) &&
             (!set_readonly || cfg.readonly) &&
-            (!new_size && !force_size || cfg.size == new_size || cfg.size >= new_size && inc_size) &&
+            (!new_size && !force_size || cfg.size == new_size) &&
             (new_name == "" || new_name == image_name))
         {
             result = (cli_result_t){ .text = "No change" };
             state = 100;
             return;
         }
-        if ((new_size != 0 || force_size) && (cfg.size < new_size || !inc_size))
+        if (new_size != 0 || force_size)
         {
             if (cfg.size >= new_size)
             {
@@ -233,7 +233,6 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_modify(json11::Json cfg)
     changer->new_name = cfg["rename"].string_value();
     changer->new_size = parse_size(cfg["resize"].as_string());
     changer->force_size = cfg["force_size"].bool_value();
-    changer->inc_size = cfg["inc_size"].bool_value();
     changer->force = cfg["force"].bool_value();
     changer->set_readonly = cfg["readonly"].bool_value();
     changer->set_readwrite = cfg["readwrite"].bool_value();

src/cli_rm.cpp

@@ -65,9 +65,6 @@ struct snap_remover_t
     int current_child = 0;
     std::function<bool(cli_result_t &)> cb;
 
-    std::vector<std::string> rebased_images, deleted_images;
-    std::vector<uint64_t> deleted_ids;
-    std::string inverse_child_name, inverse_parent_name;
     cli_result_t result;
 
     bool is_done()
@@ -125,7 +122,6 @@ resume_1:
         {
             if (merge_children[current_child] == inverse_child)
                 continue;
-            rebased_images.push_back(parent->cli->st_cli.inode_config.at(merge_children[current_child]).name);
             start_merge_child(merge_children[current_child], merge_children[current_child]);
             if (state == 100)
                 return;
@@ -138,12 +134,9 @@ resume_2:
             cb = NULL;
             if (result.err)
             {
-                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             parent->change_parent(merge_children[current_child], new_parent, &result);
             state = 3;
 resume_3:
@@ -151,7 +144,6 @@ resume_3:
                 return;
             if (result.err)
             {
-                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
@@ -173,12 +165,9 @@ resume_4:
             cb = NULL;
             if (result.err)
             {
-                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             // Delete "inverse" child data
             start_delete_source(inverse_child);
             if (state == 100)
@@ -192,12 +181,9 @@ resume_5:
             cb = NULL;
             if (result.err)
             {
-                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             // Delete "inverse" child metadata, rename parent over it,
             // and also change parent links of the previous "inverse" child
             rename_inverse_parent();
@@ -213,12 +199,6 @@ resume_6:
         {
             if (chain_list[current_child] == inverse_parent)
                 continue;
-            {
-                auto parent_it = parent->cli->st_cli.inode_config.find(chain_list[current_child]);
-                if (parent_it != parent->cli->st_cli.inode_config.end())
-                    deleted_images.push_back(parent_it->second.name);
-                deleted_ids.push_back(chain_list[current_child]);
-            }
             start_delete_source(chain_list[current_child]);
 resume_7:
             while (!cb(result))
@@ -229,12 +209,9 @@ resume_7:
             cb = NULL;
             if (result.err)
             {
-                result.data = my_result(result.data);
                 state = 100;
                 return;
             }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             delete_inode_config(chain_list[current_child]);
             if (state == 100)
                 return;
@@ -244,26 +221,11 @@ resume_8:
                 return;
         }
         state = 100;
-        result = (cli_result_t){
-            .text = "",
-            .data = my_result(result.data),
-        };
 resume_100:
         // Done
         return;
     }
 
-    json11::Json my_result(json11::Json src)
-    {
-        auto obj = src.object_items();
-        obj["deleted_ids"] = deleted_ids;
-        obj["deleted_images"] = deleted_images;
-        obj["rebased_images"] = rebased_images;
-        obj["renamed_from"] = inverse_parent_name;
-        obj["renamed_to"] = inverse_child_name;
-        return obj;
-    }
-
     void get_merge_children()
     {
         // Get all children of from..to
@@ -376,16 +338,12 @@ resume_100:
             }
             for (auto inode_result: data["responses"].array_items())
             {
-                if (inode_result["response_range"]["kvs"].array_items().size() == 0)
-                {
-                    continue;
-                }
-                auto kv = parent->cli->st_cli.parse_etcd_kv(inode_result["response_range"]["kvs"][0]);
+                auto kv = parent->cli->st_cli.parse_etcd_kv(inode_result["kvs"][0]);
                 pool_id_t pool_id = 0;
                 inode_t inode = 0;
                 char null_byte = 0;
-                int scanned = sscanf(kv.key.c_str() + parent->cli->st_cli.etcd_prefix.length()+13, "%u/%lu%c", &pool_id, &inode, &null_byte);
-                if (scanned != 2 || !inode)
+                sscanf(kv.key.c_str() + parent->cli->st_cli.etcd_prefix.length()+13, "%u/%lu%c", &pool_id, &inode, &null_byte);
+                if (!inode || null_byte != 0)
                 {
                     result = (cli_result_t){ .err = EIO, .text = "Bad key returned from etcd: "+kv.key };
                     state = 100;
@@ -419,7 +377,7 @@ resume_100:
             inode_t child = cp.first;
             uint64_t child_used = inode_used[child];
             int rank = cp.second;
-            for (int i = chain_list.size()-1-rank; i < chain_list.size(); i++)
+            for (int i = chain_list.size()-rank; i < chain_list.size(); i++)
             {
                 inode_t parent = chain_list[i];
                 uint64_t parent_used = inode_used[parent];
@@ -455,8 +413,8 @@ resume_100:
         }
         inode_config_t *child_cfg = &child_it->second;
         inode_config_t *target_cfg = &target_it->second;
-        inverse_child_name = child_cfg->name;
-        inverse_parent_name = target_cfg->name;
+        std::string child_name = child_cfg->name;
+        std::string target_name = target_cfg->name;
         std::string child_cfg_key = base64_encode(
             parent->cli->st_cli.etcd_prefix+
             "/config/inode/"+std::to_string(INODE_POOL(inverse_child))+
@@ -467,9 +425,6 @@ resume_100:
             "/config/inode/"+std::to_string(INODE_POOL(inverse_parent))+
             "/"+std::to_string(INODE_NO_POOL(inverse_parent))
         );
-        std::string target_idx_key = base64_encode(
-            parent->cli->st_cli.etcd_prefix+"/index/image/"+inverse_parent_name
-        );
         // Fill new configuration
         inode_config_t new_cfg = *child_cfg;
         new_cfg.num = target_cfg->num;
@@ -494,11 +449,6 @@ resume_100:
                     { "key", child_cfg_key },
                 } },
             },
-            json11::Json::object {
-                { "request_delete_range", json11::Json::object {
-                    { "key", target_idx_key },
-                } },
-            },
             json11::Json::object {
                 { "request_put", json11::Json::object {
                     { "key", target_cfg_key },
@@ -545,12 +495,12 @@ resume_100:
         parent->cli->st_cli.etcd_txn_slow(json11::Json::object {
             { "compare", cmp },
             { "success", txn },
-        }, [this](std::string err, json11::Json res)
+        }, [this, target_name, child_name](std::string err, json11::Json res)
         {
             parent->waiting--;
             if (err != "")
             {
-                result = (cli_result_t){ .err = EIO, .text = "Error renaming "+inverse_parent_name+" to "+inverse_child_name+": "+err };
+                result = (cli_result_t){ .err = EIO, .text = "Error renaming "+target_name+" to "+child_name+": "+err };
                 state = 100;
                 return;
             }
@@ -558,14 +508,14 @@ resume_100:
             {
                 result = (cli_result_t){
                     .err = EAGAIN,
-                    .text = "Parent ("+inverse_parent_name+"), child ("+inverse_child_name+"), or one of its children"
+                    .text = "Parent ("+target_name+"), child ("+child_name+"), or one of its children"
                         " configuration was modified during rename",
                 };
                 state = 100;
                 return;
             }
             if (parent->progress)
-                printf("Layer %s renamed to %s\n", inverse_parent_name.c_str(), inverse_child_name.c_str());
+                printf("Layer %s renamed to %s\n", target_name.c_str(), child_name.c_str());
             parent->ringloop->wakeup();
         });
     }

src/cli_rm_data.cpp

@@ -28,7 +28,6 @@ struct rm_inode_t
     cli_tool_t *parent = NULL;
     inode_list_t *lister = NULL;
     std::vector<rm_pg_t*> lists;
-    std::vector<osd_num_t> inactive_osds;
     uint64_t total_count = 0, total_done = 0, total_prev_pct = 0;
     uint64_t pgs_to_list = 0;
     bool lists_done = false;
@@ -87,16 +86,6 @@ struct rm_inode_t
             state = 100;
             return;
         }
-        inactive_osds = parent->cli->list_inode_get_inactive_osds(lister);
-        if (inactive_osds.size() && !parent->json_output)
-        {
-            fprintf(stderr, "Some data may remain after delete on OSDs which are currently down: ");
-            for (int i = 0; i < inactive_osds.size(); i++)
-            {
-                fprintf(stderr, i > 0 ? ", %lu" : "%lu", inactive_osds[i]);
-            }
-            fprintf(stderr, "\n");
-        }
         pgs_to_list = parent->cli->list_pg_count(lister);
         parent->cli->list_inode_next(lister, parent->parallel_osds);
     }
@@ -178,33 +167,16 @@ struct rm_inode_t
         }
         if (parent->progress && total_count > 0 && total_done*1000/total_count != total_prev_pct)
         {
-            fprintf(stderr, "\rRemoved %lu/%lu objects, %lu more PGs to list...", total_done, total_count, pgs_to_list);
+            printf("\rRemoved %lu/%lu objects, %lu more PGs to list...", total_done, total_count, pgs_to_list);
             total_prev_pct = total_done*1000/total_count;
         }
         if (lists_done && !lists.size())
         {
-            if (parent->progress && total_count > 0)
-            {
-                fprintf(stderr, "\n");
-            }
-            if (parent->progress && (total_done < total_count || inactive_osds.size() > 0))
-            {
-                fprintf(
-                    stderr, "Warning: Pool:%u,ID:%lu inode data may not have been fully removed.\n"
-                    " Use `vitastor-cli rm-data --pool %u --inode %lu` if you encounter it in listings.\n",
-                    pool_id, INODE_NO_POOL(inode), pool_id, INODE_NO_POOL(inode)
-                );
-            }
             result = (cli_result_t){
                 .err = error_count > 0 ? EIO : 0,
                 .text = error_count > 0 ? "Some blocks were not removed" : (
                     "Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
                     std::to_string(pool_id)+" removed"),
-                .data = json11::Json::object {
-                    { "removed_objects", total_done },
-                    { "total_objects", total_count },
-                    { "inactive_osds", inactive_osds },
-                },
             };
             state = 100;
         }

src/cli_status.cpp

@@ -12,7 +12,7 @@ static const char *obj_states[] = { "clean", "misplaced", "degraded", "incomplet
 // Print cluster status:
 // etcd, mon, osd states
 // raw/used space, object states, pool states, pg states
-// client io, recovery io, rebalance io, scrub io
+// client io, recovery io, rebalance io
 struct status_printer_t
 {
     cli_tool_t *parent;
@@ -132,8 +132,8 @@ resume_2:
             auto kv = parent->cli->st_cli.parse_etcd_kv(osd_stats[i]);
             osd_num_t stat_osd_num = 0;
             char null_byte = 0;
-            int scanned = sscanf(kv.key.c_str() + parent->cli->st_cli.etcd_prefix.size(), "/osd/stats/%lu%c", &stat_osd_num, &null_byte);
-            if (scanned != 1 || !stat_osd_num)
+            sscanf(kv.key.c_str() + parent->cli->st_cli.etcd_prefix.size(), "/osd/stats/%lu%c", &stat_osd_num, &null_byte);
+            if (!stat_osd_num || null_byte != 0)
             {
                 fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
                 continue;
@@ -252,25 +252,18 @@ resume_2:
         }
         more_states.resize(more_states.size()-2);
         std::string recovery_io;
-        int io_indent = 0;
         {
             uint64_t deg_bps = agg_stats["recovery_stats"]["degraded"]["bps"].uint64_value();
             uint64_t deg_iops = agg_stats["recovery_stats"]["degraded"]["iops"].uint64_value();
             uint64_t misp_bps = agg_stats["recovery_stats"]["misplaced"]["bps"].uint64_value();
             uint64_t misp_iops = agg_stats["recovery_stats"]["misplaced"]["iops"].uint64_value();
-            uint64_t scrub_bps = agg_stats["op_stats"]["scrub"]["bps"].uint64_value();
-            uint64_t scrub_iops = agg_stats["op_stats"]["scrub"]["iops"].uint64_value();
-            if (misp_iops > 0 || misp_bps > 0 || no_rebalance)
-                io_indent = 3;
-            else if (deg_iops > 0 || deg_bps > 0 || no_recovery)
-                io_indent = 2;
             if (deg_iops > 0 || deg_bps > 0)
             {
-                recovery_io += "    recovery: "+str_repeat(" ", io_indent-2)+std::string(no_recovery ? "disabled, " : "")+
+                recovery_io += "    recovery:  "+std::string(no_recovery ? "disabled, " : "")+
                     format_size(deg_bps)+"/s, "+format_size(deg_iops, true)+" op/s\n";
             }
             else if (no_recovery)
-                recovery_io += "    recovery: disabled\n";
+                recovery_io += "    recovery:  disabled\n";
             if (misp_iops > 0 || misp_bps > 0)
             {
                 recovery_io += "    rebalance: "+std::string(no_rebalance ? "disabled, " : "")+
@@ -278,13 +271,6 @@ resume_2:
             }
             else if (no_rebalance)
                 recovery_io += "    rebalance: disabled\n";
-            if (scrub_iops > 0 || scrub_bps > 0)
-            {
-                recovery_io += "    scrub: "+str_repeat(" ", io_indent+1)+std::string(no_scrub ? "disabled, " : "")+
-                    format_size(scrub_bps)+"/s, "+format_size(scrub_iops, true)+" op/s\n";
-            }
-            else if (no_scrub)
-                recovery_io += "    scrub: "+str_repeat(" ", io_indent+1)+"disabled\n";
         }
         printf(
             "  cluster:\n"
@@ -312,7 +298,7 @@ resume_2:
             pools_active, pool_count,
             pgs_by_state_str.c_str(),
             readonly ? " (read-only mode)" : "",
-            str_repeat(" ", io_indent).c_str(),
+            recovery_io.size() > 0 ? "   " : "",
             format_size(agg_stats["op_stats"]["primary_read"]["bps"].uint64_value()).c_str(),
             format_size(agg_stats["op_stats"]["primary_read"]["iops"].uint64_value(), true).c_str(),
             format_size(agg_stats["op_stats"]["primary_write"]["bps"].uint64_value()).c_str(),