Save cached_read_* in superblock

Instead of it, just do not verify checksums of currently mutated objects.
When clean data modification during flush runs in parallel to a read request,
that request may read a mix of old and new data. It may even read a mix of
multiple flushed versions if it lasts too long... And attempts to verify it
using temporary copies of metadata make the algorithm too complex and creepy.

.gitea/workflows/test.yml

@@ -622,6 +622,114 @@ jobs:
           echo ""
         done
 
+  test_heal_csum_32k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_32k_dmj OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_32k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_32k_dj  OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_32k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_32k     OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_4k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_4k_dmj  OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_4k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_4k_dj   OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_4k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_4k      OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_scrub:
     runs-on: ubuntu-latest
     needs: build

.gitea/workflows/tests-to-yaml.pl

@@ -7,7 +7,8 @@ for my $line (<>)
     if ($line =~ /\.\/(test_[^\.]+)/s)
     {
         chomp $line;
-        my $test_name = $1;
+        my $base_name = $1;
+        my $test_name = $base_name;
         my $timeout = 3;
         if ($test_name eq 'test_etcd_fail' || $test_name eq 'test_heal' || $test_name eq 'test_add_osd' ||
             $test_name eq 'test_interrupted_rebalance' || $test_name eq 'test_rebalance_verify')
@@ -16,7 +17,12 @@ for my $line (<>)
         }
         while ($line =~ /([^\s=]+)=(\S+)/gs)
         {
-            if ($1 eq 'SCHEME' && $2 eq 'ec')
+            if ($1 eq 'TEST_NAME')
+            {
+                $test_name = $base_name.'_'.$2;
+                last;
+            }
+            elsif ($1 eq 'SCHEME' && $2 eq 'ec')
             {
                 $test_name .= '_ec';
             }

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "0.9.2")
+set(VERSION "0.9.3")
 
 add_subdirectory(src)

README-ru.md

@@ -15,7 +15,7 @@ Vitastor архитектурно похож на Ceph, что означает
 и автоматическое распределение данных по любому числу дисков любого размера с настраиваемыми схемами
 избыточности - репликацией или с произвольными кодами коррекции ошибок.
 
-Vitastor нацелен на SSD и SSD+HDD кластеры с как минимум 10 Гбит/с сетью, поддерживает
+Vitastor нацелен в первую очередь на SSD и SSD+HDD кластеры с как минимум 10 Гбит/с сетью, поддерживает
 TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
 что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
 

README.md

@@ -14,8 +14,8 @@ Vitastor is architecturally similar to Ceph which means strong consistency,
 primary-replication, symmetric clustering and automatic data distribution over any
 number of drives of any size with configurable redundancy (replication or erasure codes/XOR).
 
-Vitastor targets SSD and SSD+HDD clusters with at least 10 Gbit/s network, supports
-TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1 ms
+Vitastor targets primarily SSD and SSD+HDD clusters with at least 10 Gbit/s network,
+supports TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1 ms
 with proper hardware which is ~10 times faster than other popular SDS's like Ceph
 or internal systems of public clouds.
 

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.9.2
+VERSION ?= v0.9.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.9.2
+          image: vitalif/vitastor-csi:v0.9.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.9.2
+          image: vitalif/vitastor-csi:v0.9.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.9.2"
+    vitastorCSIDriverVersion = "0.9.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.9.2-1) unstable; urgency=medium
+vitastor (0.9.3-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.9.2-1) unstable; urgency=medium
+vitastor (0.9.3-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/patched-qemu.Dockerfile

@@ -48,7 +48,7 @@ RUN set -e; \
     quilt add block/vitastor.c; \
     cp /root/vitastor/src/qemu_driver.c block/vitastor.c; \
     quilt refresh; \
-    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor1; \
+    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor3; \
     DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
     rm -rf /root/packages/qemu-$REL/qemu-*/

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.9.2; \
-    cd vitastor-0.9.2; \
+    cp -r /root/vitastor vitastor-0.9.3; \
+    cd vitastor-0.9.3; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.2.orig.tar.xz vitastor-0.9.2; \
-    cd vitastor-0.9.2; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.3.orig.tar.xz vitastor-0.9.3; \
+    cd vitastor-0.9.3; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config/layout-cluster.en.md

@@ -33,12 +33,13 @@ Size of objects (data blocks) into which all physical and virtual drives
 in Vitastor, affects memory usage, write amplification and I/O load
 distribution effectiveness.
 
-Recommended default block size is 128 KB for SSD and 4 MB for HDD. In fact,
-it's possible to use 4 MB for SSD too - it will lower memory usage, but
+Recommended default block size is 128 KB for SSD and 1 MB for HDD. In fact,
+it's possible to use 1 MB for SSD too - it will lower memory usage, but
 may increase average WA and reduce linear performance.
 
 OSD memory usage is roughly (SIZE / BLOCK * 68 bytes) which is roughly
 544 MB per 1 TB of used disk space with the default 128 KB block size.
+With 1 MB it's 8 times lower.
 
 ## bitmap_granularity
 

docs/config/layout-cluster.ru.md

@@ -33,14 +33,14 @@ OSD) могут сосуществовать в одном кластере Vita
 настроек, влияет на потребление памяти, объём избыточной записи (write
 amplification) и эффективность распределения нагрузки по OSD.
 
-Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 4
-мегабайта для HDD. В принципе, для SSD можно тоже использовать 4 мегабайта,
+Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 1 мегабайт
+для HDD. В принципе, для SSD можно тоже использовать блок размером 1 мегабайт,
 это понизит использование памяти, но ухудшит распределение нагрузки и в
 среднем увеличит WA.
 
 Потребление памяти OSD составляет примерно (РАЗМЕР / БЛОК * 68 байт),
 т.е. примерно 544 МБ памяти на 1 ТБ занятого места на диске при
-стандартном 128 КБ блоке.
+стандартном 128 КБ блоке. При 1 МБ блоке памяти нужно в 8 раз меньше.
 
 ## bitmap_granularity
 

docs/config/layout-osd.en.md

@@ -24,6 +24,8 @@ initialization and can't be changed after it without losing data.
 - [disable_journal_fsync](#disable_journal_fsync)
 - [disable_device_lock](#disable_device_lock)
 - [disk_alignment](#disk_alignment)
+- [data_csum_type](#data_csum_type)
+- [csum_block_size](#csum_block_size)
 
 ## data_device
 
@@ -174,3 +176,42 @@ Intel Optane (probably, not tested yet).
 
 Clients don't need to be aware of disk_alignment, so it's not required to
 put a modified value into etcd key /vitastor/config/global.
+
+## data_csum_type
+
+- Type: string
+- Default: none
+
+Data checksum type to use. May be "crc32c" or "none". Set to "crc32c" to
+enable data checksums.
+
+## csum_block_size
+
+- Type: integer
+- Default: 4096
+
+Checksum calculation block size.
+
+Must be equal or a multiple of [bitmap_granularity](layout-cluster.en.md#bitmap_granularity)
+(which is usually 4 KB).
+
+Checksums increase metadata size by 4 bytes per each csum_block_size of data.
+
+Checksums are always a compromise:
+1. You either sacrifice +1 GB RAM per 1 TB of data
+2. Or you raise csum_block_size, for example, to 32k and sacrifice
+   50% random write iops due to checksum read-modify-write
+3. Or you turn off [inmemory_metadata](osd.en.md#inmemory_metadata) and
+   sacrifice 50% random read iops due to checksum reads
+
+Option 1 (default) is recommended for all-flash setups because these usually
+have enough RAM.
+
+Option 2 is recommended for HDD-only setups. HDD-only setups usually do NOT
+have enough RAM for the default 4 KB csum_block_size.
+
+Option 3 is recommended for SSD+HDD setups (because metadata SSDs will handle
+extra reads without any performance drop) and also *maybe* for NVMe all-flash
+setups when you don't have enough RAM (because NVMe drives have plenty
+of read iops to spare). You may also consider enabling
+[cached_read_meta](osd.en.md#cached_read_meta) in this case.

docs/config/layout-osd.ru.md

@@ -25,6 +25,8 @@
 - [disable_journal_fsync](#disable_journal_fsync)
 - [disable_device_lock](#disable_device_lock)
 - [disk_alignment](#disk_alignment)
+- [data_csum_type](#data_csum_type)
+- [csum_block_size](#csum_block_size)
 
 ## data_device
 
@@ -183,3 +185,52 @@ journal_block_size и meta_block_size. Однако единственные SSD
 
 Клиентам не обязательно знать про disk_alignment, так что помещать значение
 этого параметра в etcd в /vitastor/config/global не нужно.
+
+## data_csum_type
+
+- Тип: строка
+- Значение по умолчанию: none
+
+Тип используемых OSD контрольных сумм данных. Может быть "crc32c" или "none".
+Установите в "crc32c", чтобы включить расчёт и проверку контрольных сумм данных.
+
+Следует понимать, что контрольные суммы в зависимости от размера блока их
+расчёта либо увеличивают потребление памяти, либо снижают производительность.
+Подробнее смотрите в описании параметра [csum_block_size](#csum_block_size).
+
+## csum_block_size
+
+- Тип: целое число
+- Значение по умолчанию: 4096
+
+Размер блока расчёта контрольных сумм.
+
+Должен быть равен или кратен [bitmap_granularity](layout-cluster.ru.md#bitmap_granularity)
+(который обычно равен 4 КБ).
+
+Контрольные суммы увеличивают размер метаданных на 4 байта на каждые
+csum_block_size данных.
+
+Контрольные суммы - это всегда компромисс:
+1. Вы либо жертвуете потреблением +1 ГБ памяти на 1 ТБ дискового пространства
+2. Либо вы повышаете csum_block_size до, скажем, 32k и жертвуете 50%
+   скорости случайной записи из-за цикла чтения-изменения-записи для расчёта
+   новых контрольных сумм
+3. Либо вы отключаете [inmemory_metadata](osd.ru.md#inmemory_metadata) и
+   жертвуете 50% скорости случайного чтения из-за чтения контрольных сумм
+   с диска
+
+Вариант 1 (при настройках по умолчанию) рекомендуется для SSD (All-Flash)
+кластеров, потому что памяти в них обычно хватает.
+
+Вариант 2 рекомендуется для кластеров на одних жёстких дисках (без SSD
+под метаданные). На 4 кб блок контрольной суммы памяти в таких кластерах
+обычно НЕ хватает.
+
+Вариант 3 рекомендуется для гибридных кластеров (SSD+HDD), потому что
+скорости SSD под метаданными хватит, чтобы обработать дополнительные чтения
+без снижения производительности. Также вариант 3 *может* рекомендоваться
+для All-Flash кластеров на основе NVMe-дисков, когда памяти НЕ достаточно,
+потому что NVMe-диски имеют огромный запас производительности по чтению.
+В таких случаях, возможно, также имеет смысл включать параметр
+[cached_read_meta](osd.ru.md#cached_read_meta).

docs/config/osd.en.md

@@ -31,6 +31,9 @@ them, even without restarting by updating configuration in etcd.
 - [max_flusher_count](#max_flusher_count)
 - [inmemory_metadata](#inmemory_metadata)
 - [inmemory_journal](#inmemory_journal)
+- [cached_read_data](#cached_read_data)
+- [cached_read_meta](#cached_read_meta)
+- [cached_read_journal](#cached_read_journal)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
 - [throttle_small_writes](#throttle_small_writes)
@@ -255,6 +258,46 @@ is typically very small because it's sufficient to have 16-32 MB journal
 for SSD OSDs. However, in theory it's possible that you'll want to turn it
 off for hybrid (HDD+SSD) OSDs with large journals on quick devices.
 
+## cached_read_data
+
+- Type: boolean
+- Default: false
+
+Read data through Linux page cache, i.e. use a file descriptor opened without
+O_DIRECT for data reads. May improve read performance for frequently accessed
+data if it fits in RAM. Memory in page cache is shared by all processes and
+not accounted in OSD memory consumption.
+
+## cached_read_meta
+
+- Type: boolean
+- Default: false
+
+Read metadata through Linux page cache. May be beneficial when checksums
+are enabled and [inmemory_metadata](#inmemory_metadata) is disabled, because
+in this case metadata blocks are read from disk to verify checksums on every
+read request and caching them may reduce this extra read load.
+
+Absolutely pointless to enable with enabled inmemory_metadata because all
+metadata is kept in memory anyway, and likely pointless without checksums,
+because in that case, metadata blocks are read from disk only during journal
+flushing.
+
+If the same device is used for data and metadata, enabling [cached_read_data](#cached_read_data)
+also enables this parameter, given that it isn't turned off explicitly.
+
+## cached_read_journal
+
+- Type: boolean
+- Default: false
+
+Read buffered data from journal through Linux page cache. Does not have sense
+without disabling [inmemory_journal](#inmemory_journal), which, again, is
+enabled by default.
+
+If the same device is used for metadata and journal, enabling [cached_read_meta](#cached_read_meta)
+also enables this parameter, given that it isn't turned off explicitly.
+
 ## journal_sector_buffer_count
 
 - Type: integer

docs/config/osd.ru.md

@@ -32,6 +32,9 @@
 - [max_flusher_count](#max_flusher_count)
 - [inmemory_metadata](#inmemory_metadata)
 - [inmemory_journal](#inmemory_journal)
+- [cached_read_data](#cached_read_data)
+- [cached_read_meta](#cached_read_meta)
+- [cached_read_journal](#cached_read_journal)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
 - [throttle_small_writes](#throttle_small_writes)
@@ -263,6 +266,51 @@ Flusher - это микро-поток (корутина), которая коп
 параметра может оказаться полезным для гибридных OSD (HDD+SSD) с большими
 журналами, расположенными на быстром по сравнению с HDD устройстве.
 
+## cached_read_data
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Читать данные через системный кэш Linux (page cache), то есть, использовать
+для чтения данных файловый дескриптор, открытый без флага O_DIRECT. Может
+улучшить производительность чтения для часто используемых данных, если они
+помещаются в память. Память кэша разделяется между всеми процессами в
+системе и не учитывается в потреблении памяти процессом OSD.
+
+## cached_read_meta
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Читать метаданные через системный кэш Linux. Может быть полезно, когда
+включены контрольные суммы, а параметр [inmemory_metadata](#inmemory_metadata)
+отключён, так как в этом случае блоки метаданных читаются с диска при каждом
+запросе чтения для проверки контрольных сумм и их кэширование может снизить
+дополнительную нагрузку на диск.
+
+Абсолютно бессмысленно включать данный параметр, если параметр
+inmemory_metadata включён (по умолчанию это так), и также вероятно
+бессмысленно включать его, если не включены контрольные суммы, так как в
+этом случае блоки метаданных читаются с диска только во время сброса
+журнала.
+
+Если одно и то же устройство используется для данных и метаданных, включение
+[cached_read_data](#cached_read_data) также включает данный параметр, при
+условии, что он не отключён явным образом.
+
+## cached_read_journal
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Читать буферизованные в журнале данные через системный кэш Linux. Не имеет
+смысла без отключения параметра [inmemory_journal](#inmemory_journal),
+который, опять же, по умолчанию включён.
+
+Если одно и то же устройство используется для метаданных и журнала,
+включение [cached_read_meta](#cached_read_meta) также включает данный
+параметр, при условии, что он не отключён явным образом.
+
 ## journal_sector_buffer_count
 
 - Тип: целое число

docs/config/src/layout-cluster.yml

@@ -7,26 +7,27 @@
     in Vitastor, affects memory usage, write amplification and I/O load
     distribution effectiveness.
 
-    Recommended default block size is 128 KB for SSD and 4 MB for HDD. In fact,
-    it's possible to use 4 MB for SSD too - it will lower memory usage, but
+    Recommended default block size is 128 KB for SSD and 1 MB for HDD. In fact,
+    it's possible to use 1 MB for SSD too - it will lower memory usage, but
     may increase average WA and reduce linear performance.
 
     OSD memory usage is roughly (SIZE / BLOCK * 68 bytes) which is roughly
     544 MB per 1 TB of used disk space with the default 128 KB block size.
+    With 1 MB it's 8 times lower.
   info_ru: |
     Размер объектов (блоков данных), на которые делятся физические и виртуальные
     диски в Vitastor (в рамках каждого пула). Одна из ключевых на данный момент
     настроек, влияет на потребление памяти, объём избыточной записи (write
     amplification) и эффективность распределения нагрузки по OSD.
 
-    Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 4
-    мегабайта для HDD. В принципе, для SSD можно тоже использовать 4 мегабайта,
+    Рекомендуемые по умолчанию размеры блока - 128 килобайт для SSD и 1 мегабайт
+    для HDD. В принципе, для SSD можно тоже использовать блок размером 1 мегабайт,
     это понизит использование памяти, но ухудшит распределение нагрузки и в
     среднем увеличит WA.
 
     Потребление памяти OSD составляет примерно (РАЗМЕР / БЛОК * 68 байт),
     т.е. примерно 544 МБ памяти на 1 ТБ занятого места на диске при
-    стандартном 128 КБ блоке.
+    стандартном 128 КБ блоке. При 1 МБ блоке памяти нужно в 8 раз меньше.
 - name: bitmap_granularity
   type: int
   default: 4096

docs/config/src/layout-osd.yml

@@ -204,3 +204,77 @@
 
     Клиентам не обязательно знать про disk_alignment, так что помещать значение
     этого параметра в etcd в /vitastor/config/global не нужно.
+- name: data_csum_type
+  type: string
+  default: none
+  info: |
+    Data checksum type to use. May be "crc32c" or "none". Set to "crc32c" to
+    enable data checksums.
+  info_ru: |
+    Тип используемых OSD контрольных сумм данных. Может быть "crc32c" или "none".
+    Установите в "crc32c", чтобы включить расчёт и проверку контрольных сумм данных.
+
+    Следует понимать, что контрольные суммы в зависимости от размера блока их
+    расчёта либо увеличивают потребление памяти, либо снижают производительность.
+    Подробнее смотрите в описании параметра [csum_block_size](#csum_block_size).
+- name: csum_block_size
+  type: int
+  default: 4096
+  info: |
+    Checksum calculation block size.
+
+    Must be equal or a multiple of [bitmap_granularity](layout-cluster.en.md#bitmap_granularity)
+    (which is usually 4 KB).
+
+    Checksums increase metadata size by 4 bytes per each csum_block_size of data.
+
+    Checksums are always a compromise:
+    1. You either sacrifice +1 GB RAM per 1 TB of data
+    2. Or you raise csum_block_size, for example, to 32k and sacrifice
+       50% random write iops due to checksum read-modify-write
+    3. Or you turn off [inmemory_metadata](osd.en.md#inmemory_metadata) and
+       sacrifice 50% random read iops due to checksum reads
+
+    Option 1 (default) is recommended for all-flash setups because these usually
+    have enough RAM.
+
+    Option 2 is recommended for HDD-only setups. HDD-only setups usually do NOT
+    have enough RAM for the default 4 KB csum_block_size.
+
+    Option 3 is recommended for SSD+HDD setups (because metadata SSDs will handle
+    extra reads without any performance drop) and also *maybe* for NVMe all-flash
+    setups when you don't have enough RAM (because NVMe drives have plenty
+    of read iops to spare). You may also consider enabling
+    [cached_read_meta](osd.en.md#cached_read_meta) in this case.
+  info_ru: |
+    Размер блока расчёта контрольных сумм.
+
+    Должен быть равен или кратен [bitmap_granularity](layout-cluster.ru.md#bitmap_granularity)
+    (который обычно равен 4 КБ).
+
+    Контрольные суммы увеличивают размер метаданных на 4 байта на каждые
+    csum_block_size данных.
+
+    Контрольные суммы - это всегда компромисс:
+    1. Вы либо жертвуете потреблением +1 ГБ памяти на 1 ТБ дискового пространства
+    2. Либо вы повышаете csum_block_size до, скажем, 32k и жертвуете 50%
+       скорости случайной записи из-за цикла чтения-изменения-записи для расчёта
+       новых контрольных сумм
+    3. Либо вы отключаете [inmemory_metadata](osd.ru.md#inmemory_metadata) и
+       жертвуете 50% скорости случайного чтения из-за чтения контрольных сумм
+       с диска
+
+    Вариант 1 (при настройках по умолчанию) рекомендуется для SSD (All-Flash)
+    кластеров, потому что памяти в них обычно хватает.
+
+    Вариант 2 рекомендуется для кластеров на одних жёстких дисках (без SSD
+    под метаданные). На 4 кб блок контрольной суммы памяти в таких кластерах
+    обычно НЕ хватает.
+
+    Вариант 3 рекомендуется для гибридных кластеров (SSD+HDD), потому что
+    скорости SSD под метаданными хватит, чтобы обработать дополнительные чтения
+    без снижения производительности. Также вариант 3 *может* рекомендоваться
+    для All-Flash кластеров на основе NVMe-дисков, когда памяти НЕ достаточно,
+    потому что NVMe-диски имеют огромный запас производительности по чтению.
+    В таких случаях, возможно, также имеет смысл включать параметр
+    [cached_read_meta](osd.ru.md#cached_read_meta).

docs/config/src/osd.yml

@@ -260,6 +260,70 @@
     достаточно 16- или 32-мегабайтного журнала. Однако в теории отключение
     параметра может оказаться полезным для гибридных OSD (HDD+SSD) с большими
     журналами, расположенными на быстром по сравнению с HDD устройстве.
+- name: cached_read_data
+  type: bool
+  default: false
+  info: |
+    Read data through Linux page cache, i.e. use a file descriptor opened without
+    O_DIRECT for data reads. May improve read performance for frequently accessed
+    data if it fits in RAM. Memory in page cache is shared by all processes and
+    not accounted in OSD memory consumption.
+  info_ru: |
+    Читать данные через системный кэш Linux (page cache), то есть, использовать
+    для чтения данных файловый дескриптор, открытый без флага O_DIRECT. Может
+    улучшить производительность чтения для часто используемых данных, если они
+    помещаются в память. Память кэша разделяется между всеми процессами в
+    системе и не учитывается в потреблении памяти процессом OSD.
+- name: cached_read_meta
+  type: bool
+  default: false
+  info: |
+    Read metadata through Linux page cache. May be beneficial when checksums
+    are enabled and [inmemory_metadata](#inmemory_metadata) is disabled, because
+    in this case metadata blocks are read from disk to verify checksums on every
+    read request and caching them may reduce this extra read load.
+
+    Absolutely pointless to enable with enabled inmemory_metadata because all
+    metadata is kept in memory anyway, and likely pointless without checksums,
+    because in that case, metadata blocks are read from disk only during journal
+    flushing.
+
+    If the same device is used for data and metadata, enabling [cached_read_data](#cached_read_data)
+    also enables this parameter, given that it isn't turned off explicitly.
+  info_ru: |
+    Читать метаданные через системный кэш Linux. Может быть полезно, когда
+    включены контрольные суммы, а параметр [inmemory_metadata](#inmemory_metadata)
+    отключён, так как в этом случае блоки метаданных читаются с диска при каждом
+    запросе чтения для проверки контрольных сумм и их кэширование может снизить
+    дополнительную нагрузку на диск.
+
+    Абсолютно бессмысленно включать данный параметр, если параметр
+    inmemory_metadata включён (по умолчанию это так), и также вероятно
+    бессмысленно включать его, если не включены контрольные суммы, так как в
+    этом случае блоки метаданных читаются с диска только во время сброса
+    журнала.
+
+    Если одно и то же устройство используется для данных и метаданных, включение
+    [cached_read_data](#cached_read_data) также включает данный параметр, при
+    условии, что он не отключён явным образом.
+- name: cached_read_journal
+  type: bool
+  default: false
+  info: |
+    Read buffered data from journal through Linux page cache. Does not have sense
+    without disabling [inmemory_journal](#inmemory_journal), which, again, is
+    enabled by default.
+
+    If the same device is used for metadata and journal, enabling [cached_read_meta](#cached_read_meta)
+    also enables this parameter, given that it isn't turned off explicitly.
+  info_ru: |
+    Читать буферизованные в журнале данные через системный кэш Linux. Не имеет
+    смысла без отключения параметра [inmemory_journal](#inmemory_journal),
+    который, опять же, по умолчанию включён.
+
+    Если одно и то же устройство используется для метаданных и журнала,
+    включение [cached_read_meta](#cached_read_meta) также включает данный
+    параметр, при условии, что он не отключён явным образом.
 - name: journal_sector_buffer_count
   type: int
   default: 32

docs/installation/proxmox.ru.md

@@ -1,4 +1,4 @@
-[Документация](../../README-ru.md#документация) → Установка → Proxmox
+[Документация](../../README-ru.md#документация) → Установка → Proxmox VE
 
 -----
 

docs/intro/features.en.md

@@ -30,6 +30,7 @@
 - [Write throttling to smooth random write workloads in SSD+HDD configurations](../config/osd.en.md#throttle_small_writes)
 - [RDMA/RoCEv2 support via libibverbs](../config/network.en.md#rdma_device)
 - [Scrubbing without checksums](../config/osd.en.md#auto_scrub) (verification of copies)
+- [Checksums](../config/layout-osd.en.md#data_csum_type)
 
 ## Plugins and tools
 
@@ -55,7 +56,6 @@ The following features are planned for the future:
 - iSCSI proxy
 - Multi-threaded client
 - Faster failover
-- Checksums
 - Tiered storage (SSD caching)
 - NVDIMM support
 - Compression (possibly)

docs/intro/features.ru.md

@@ -32,6 +32,7 @@
 - [Сглаживание производительности случайной записи в SSD+HDD конфигурациях](../config/osd.ru.md#throttle_small_writes)
 - [Поддержка RDMA/RoCEv2 через libibverbs](../config/network.ru.md#rdma_device)
 - [Фоновая проверка целостности без контрольных сумм](../config/osd.ru.md#auto_scrub) (сверка копий)
+- [Контрольные суммы](../config/layout-osd.ru.md#data_csum_type)
 
 ## Драйверы и инструменты
 
@@ -55,7 +56,6 @@
 - iSCSI-прокси
 - Многопоточный клиент
 - Более быстрое переключение при отказах
-- Контрольные суммы
 - Поддержка SSD-кэширования (tiered storage)
 - Поддержка NVDIMM
 - Возможно, сжатие

docs/intro/quickstart.en.md

@@ -7,6 +7,7 @@
 # Quick Start
 
 - [Preparation](#preparation)
+- [Recommended drives](#recommended-drives)
 - [Configure monitors](#configure-monitors)
 - [Configure OSDs](#configure-osds)
 - [Create a pool](#create-a-pool)
@@ -19,10 +20,20 @@
 - Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
   with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
   [here](../config/layout-cluster.en.md#immediate_commit).
+- If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
+  Toshiba MG08, Seagate EXOS or something similar. If your drives don't have such cache then
+  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
 - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
 - [Install Vitastor packages](../installation/packages.en.md).
 
+## Recommended drives
+
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+- HDD: HGST Ultrastar, Toshiba MG06/MG07/MG08, Seagate EXOS
+
 ## Configure monitors
 
 On the monitor hosts:
@@ -45,9 +56,10 @@ On the monitor hosts:
   }
   ```
 - Initialize OSDs:
-  - SSD-only: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`. You can add
-    `--disable_data_fsync off` to leave disk cache enabled if you use desktop
-    SSDs without capacitors.
+  - SSD-only or HDD-only: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`.
+    Add `--disable_data_fsync off` to leave disk write cache enabled if you use
+    desktop SSDs without capacitors. Do NOT add `--disable_data_fsync off` if you
+    use HDDs or SSD+HDD.
   - Hybrid, SSD+HDD: `vitastor-disk prepare --hybrid /dev/sdXXX [/dev/sdYYY ...]`.
     Pass all your devices (HDD and SSD) to this script — it will partition disks and initialize journals on its own.
     This script skips HDDs which are already partitioned so if you want to use non-empty disks for

docs/intro/quickstart.ru.md

@@ -7,6 +7,7 @@
 # Быстрый старт
 
 - [Подготовка](#подготовка)
+- [Рекомендуемые диски](#рекомендуемые-диски)
 - [Настройте мониторы](#настройте-мониторы)
 - [Настройте OSD](#настройте-osd)
 - [Создайте пул](#создайте-пул)
@@ -19,10 +20,20 @@
 - Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
   использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
   О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
+- Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
+  Toshiba MG08, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
+  обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места).
 - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2.
 - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
 - [Установите пакеты Vitastor](../installation/packages.ru.md).
 
+## Рекомендуемые диски
+
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+- HDD: HGST Ultrastar, Toshiba MG06/MG07/MG08, Seagate EXOS
+
 ## Настройте мониторы
 
 На хостах, выделенных под мониторы:
@@ -45,9 +56,10 @@
   }
   ```
 - Инициализуйте OSD:
-  - SSD: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`. Если вы используете
-    десктопные SSD без конденсаторов, можете оставить кэш включённым, добавив
-    опцию `--disable_data_fsync off`.
+  - Только SSD или только HDD: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`.
+    Если вы используете десктопные SSD без конденсаторов, добавьте опцию `--disable_data_fsync off`,
+    чтобы оставить кэш записи диска включённым. НЕ добавляйте эту опцию, если используете
+    жёсткие диски (HDD).
   - Гибридные, SSD+HDD: `vitastor-disk prepare --hybrid /dev/sdXXX [/dev/sdYYY ...]`.
     Передайте все ваши SSD и HDD скрипту в командной строке подряд, скрипт автоматически выделит
     разделы под журналы на SSD и данные на HDD. Скрипт пропускает HDD, на которых уже есть разделы

docs/usage/disk.en.md

@@ -86,6 +86,8 @@ Options (both modes):
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity
+--data_csum_type none      Set data checksum type (crc32c or none)
+--csum_block_size 4k       Set data checksum block size
 --data_device_block 4k     Override data device block size
 --meta_device_block 4k     Override metadata device block size
 --journal_device_block 4k  Override journal device block size
@@ -100,8 +102,9 @@ checks the device cache status on start and tries to disable cache for SATA/SAS
 If it doesn't succeed it issues a warning in the system log.
 
 You can also pass other OSD options here as arguments and they'll be persisted
-to the superblock: max_write_iodepth, max_write_iodepth, min_flusher_count,
-max_flusher_count, inmemory_metadata, inmemory_journal, journal_sector_buffer_count,
+in the superblock: cached_read_data, cached_read_meta, cached_read_journal,
+inmemory_metadata, inmemory_journal, max_write_iodepth,
+min_flusher_count, max_flusher_count, journal_sector_buffer_count,
 journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,
 throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.
 See [Runtime OSD Parameters](../config/osd.en.md) for details.
@@ -249,7 +252,9 @@ Options (see also [Cluster-Wide Disk Layout Parameters](../config/layout-cluster
 ```
 --object_size 128k       Set blockstore block size
 --bitmap_granularity 4k  Set bitmap granularity
---journal_size 32M       Set journal size
+--journal_size 16M       Set journal size
+--data_csum_type none    Set data checksum type (crc32c or none)
+--csum_block_size 4k     Set data checksum block size
 --device_block_size 4k   Set device block size
 --journal_offset 0       Set journal offset
 --device_size 0          Set device size

docs/usage/disk.ru.md

@@ -87,6 +87,8 @@ vitastor-disk - инструмент командной строки для уп
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт
+--data_csum_type none      Задать тип контрольных сумм (crc32c или none)
+--csum_block_size 4k       Задать размер блока расчёта контрольных сумм
 --data_device_block 4k     Задать размер блока устройства данных
 --meta_device_block 4k     Задать размер блока метаданных
 --journal_device_block 4k  Задать размер блока журнала
@@ -101,8 +103,9 @@ vitastor-disk - инструмент командной строки для уп
 это не удаётся, в системный журнал выводится предупреждение.
 
 Вы можете передать данной команде и некоторые другие опции OSD в качестве аргументов
-и они тоже будут сохранены в суперблок: max_write_iodepth, max_write_iodepth, min_flusher_count,
-max_flusher_count, inmemory_metadata, inmemory_journal, journal_sector_buffer_count,
+и они тоже будут сохранены в суперблок: cached_read_data, cached_read_meta,
+cached_read_journal, inmemory_metadata, inmemory_journal, max_write_iodepth,
+min_flusher_count, max_flusher_count, journal_sector_buffer_count,
 journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,
 throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.
 Читайте об этих параметрах подробнее в разделе [Изменяемые параметры OSD](../config/osd.ru.md).
@@ -254,7 +257,9 @@ OSD отключены fsync-и.
 ```
 --object_size 128k       Размер блока хранилища
 --bitmap_granularity 4k  Гранулярность битовых карт
---journal_size 32M       Размер журнала
+--journal_size 16M       Размер журнала
+--data_csum_type none    Задать тип контрольных сумм (crc32c или none)
+--csum_block_size 4k     Задать размер блока расчёта контрольных сумм
 --device_block_size 4k   Размер блока устройства
 --journal_offset 0       Смещение журнала
 --device_size 0          Размер устройства

docs/usage/nbd.en.md

@@ -13,6 +13,8 @@ remains decent (see an example [here](../performance/comparison1.en.md#vitastor-
 
 Vitastor Kubernetes CSI driver is based on NBD.
 
+See also [VDUSE](qemu.en.md#vduse).
+
 ## Map image
 
 To create a local block device for a Vitastor image run:

docs/usage/nbd.ru.md

@@ -16,6 +16,8 @@ NBD немного снижает производительность из-за
 
 CSI-драйвер Kubernetes Vitastor основан на NBD.
 
+Смотрите также [VDUSE](qemu.ru.md#vduse).
+
 ## Подключить устройство
 
 Чтобы создать локальное блочное устройство для образа, выполните команду:

docs/usage/qemu.en.md

@@ -83,3 +83,43 @@ qemu-img rebase -u -b '' testimg.qcow2
 This can be used for backups. Just note that exporting an image that is currently being written to
 is of course unsafe and doesn't produce a consistent result, so only export snapshots if you do this
 on a live VM.
+
+## VDUSE
+
+Linux kernel, starting with version 5.15, supports a new interface for attaching virtual disks
+to the host - VDUSE (vDPA Device in Userspace). QEMU, starting with 7.2, has support for
+exporting QEMU block devices over this protocol using qemu-storage-daemon.
+
+VDUSE has the same problem as other FUSE-like interfaces in Linux: if a userspace process hangs,
+for example, if it loses connectivity with Vitastor cluster - active processes doing I/O may
+hang in the D state (uninterruptible sleep) and you won't be able to kill them even with kill -9.
+In this case reboot will be the only way to remove VDUSE devices from system.
+
+On the other hand, VDUSE is faster than [NBD](nbd.en.md), so you may prefer to use it if
+performance is important for you. Approximate performance numbers:
+direct fio benchmark - 115000 iops, NBD - 60000 iops, VDUSE - 90000 iops.
+
+To try VDUSE you need at least Linux 5.15, built with VDUSE support
+(CONFIG_VIRTIO_VDPA=m and CONFIG_VDPA_USER=m). Debian Linux kernels have these options
+disabled by now, so if you want to try it on Debian, use a kernel from Ubuntu
+[kernel-ppa/mainline](https://kernel.ubuntu.com/~kernel-ppa/mainline/) or Proxmox.
+
+Commands to attach Vitastor image as a VDUSE device:
+
+```
+modprobe vduse virtio-vdpa
+qemu-storage-daemon --daemonize --blockdev '{"node-name":"test1","driver":"vitastor",\
+  "etcd-host":"192.168.7.2:2379/v3","image":"testosd1","cache":{"direct":true,"no-flush":false},"discard":"unmap"}' \
+  --export vduse-blk,id=test1,node-name=test1,name=test1,num-queues=16,queue-size=128,writable=true
+vdpa dev add name test1 mgmtdev vduse
+```
+
+After running these commands /dev/vda device will appear in the system and you'll be able to
+use it as a normal disk.
+
+To remove the device:
+
+```
+vdpa dev del test1
+kill <qemu-storage-daemon_process_PID>
+```

docs/usage/qemu.ru.md

@@ -87,3 +87,43 @@ qemu-img rebase -u -b '' testimg.qcow2
 Это можно использовать для резервного копирования. Только помните, что экспортировать образ, в который
 в то же время идёт запись, небезопасно - результат чтения не будет целостным. Так что если вы работаете
 с активными виртуальными машинами, экспортируйте только их снимки, но не сам образ.
+
+## VDUSE
+
+В Linux, начиная с версии ядра 5.15, доступен новый интерфейс для подключения виртуальных дисков
+к системе - VDUSE (vDPA Device in Userspace), а в QEMU, начиная с версии 7.2, есть поддержка
+экспорта блочных устройств QEMU по этому протоколу через qemu-storage-daemon.
+
+VDUSE страдает общей проблемой FUSE-подобных интерфейсов в Linux: если пользовательский процесс
+подвиснет, например, если будет потеряна связь с кластером Vitastor - читающие/пишущие в кластер
+процессы могут "залипнуть" в состоянии D (непрерываемый сон) и их будет невозможно убить даже
+через kill -9. В этом случае удалить из системы устройство можно только перезагрузившись.
+
+С другой стороны, VDUSE быстрее по сравнению с [NBD](nbd.ru.md), поэтому его может
+быть предпочтительно использовать там, где производительность важнее. Порядок показателей:
+прямое тестирование через fio - 115000 iops, NBD - 60000 iops, VDUSE - 90000 iops.
+
+Чтобы использовать VDUSE, вам нужно ядро Linux версии хотя бы 5.15, собранное с поддержкой
+VDUSE (CONFIG_VIRTIO_VDPA=m и CONFIG_VDPA_USER=m). В ядрах в Debian Linux поддержка пока
+отключена - если хотите попробовать эту функцию на Debian, поставьте ядро из Ubuntu
+[kernel-ppa/mainline](https://kernel.ubuntu.com/~kernel-ppa/mainline/) или из Proxmox.
+
+Команды для подключения виртуального диска через VDUSE:
+
+```
+modprobe vduse virtio-vdpa
+qemu-storage-daemon --daemonize --blockdev '{"node-name":"test1","driver":"vitastor",\
+  "etcd-host":"192.168.7.2:2379/v3","image":"testosd1","cache":{"direct":true,"no-flush":false},"discard":"unmap"}' \
+  --export vduse-blk,id=test1,node-name=test1,name=test1,num-queues=16,queue-size=128,writable=true
+vdpa dev add name test1 mgmtdev vduse
+```
+
+После этого в системе появится устройство /dev/vda, которое можно будет использовать как
+обычный диск.
+
+Для удаления устройства из системы:
+
+```
+vdpa dev del test1
+kill <PID_процесса_qemu-storage-daemon>
+```

mon/make-etcd

@@ -63,8 +63,9 @@ Wants=network-online.target local-fs.target time-sync.target
 
 [Service]
 Restart=always
+Environment=GOGC=50
 ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
-    --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
+    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
     --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
     --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
     --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\

mon/mon.js

@@ -1608,7 +1608,7 @@ class Mon
                 }
             }
         }
-        return inode_stats;
+        return { inode_stats, seen_pools };
     }
 
     serialize_bigints(obj)
@@ -1634,7 +1634,7 @@ class Mon
         const timestamp = Date.now();
         const { object_counts, object_bytes } = this.sum_object_counts();
         let stats = this.sum_op_stats(timestamp, this.prev_stats);
-        let inode_stats = this.sum_inode_stats(
+        let { inode_stats, seen_pools } = this.sum_inode_stats(
             this.prev_stats ? this.prev_stats.inode_stats : null,
             timestamp, this.prev_stats ? this.prev_stats.timestamp : null
         );
@@ -1669,12 +1669,22 @@ class Mon
         }
         for (const pool_id in this.state.pool.stats)
         {
-            const pool_stats = { ...this.state.pool.stats[pool_id] };
-            this.serialize_bigints(pool_stats);
-            txn.push({ requestPut: {
-                key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
-                value: b64(JSON.stringify(pool_stats)),
-            } });
+            if (!seen_pools[pool_id])
+            {
+                txn.push({ requestDeleteRange: {
+                    key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
+                } });
+                delete this.state.pool.stats[pool_id];
+            }
+            else
+            {
+                const pool_stats = { ...this.state.pool.stats[pool_id] };
+                this.serialize_bigints(pool_stats);
+                txn.push({ requestPut: {
+                    key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
+                    value: b64(JSON.stringify(pool_stats)),
+                } });
+            }
         }
         if (txn.length)
         {

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.9.2'
+VERSION = '0.9.3'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.9.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.2$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.9.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.3$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el7.tar.gz
+Source0:        vitastor-0.9.3.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el8.tar.gz
+Source0:        vitastor-0.9.3.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el9.tar.gz
+Source0:        vitastor-0.9.3.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.9.2")
+add_definitions(-DVERSION="0.9.3")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/allocator.cpp

@@ -143,34 +143,83 @@ uint64_t allocator::get_free_count()
     return free;
 }
 
+// FIXME: Move to utils?
 void bitmap_set(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
 {
-    if (start == 0)
+    if (start == 0 && len == 32*bitmap_granularity)
+        *((uint32_t*)bitmap) = UINT32_MAX;
+    else if (start == 0 && len == 64*bitmap_granularity)
+        *((uint64_t*)bitmap) = UINT64_MAX;
+    else
     {
-        if (len == 32*bitmap_granularity)
+        unsigned bit_start = start / bitmap_granularity;
+        unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
+        while (bit_start < bit_end)
         {
-            *((uint32_t*)bitmap) = UINT32_MAX;
-            return;
-        }
-        else if (len == 64*bitmap_granularity)
-        {
-            *((uint64_t*)bitmap) = UINT64_MAX;
-            return;
-        }
-    }
-    unsigned bit_start = start / bitmap_granularity;
-    unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
-    while (bit_start < bit_end)
-    {
-        if (!(bit_start & 7) && bit_end >= bit_start+8)
-        {
-            ((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
-            bit_start += 8;
-        }
-        else
-        {
-            ((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
-            bit_start++;
+            if (!(bit_start & 7) && bit_end >= bit_start+8)
+            {
+                ((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
+                bit_start += 8;
+            }
+            else
+            {
+                ((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
+                bit_start++;
+            }
         }
     }
 }
+
+void bitmap_clear(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
+{
+    if (start == 0 && len == 32*bitmap_granularity)
+        *((uint32_t*)bitmap) = 0;
+    else if (start == 0 && len == 64*bitmap_granularity)
+        *((uint64_t*)bitmap) = 0;
+    else
+    {
+        unsigned bit_start = start / bitmap_granularity;
+        unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
+        while (bit_start < bit_end)
+        {
+            if (!(bit_start & 7) && bit_end >= bit_start+8)
+            {
+                ((uint8_t*)bitmap)[bit_start / 8] = 0;
+                bit_start += 8;
+            }
+            else
+            {
+                ((uint8_t*)bitmap)[bit_start / 8] &= (0xFF ^ (1 << (bit_start % 8)));
+                bit_start++;
+            }
+        }
+    }
+}
+
+bool bitmap_check(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
+{
+    bool r = false;
+    if (start == 0 && len == 32*bitmap_granularity)
+        r = !!*((uint32_t*)bitmap);
+    else if (start == 0 && len == 64*bitmap_granularity)
+        r = !!*((uint64_t*)bitmap);
+    else
+    {
+        unsigned bit_start = start / bitmap_granularity;
+        unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
+        while (bit_start < bit_end)
+        {
+            if (!(bit_start & 7) && bit_end >= bit_start+8)
+            {
+                r = r || !!((uint8_t*)bitmap)[bit_start / 8];
+                bit_start += 8;
+            }
+            else
+            {
+                r = r || (((uint8_t*)bitmap)[bit_start / 8] & (1 << (bit_start % 8)));
+                bit_start++;
+            }
+        }
+    }
+    return r;
+}

src/allocator.h

@@ -23,3 +23,5 @@ public:
 };
 
 void bitmap_set(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);
+void bitmap_clear(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);
+bool bitmap_check(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);

src/blockstore.h

@@ -77,6 +77,7 @@ Output:
   -EINVAL = invalid input parameters
   -ENOENT = requested object/version does not exist for reads
   -ENOSPC = no space left in the store for writes
+  -EDOM = checksum error.
 - version = the version actually read or written
 
 ## BS_OP_DELETE

src/blockstore_disk.cpp

@@ -40,10 +40,31 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     data_block_size = parse_size(config["block_size"]);
     journal_device = config["journal_device"];
     journal_offset = parse_size(config["journal_offset"]);
-    disk_alignment = strtoull(config["disk_alignment"].c_str(), NULL, 10);
-    journal_block_size = strtoull(config["journal_block_size"].c_str(), NULL, 10);
-    meta_block_size = strtoull(config["meta_block_size"].c_str(), NULL, 10);
-    bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
+    disk_alignment = parse_size(config["disk_alignment"]);
+    journal_block_size = parse_size(config["journal_block_size"]);
+    meta_block_size = parse_size(config["meta_block_size"]);
+    bitmap_granularity = parse_size(config["bitmap_granularity"]);
+    meta_format = stoull_full(config["meta_format"]);
+    cached_read_data = config["cached_read_data"] == "true" || config["cached_read_data"] == "yes" || config["cached_read_data"] == "1";
+    cached_read_meta = cached_read_data && (meta_device == data_device || meta_device == "") &&
+        config.find("cached_read_meta") == config.end() ||
+        config["cached_read_meta"] == "true" || config["cached_read_meta"] == "yes" || config["cached_read_meta"] == "1";
+    cached_read_journal = cached_read_meta && (journal_device == meta_device || journal_device == "") &&
+        config.find("cached_read_journal") == config.end() ||
+        config["cached_read_journal"] == "true" || config["cached_read_journal"] == "yes" || config["cached_read_journal"] == "1";
+    if (config["data_csum_type"] == "crc32c")
+    {
+        data_csum_type = BLOCKSTORE_CSUM_CRC32C;
+    }
+    else if (config["data_csum_type"] == "" || config["data_csum_type"] == "none")
+    {
+        data_csum_type = BLOCKSTORE_CSUM_NONE;
+    }
+    else
+    {
+        throw std::runtime_error("data_csum_type="+config["data_csum_type"]+" is unsupported, only \"crc32c\" and \"none\" are supported");
+    }
+    csum_block_size = parse_size(config["csum_block_size"]);
     // Validate
     if (!data_block_size)
     {
@@ -91,7 +112,23 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
     if (data_block_size % bitmap_granularity)
     {
-        throw std::runtime_error("Block size must be a multiple of sparse write tracking granularity");
+        throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
+    }
+    if (!data_csum_type)
+    {
+        csum_block_size = 0;
+    }
+    else if (!csum_block_size)
+    {
+        csum_block_size = bitmap_granularity;
+    }
+    if (csum_block_size && (csum_block_size % bitmap_granularity))
+    {
+        throw std::runtime_error("Checksum block size must be a multiple of sparse write tracking granularity");
+    }
+    if (csum_block_size && (data_block_size % csum_block_size))
+    {
+        throw std::runtime_error("Checksum block size must be a divisor of data block size");
     }
     if (meta_device == "")
     {
@@ -110,7 +147,9 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
         throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
     }
     clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
-    clean_entry_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
+    clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
+        ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
+    clean_entry_size = sizeof(clean_disk_entry) + clean_dyn_size + 4 /*entry_csum*/;
 }
 
 void blockstore_disk_t::calc_lengths(bool skip_meta_check)
@@ -160,6 +199,25 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
     // required metadata size
     block_count = data_len / data_block_size;
     meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
+    if (meta_format == BLOCKSTORE_META_FORMAT_V1 ||
+        !meta_format && !skip_meta_check && meta_area_size < meta_len && !data_csum_type)
+    {
+        uint64_t clean_entry_v0_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
+        uint64_t meta_v0_len = (1 + (block_count - 1 + meta_block_size / clean_entry_v0_size)
+            / (meta_block_size / clean_entry_v0_size)) * meta_block_size;
+        if (meta_format == BLOCKSTORE_META_FORMAT_V1 || meta_area_size >= meta_v0_len)
+        {
+            // Old metadata fits.
+            printf("Warning: Using old metadata format without checksums because the new format doesn't fit into provided area\n");
+            clean_entry_size = clean_entry_v0_size;
+            meta_len = meta_v0_len;
+            meta_format = BLOCKSTORE_META_FORMAT_V1;
+        }
+        else
+            meta_format = BLOCKSTORE_META_FORMAT_V2;
+    }
+    else
+        meta_format = BLOCKSTORE_META_FORMAT_V2;
     if (!skip_meta_check && meta_area_size < meta_len)
     {
         throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(meta_len)+" bytes");
@@ -237,6 +295,18 @@ void blockstore_disk_t::open_data()
     {
         throw std::runtime_error(std::string("Failed to lock data device: ") + strerror(errno));
     }
+    if (cached_read_data)
+    {
+        read_data_fd = open(data_device.c_str(), O_RDWR);
+        if (read_data_fd == -1)
+        {
+            throw std::runtime_error("Failed to open data device "+data_device+": "+std::string(strerror(errno)));
+        }
+    }
+    else
+    {
+        read_data_fd = data_fd;
+    }
 }
 
 void blockstore_disk_t::open_meta()
@@ -257,6 +327,18 @@ void blockstore_disk_t::open_meta()
         {
             throw std::runtime_error(std::string("Failed to lock metadata device: ") + strerror(errno));
         }
+        if (cached_read_meta)
+        {
+            read_meta_fd = open(meta_device.c_str(), O_RDWR);
+            if (read_meta_fd == -1)
+            {
+                throw std::runtime_error("Failed to open metadata device "+meta_device+": "+std::string(strerror(errno)));
+            }
+        }
+        else
+        {
+            read_meta_fd = meta_fd;
+        }
     }
     else
     {
@@ -275,6 +357,22 @@ void blockstore_disk_t::open_meta()
             ") is not a multiple of data device sector size ("+std::to_string(meta_device_sect)+")"
         );
     }
+    if (!cached_read_meta)
+    {
+        read_meta_fd = meta_fd;
+    }
+    else if (meta_device == data_device && cached_read_data)
+    {
+        read_meta_fd = read_data_fd;
+    }
+    else
+    {
+        read_meta_fd = open(meta_device.c_str(), O_RDWR);
+        if (read_meta_fd == -1)
+        {
+            throw std::runtime_error("Failed to open metadata device "+meta_device+": "+std::string(strerror(errno)));
+        }
+    }
 }
 
 void blockstore_disk_t::open_journal()
@@ -309,6 +407,26 @@ void blockstore_disk_t::open_journal()
             ") is not a multiple of journal device sector size ("+std::to_string(journal_device_sect)+")"
         );
     }
+    if (!cached_read_journal)
+    {
+        read_journal_fd = journal_fd;
+    }
+    else if (journal_device == meta_device && cached_read_meta)
+    {
+        read_journal_fd = read_meta_fd;
+    }
+    else if (journal_device == data_device && cached_read_data)
+    {
+        read_journal_fd = read_data_fd;
+    }
+    else
+    {
+        read_journal_fd = open(journal_device.c_str(), O_RDWR);
+        if (read_journal_fd == -1)
+        {
+            throw std::runtime_error("Failed to open journal device "+journal_device+": "+std::string(strerror(errno)));
+        }
+    }
 }
 
 void blockstore_disk_t::close_all()
@@ -319,5 +437,12 @@ void blockstore_disk_t::close_all()
         close(meta_fd);
     if (journal_fd >= 0 && journal_fd != meta_fd)
         close(journal_fd);
+    if (read_data_fd >= 0 && read_data_fd != data_fd)
+        close(read_data_fd);
+    if (read_meta_fd >= 0 && read_meta_fd != meta_fd)
+        close(read_meta_fd);
+    if (read_journal_fd >= 0 && read_journal_fd != journal_fd)
+        close(read_journal_fd);
     data_fd = meta_fd = journal_fd = -1;
+    read_data_fd = read_meta_fd = read_journal_fd = -1;
 }

src/blockstore_disk.h

@@ -8,6 +8,10 @@
 #include <string>
 #include <map>
 
+#define BLOCKSTORE_CSUM_NONE 0
+// Lower byte of checksum type is its length
+#define BLOCKSTORE_CSUM_CRC32C 0x104
+
 struct blockstore_disk_t
 {
     std::string data_device, meta_device, journal_device;
@@ -21,17 +25,24 @@ struct blockstore_disk_t
     uint64_t meta_block_size = 4096;
     // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
     uint64_t bitmap_granularity = 4096;
+    // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
+    uint32_t data_csum_type = BLOCKSTORE_CSUM_NONE;
+    // Checksum block size, must be a multiple of bitmap_granularity
+    uint32_t csum_block_size = 4096;
     // By default, Blockstore locks all opened devices exclusively. This option can be used to disable locking
     bool disable_flock = false;
+    // Use linux page cache for reads. If enabled, separate buffered FDs will be opened for reading
+    bool cached_read_data = false, cached_read_meta = false, cached_read_journal = false;
 
     int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_len;
+    int read_meta_fd = -1, read_data_fd = -1, read_journal_fd = -1;
+    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_len, meta_format = 0;
     uint64_t data_offset, data_device_sect, data_device_size, data_len;
     uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
 
     uint32_t block_order;
     uint64_t block_count;
-    uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0;
+    uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0, clean_dyn_size = 0;
 
     void parse_config(std::map<std::string, std::string> & config);
     void open_data();
@@ -39,4 +50,13 @@ struct blockstore_disk_t
     void open_journal();
     void calc_lengths(bool skip_meta_check = false);
     void close_all();
+
+    inline uint64_t dirty_dyn_size(uint64_t offset, uint64_t len)
+    {
+        // Checksums may be partial if write is not aligned with csum_block_size
+        return clean_entry_bitmap_size + (csum_block_size && len > 0
+            ? ((offset+len+csum_block_size-1)/csum_block_size - offset/csum_block_size)
+                * (data_csum_type & 0xFF)
+            : 0);
+    }
 };

src/blockstore_flush.h

@@ -1,10 +1,22 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
+#define COPY_BUF_JOURNAL 1
+#define COPY_BUF_DATA 2
+#define COPY_BUF_ZERO 4
+#define COPY_BUF_CSUM_FILL 8
+#define COPY_BUF_COALESCED 16
+#define COPY_BUF_META_BLOCK 32
+#define COPY_BUF_JOURNALED_BIG 64
+
 struct copy_buffer_t
 {
-    uint64_t offset, len;
+    int copy_flags;
+    uint64_t offset, len, disk_offset;
+    uint64_t journal_sector; // only for reads: sector+1 if used and !journal.inmemory, otherwise 0
     void *buf;
+    uint8_t *csum_buf;
+    int *dyn_data;
 };
 
 struct meta_sector_t
@@ -37,7 +49,7 @@ class journal_flusher_co
 {
     blockstore_impl_t *bs;
     journal_flusher_t *flusher;
-    int wait_state, wait_count;
+    int wait_state, wait_count, wait_journal_count;
     struct io_uring_sqe *sqe;
     struct ring_data_t *data;
 
@@ -46,28 +58,39 @@ class journal_flusher_co
     obj_ver_id cur;
     std::map<obj_ver_id, dirty_entry>::iterator dirty_it, dirty_start, dirty_end;
     std::map<object_id, uint64_t>::iterator repeat_it;
-    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
+    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_rj, simple_callback_w;
 
     bool skip_copy, has_delete, has_writes;
     std::vector<copy_buffer_t> v;
     std::vector<copy_buffer_t>::iterator it;
+    int i;
+    bool fill_incomplete, cleared_incomplete;
+    int read_to_fill_incomplete;
     int copy_count;
-    uint64_t clean_loc, old_clean_loc;
+    uint64_t clean_loc, clean_ver, old_clean_loc, old_clean_ver;
     flusher_meta_write_t meta_old, meta_new;
     bool clean_init_bitmap;
     uint64_t clean_bitmap_offset, clean_bitmap_len;
-    void *new_clean_bitmap;
+    uint8_t *clean_init_dyn_ptr;
+    uint8_t *new_clean_bitmap;
 
     uint64_t new_trim_pos;
 
-    // local: scan_dirty()
-    uint64_t offset, end_offset, submit_offset, submit_len;
-
     friend class journal_flusher_t;
-    bool scan_dirty(int wait_base);
+    void scan_dirty();
+    bool read_dirty(int wait_base);
+    bool modify_meta_do_reads(int wait_base);
+    bool wait_meta_reads(int wait_base);
     bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base);
+    bool clear_incomplete_csum_block_bits(int wait_base);
+    void calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites);
+    void update_metadata_entry();
+    bool write_meta_block(flusher_meta_write_t & meta_block, int wait_base);
     void update_clean_db();
+    void free_data_blocks();
     bool fsync_batch(bool fsync_meta, int wait_base);
+    bool trim_journal(int wait_base);
+    void free_buffers();
 public:
     journal_flusher_co();
     bool loop();
@@ -95,14 +118,16 @@ class journal_flusher_t
 
     std::map<uint64_t, meta_sector_t> meta_sectors;
     std::deque<object_id> flush_queue;
-    std::map<object_id, uint64_t> flush_versions;
+    std::map<object_id, uint64_t> flush_versions; // FIXME: consider unordered_map?
 
     bool try_find_older(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
+    bool try_find_other(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
 
 public:
     journal_flusher_t(blockstore_impl_t *bs);
     ~journal_flusher_t();
     void loop();
+    bool is_trim_wanted() { return trim_wanted; }
     bool is_active();
     void mark_trim_possible();
     void request_trim();
@@ -111,4 +136,5 @@ public:
     void unshift_flush(obj_ver_id oid, bool force);
     void remove_flush(object_id oid);
     void dump_diagnostics();
+    bool is_mutated(uint64_t clean_loc);
 };

src/blockstore_impl.cpp

@@ -13,6 +13,7 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *
     initialized = 0;
     parse_config(config, true);
     zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
+    alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0;
     try
     {
         dsk.open_data();
@@ -38,8 +39,8 @@ blockstore_impl_t::~blockstore_impl_t()
     dsk.close_all();
     if (metadata_buffer)
         free(metadata_buffer);
-    if (clean_bitmap)
-        free(clean_bitmap);
+    if (clean_bitmaps)
+        free(clean_bitmaps);
 }
 
 bool blockstore_impl_t::is_started()

src/blockstore_impl.h

@@ -93,11 +93,10 @@
 
 // "VITAstor"
 #define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
-#define BLOCKSTORE_META_VERSION_V1 1
+#define BLOCKSTORE_META_FORMAT_V1 1
+#define BLOCKSTORE_META_FORMAT_V2 2
 
 // metadata header (superblock)
-// FIXME: After adding the OSD superblock, add a key to metadata
-// and journal headers to check if they belong to the same OSD
 struct __attribute__((__packed__)) blockstore_meta_header_v1_t
 {
     uint64_t zero;
@@ -108,14 +107,29 @@ struct __attribute__((__packed__)) blockstore_meta_header_v1_t
     uint32_t bitmap_granularity;
 };
 
+struct __attribute__((__packed__)) blockstore_meta_header_v2_t
+{
+    uint64_t zero;
+    uint64_t magic;
+    uint64_t version;
+    uint32_t meta_block_size;
+    uint32_t data_block_size;
+    uint32_t bitmap_granularity;
+    uint32_t data_csum_type;
+    uint32_t csum_block_size;
+    uint32_t header_csum;
+};
+
 // 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
 // per "clean" entry on disk with fixed metadata tables
-// FIXME: maybe add crc32's to metadata
 struct __attribute__((__packed__)) clean_disk_entry
 {
     object_id oid;
     uint64_t version;
     uint8_t bitmap[];
+    // Two more fields come after bitmap in metadata version 2:
+    // uint32_t data_csum[];
+    // uint32_t entry_csum;
 };
 
 // 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
@@ -125,7 +139,7 @@ struct __attribute__((__packed__)) clean_entry
     uint64_t location;
 };
 
-// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry)
+// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums
 struct __attribute__((__packed__)) dirty_entry
 {
     uint32_t state;
@@ -134,7 +148,7 @@ struct __attribute__((__packed__)) dirty_entry
     uint32_t offset;   // data offset within object (stripe)
     uint32_t len;      // data length
     uint64_t journal_sector; // journal sector used for this entry
-    void* bitmap;   // either external bitmap itself when it fits, or a pointer to it when it doesn't
+    void* dyn_data;    // dynamic data: external bitmap and data block checksums. may be a pointer to the in-memory journal
 };
 
 // - Sync must be submitted after previous writes/deletes (not before!)
@@ -163,12 +177,23 @@ struct __attribute__((__packed__)) dirty_entry
 // Suspend operation until there is some free space on the data device
 #define WAIT_FREE 5
 
-struct fulfill_read_t
+struct used_clean_obj_t
 {
-    uint64_t offset, len;
-    uint64_t journal_sector; // sector+1 if used and !journal.inmemory, otherwise 0
+    int refs;
+    bool was_freed; // was freed by a parallel flush?
+    bool was_changed; // was changed by a parallel flush?
 };
 
+// https://github.com/algorithm-ninja/cpp-btree
+// https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing
+// with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB
+typedef btree::btree_map<object_id, clean_entry> blockstore_clean_db_t;
+typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
+
+#include "blockstore_init.h"
+
+#include "blockstore_flush.h"
+
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
 #define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
 
@@ -181,7 +206,8 @@ struct blockstore_op_private_t
     int op_state;
 
     // Read
-    std::vector<fulfill_read_t> read_vec;
+    uint64_t clean_block_used;
+    std::vector<copy_buffer_t> read_vec;
 
     // Sync, write
     int min_flushed_journal_sector, max_flushed_journal_sector;
@@ -197,16 +223,6 @@ struct blockstore_op_private_t
     int sync_small_checked, sync_big_checked;
 };
 
-// https://github.com/algorithm-ninja/cpp-btree
-// https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing
-// with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB
-typedef btree::btree_map<object_id, clean_entry> blockstore_clean_db_t;
-typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
-
-#include "blockstore_init.h"
-
-#include "blockstore_flush.h"
-
 typedef uint32_t pool_id_t;
 typedef uint64_t pool_pg_id_t;
 
@@ -253,7 +269,7 @@ class blockstore_impl_t
 
     std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
     std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
-    uint8_t *clean_bitmap = NULL;
+    uint8_t *clean_bitmaps = NULL;
     blockstore_dirty_db_t dirty_db;
     std::vector<blockstore_op_t*> submit_queue;
     std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
@@ -267,6 +283,10 @@ class blockstore_impl_t
     journal_flusher_t *flusher;
     int big_to_flush = 0;
     int write_iodepth = 0;
+    bool alloc_dyn_data = false;
+
+    // clean data blocks referenced by read operations
+    std::map<uint64_t, used_clean_obj_t> used_clean_objects;
 
     bool live = false, queue_stall = false;
     ring_loop_t *ringloop;
@@ -310,8 +330,30 @@ class blockstore_impl_t
 
     // Read
     int dequeue_read(blockstore_op_t *read_op);
-    int fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-        uint32_t item_state, uint64_t item_version, uint64_t item_location, uint64_t journal_sector);
+    void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
+        std::function<int(int, bool, uint32_t, uint32_t)> callback);
+    int fulfill_read(blockstore_op_t *read_op,
+        uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
+        uint32_t item_state, uint64_t item_version, uint64_t item_location,
+        uint64_t journal_sector, uint8_t *csum, int *dyn_data);
+    bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
+        uint8_t *clean_entry_bitmap, int *dyn_data,
+        uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
+    int fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
+        uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end);
+    int pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
+        uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
+        uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf);
+    bool read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
+        uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end);
+    bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc);
+    uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos);
+    bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
+        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
+    bool verify_journal_checksums(uint8_t *csums, uint32_t offset,
+        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
+    bool verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal,
+        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
     int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
         uint32_t item_state, uint64_t item_version);
     void handle_read_event(ring_data_t *data, blockstore_op_t *op);
@@ -342,6 +384,7 @@ class blockstore_impl_t
     int continue_rollback(blockstore_op_t *op);
     void mark_rolled_back(const obj_ver_id & ov);
     void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc);
+    void free_dirty_dyn_data(dirty_entry & e);
 
     // List
     void process_list(blockstore_op_t *op);

src/blockstore_init.cpp

@@ -65,7 +65,7 @@ int blockstore_init_meta::loop()
     GET_SQE();
     data->iov = { metadata_buffer, bs->dsk.meta_block_size };
     data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
-    my_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
+    my_uring_prep_readv(sqe, bs->dsk.read_meta_fd, &data->iov, 1, bs->dsk.meta_offset);
     bs->ringloop->submit();
     submitted++;
 resume_1:
@@ -77,13 +77,20 @@ resume_1:
     if (iszero((uint64_t*)metadata_buffer, bs->dsk.meta_block_size / sizeof(uint64_t)))
     {
         {
-            blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)metadata_buffer;
+            blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
             hdr->zero = 0;
             hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-            hdr->version = BLOCKSTORE_META_VERSION_V1;
+            hdr->version = bs->dsk.meta_format;
             hdr->meta_block_size = bs->dsk.meta_block_size;
             hdr->data_block_size = bs->dsk.data_block_size;
             hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
+            {
+                hdr->data_csum_type = bs->dsk.data_csum_type;
+                hdr->csum_block_size = bs->dsk.csum_block_size;
+                hdr->header_csum = 0;
+                hdr->header_csum = crc32c(0, hdr, sizeof(*hdr));
+            }
         }
         if (bs->readonly)
         {
@@ -109,28 +116,62 @@ resume_1:
     }
     else
     {
-        blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)metadata_buffer;
-        if (hdr->zero != 0 ||
-            hdr->magic != BLOCKSTORE_META_MAGIC_V1 ||
-            hdr->version != BLOCKSTORE_META_VERSION_V1)
+        blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
+        if (hdr->zero != 0 || hdr->magic != BLOCKSTORE_META_MAGIC_V1 || hdr->version < BLOCKSTORE_META_FORMAT_V1)
         {
             printf(
-                "Metadata is corrupt or old version.\n"
-                " If this is a new OSD please zero out the metadata area before starting it.\n"
-                " If you need to upgrade from 0.5.x please request it via the issue tracker.\n"
+                "Metadata is corrupt or too old (pre-0.6.x).\n"
+                " If this is a new OSD, please zero out the metadata area before starting it.\n"
+                " If you need to upgrade from 0.5.x, convert metadata with vitastor-disk.\n"
+            );
+            exit(1);
+        }
+        if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
+        {
+            uint32_t csum = hdr->header_csum;
+            hdr->header_csum = 0;
+            if (crc32c(0, hdr, sizeof(*hdr)) != csum)
+            {
+                printf("Metadata header is corrupt (checksum mismatch).\n");
+                exit(1);
+            }
+            hdr->header_csum = csum;
+            bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V2;
+        }
+        else if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
+        {
+            hdr->data_csum_type = 0;
+            hdr->csum_block_size = 0;
+            hdr->header_csum = 0;
+            // Enable compatibility mode - entries without checksums
+            bs->dsk.clean_entry_size = sizeof(clean_disk_entry) + bs->dsk.clean_entry_bitmap_size*2;
+            bs->dsk.meta_len = (1 + (bs->dsk.block_count - 1 + bs->dsk.meta_block_size / bs->dsk.clean_entry_size)
+                / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size;
+            bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V1;
+            printf("Warning: Starting with metadata in the old format without checksums, as stored on disk\n");
+        }
+        else if (hdr->version > BLOCKSTORE_META_FORMAT_V2)
+        {
+            printf(
+                "Metadata format is too new for me (stored version is %lu, max supported %u).\n",
+                hdr->version, BLOCKSTORE_META_FORMAT_V2
             );
             exit(1);
         }
         if (hdr->meta_block_size != bs->dsk.meta_block_size ||
             hdr->data_block_size != bs->dsk.data_block_size ||
-            hdr->bitmap_granularity != bs->dsk.bitmap_granularity)
+            hdr->bitmap_granularity != bs->dsk.bitmap_granularity ||
+            hdr->data_csum_type != bs->dsk.data_csum_type ||
+            hdr->csum_block_size != bs->dsk.csum_block_size)
         {
             printf(
                 "Configuration stored in metadata superblock"
-                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u)"
-                " differs from OSD configuration (%lu/%u/%lu).\n",
+                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
+                " differs from OSD configuration (%lu/%u/%lu, %u/%u).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
-                bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity
+                hdr->data_csum_type, hdr->csum_block_size,
+                bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
+                bs->dsk.data_csum_type, bs->dsk.csum_block_size
             );
             exit(1);
         }
@@ -161,7 +202,7 @@ resume_2:
                 data->iov = { bufs[i].buf, bufs[i].size };
                 data->callback = [this, i](ring_data_t *data) { handle_event(data, i); };
                 if (!zero_on_init)
-                    my_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
+                    my_uring_prep_readv(sqe, bs->dsk.read_meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 else
                 {
                     // Fill metadata with zeroes
@@ -218,7 +259,7 @@ resume_2:
             GET_SQE();
             data->iov = { metadata_buffer, bs->dsk.meta_block_size };
             data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
-            my_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
+            my_uring_prep_readv(sqe, bs->dsk.read_meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
             submitted++;
 resume_5:
             if (submitted > 0)
@@ -279,12 +320,22 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
     for (uint64_t i = 0; i < max_i; i++)
     {
         clean_disk_entry *entry = (clean_disk_entry*)(buf + i*bs->dsk.clean_entry_size);
-        if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size)
-        {
-            memcpy(bs->clean_bitmap + (done_cnt+i)*2*bs->dsk.clean_entry_bitmap_size, &entry->bitmap, 2*bs->dsk.clean_entry_bitmap_size);
-        }
         if (entry->oid.inode > 0)
         {
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
+            {
+                // Check entry crc32
+                uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + bs->dsk.clean_entry_size - 4);
+                if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4))
+                {
+                    printf("Metadata entry %lu is corrupt (checksum mismatch), skipping\n", done_cnt+i);
+                    continue;
+                }
+            }
+            if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size)
+            {
+                memcpy(bs->clean_bitmaps + (done_cnt+i) * 2 * bs->dsk.clean_entry_bitmap_size, &entry->bitmap, 2 * bs->dsk.clean_entry_bitmap_size);
+            }
             auto & clean_db = bs->clean_db_shard(entry->oid);
             auto clean_it = clean_db.find(entry->oid);
             if (clean_it == clean_db.end() || clean_it->second.version < entry->version)
@@ -416,7 +467,7 @@ int blockstore_init_journal::loop()
     data = ((ring_data_t*)sqe->user_data);
     data->iov = { submitted_buf, bs->journal.block_size };
     data->callback = simple_callback;
-    my_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset);
+    my_uring_prep_readv(sqe, bs->dsk.read_journal_fd, &data->iov, 1, bs->journal.offset);
     bs->ringloop->submit();
     wait_count = 1;
 resume_1:
@@ -440,7 +491,9 @@ resume_1:
             .size = sizeof(journal_entry_start),
             .reserved = 0,
             .journal_start = bs->journal.block_size,
-            .version = JOURNAL_VERSION,
+            .version = JOURNAL_VERSION_V2,
+            .data_csum_type = bs->dsk.data_csum_type,
+            .csum_block_size = bs->dsk.csum_block_size,
         };
         ((journal_entry_start*)submitted_buf)->crc32 = je_crc32((journal_entry*)submitted_buf);
         if (bs->readonly)
@@ -492,18 +545,36 @@ resume_1:
         if (je_start->magic != JOURNAL_MAGIC ||
             je_start->type != JE_START ||
             je_crc32((journal_entry*)je_start) != je_start->crc32 ||
-            je_start->size != sizeof(journal_entry_start) && je_start->size != JE_START_LEGACY_SIZE)
+            je_start->size != JE_START_V0_SIZE && je_start->size != JE_START_V1_SIZE && je_start->size != JE_START_V2_SIZE)
         {
             // Entry is corrupt
-            fprintf(stderr, "First entry of the journal is corrupt\n");
+            fprintf(stderr, "First entry of the journal is corrupt or unsupported\n");
             exit(1);
         }
-        if (je_start->size == JE_START_LEGACY_SIZE || je_start->version != JOURNAL_VERSION)
+        if (je_start->size == JE_START_V0_SIZE ||
+            (je_start->version != JOURNAL_VERSION_V1 || je_start->size != JE_START_V1_SIZE) &&
+            (je_start->version != JOURNAL_VERSION_V2 || je_start->size != JE_START_V2_SIZE))
         {
             fprintf(
-                stderr, "The code only supports journal version %d, but it is %lu on disk."
-                    " Please use the previous version to flush the journal before upgrading OSD\n",
-                JOURNAL_VERSION, je_start->size == JE_START_LEGACY_SIZE ? 0 : je_start->version
+                stderr, "The code only supports journal versions 2 and 1, but it is %lu on disk."
+                    " Please use vitastor-disk to rewrite the journal\n",
+                je_start->size == JE_START_V0_SIZE ? 0 : je_start->version
+            );
+            exit(1);
+        }
+        if (je_start->version == JOURNAL_VERSION_V1)
+        {
+            je_start->data_csum_type = 0;
+            je_start->csum_block_size = 0;
+        }
+        if (je_start->data_csum_type != bs->dsk.data_csum_type ||
+            je_start->csum_block_size != bs->dsk.csum_block_size)
+        {
+            printf(
+                "Configuration stored in journal superblock (data_csum_type=%u, csum_block_size=%u)"
+                " differs from OSD configuration (%u/%u).\n",
+                je_start->data_csum_type, je_start->csum_block_size,
+                bs->dsk.data_csum_type, bs->dsk.csum_block_size
             );
             exit(1);
         }
@@ -536,7 +607,7 @@ resume_1:
                     end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE,
                 };
                 data->callback = [this](ring_data_t *data1) { handle_event(data1); };
-                my_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + journal_pos);
+                my_uring_prep_readv(sqe, bs->dsk.read_journal_fd, &data->iov, 1, bs->journal.offset + journal_pos);
                 bs->ringloop->submit();
             }
             while (done.size() > 0)
@@ -705,11 +776,14 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     snprintf(err, 1024, "BUG: calculated journal data offset (%08lx) != stored journal data offset (%08lx)", location, je->small_write.data_offset);
                     throw std::runtime_error(err);
                 }
-                uint32_t data_crc32 = 0;
+                small_write_data.clear();
                 if (location >= done_pos && location+je->small_write.len <= done_pos+len)
                 {
                     // data is within this buffer
-                    data_crc32 = crc32c(0, (uint8_t*)buf + location - done_pos, je->small_write.len);
+                    small_write_data.push_back((iovec){
+                        .iov_base = (uint8_t*)buf + location - done_pos,
+                        .iov_len = je->small_write.len,
+                    });
                 }
                 else
                 {
@@ -724,7 +798,10 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                                 ? location+je->small_write.len : done[i].pos+done[i].len);
                             uint64_t part_begin = (location < done[i].pos ? done[i].pos : location);
                             covered += part_end - part_begin;
-                            data_crc32 = crc32c(data_crc32, (uint8_t*)done[i].buf + part_begin - done[i].pos, part_end - part_begin);
+                            small_write_data.push_back((iovec){
+                                .iov_base = (uint8_t*)done[i].buf + part_begin - done[i].pos,
+                                .iov_len = part_end - part_begin,
+                            });
                         }
                     }
                     if (covered < je->small_write.len)
@@ -734,12 +811,102 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         return 2;
                     }
                 }
-                if (data_crc32 != je->small_write.crc32_data)
+                bool data_csum_valid = true;
+                if (!bs->dsk.csum_block_size)
+                {
+                    uint32_t data_crc32 = 0;
+                    for (auto & sd: small_write_data)
+                    {
+                        data_crc32 = crc32c(data_crc32, sd.iov_base, sd.iov_len);
+                    }
+                    data_csum_valid = data_crc32 == je->small_write.crc32_data;
+                    if (!data_csum_valid)
+                    {
+                        printf(
+                            "Journal entry data is corrupt for small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u - data crc32 %x != %x\n",
+                            je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
+                            je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version,
+                            je->small_write.offset, je->small_write.len,
+                            data_crc32, je->small_write.crc32_data
+                        );
+                    }
+                }
+                else if (je->small_write.len > 0)
+                {
+                    // FIXME: deduplicate with disk_tool_journal.cpp
+                    // like in enqueue_write()
+                    uint32_t start = je->small_write.offset / bs->dsk.csum_block_size;
+                    uint32_t end = (je->small_write.offset+je->small_write.len-1) / bs->dsk.csum_block_size;
+                    uint32_t data_csum_size = (end-start+1) * (bs->dsk.data_csum_type & 0xFF);
+                    uint32_t required_size = sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size + data_csum_size;
+                    if (je->size != required_size)
+                    {
+                        printf(
+                            "Journal entry data has invalid size for small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u - should be %u bytes but is %u bytes\n",
+                            je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
+                            je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version,
+                            je->small_write.offset, je->small_write.len,
+                            required_size, je->size
+                        );
+                        data_csum_valid = false;
+                    }
+                    else
+                    {
+                        int sd_num = 0;
+                        size_t sd_pos = 0;
+                        uint32_t *block_csums = (uint32_t*)((uint8_t*)je + sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size);
+                        for (uint32_t pos = start; pos <= end; pos++, block_csums++)
+                        {
+                            size_t block_left = (pos == start
+                                ? (start == end
+                                    ? je->small_write.len
+                                    : bs->dsk.csum_block_size - je->small_write.offset%bs->dsk.csum_block_size)
+                                : (pos < end
+                                    ? bs->dsk.csum_block_size
+                                    : (je->small_write.offset + je->small_write.len)%bs->dsk.csum_block_size));
+                            if (pos > start && pos == end && block_left == 0)
+                            {
+                                // full last block
+                                block_left = bs->dsk.csum_block_size;
+                            }
+                            uint32_t block_crc32 = 0;
+                            while (block_left > 0)
+                            {
+                                assert(sd_num < small_write_data.size());
+                                if (small_write_data[sd_num].iov_len >= sd_pos+block_left)
+                                {
+                                    block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, block_left);
+                                    sd_pos += block_left;
+                                    break;
+                                }
+                                else
+                                {
+                                    block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, small_write_data[sd_num].iov_len-sd_pos);
+                                    block_left -= (small_write_data[sd_num].iov_len-sd_pos);
+                                    sd_pos = 0;
+                                    sd_num++;
+                                }
+                            }
+                            if (block_crc32 != *block_csums)
+                            {
+                                printf(
+                                    "Journal entry data is corrupt for small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u - block %u crc32 %x != %x\n",
+                                    je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
+                                    je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version,
+                                    je->small_write.offset, je->small_write.len,
+                                    pos, block_crc32, *block_csums
+                                );
+                                data_csum_valid = false;
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (!data_csum_valid)
                 {
                     // journal entry is corrupt, stop here
                     // interesting thing is that we must clear the corrupt entry if we're not readonly,
                     // because we don't write next entries in the same journal block
-                    printf("Journal entry data is corrupt (data crc32 %x != %x)\n", data_crc32, je->small_write.crc32_data);
                     memset((uint8_t*)buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos);
                     bs->journal.next_free = prev_free;
                     init_write_buf = (uint8_t*)buf + proc_pos - done_pos;
@@ -755,11 +922,14 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .oid = je->small_write.oid,
                         .version = je->small_write.version,
                     };
-                    void *bmp = NULL;
-                    void *bmp_from = (uint8_t*)je + sizeof(journal_entry_small_write);
-                    if (bs->dsk.clean_entry_bitmap_size <= sizeof(void*))
+                    uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->small_write.offset, je->small_write.len);
+                    void *dyn = NULL;
+                    void *dyn_from = (uint8_t*)je + sizeof(journal_entry_small_write);
+                    if (!bs->alloc_dyn_data)
                     {
-                        memcpy(&bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
+                        // Bitmap without checksum is only 4 bytes for 128k objects, save it inline
+                        // It can even contain 4 byte bitmap + 4 byte CRC32 for 4 kb writes :)
+                        memcpy(&dyn, dyn_from, dyn_size);
                     }
                     else
                     {
@@ -767,8 +937,9 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         // allocations for entry bitmaps. This can only be fixed by using
                         // a patched map with dynamic entry size, but not the btree_map,
                         // because it doesn't keep iterators valid all the time.
-                        bmp = malloc_or_die(bs->dsk.clean_entry_bitmap_size);
-                        memcpy(bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
+                        dyn = malloc_or_die(dyn_size+sizeof(int));
+                        *((int*)dyn) = 1;
+                        memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size);
                     }
                     bs->dirty_db.emplace(ov, (dirty_entry){
                         .state = (BS_ST_SMALL_WRITE | BS_ST_SYNCED),
@@ -777,7 +948,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .offset = je->small_write.offset,
                         .len = je->small_write.len,
                         .journal_sector = proc_pos,
-                        .bitmap = bmp,
+                        .dyn_data = dyn,
                     });
                     bs->journal.used_sectors[proc_pos]++;
 #ifdef BLOCKSTORE_DEBUG
@@ -836,11 +1007,13 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .oid = je->big_write.oid,
                         .version = je->big_write.version,
                     };
-                    void *bmp = NULL;
-                    void *bmp_from = (uint8_t*)je + sizeof(journal_entry_big_write);
-                    if (bs->dsk.clean_entry_bitmap_size <= sizeof(void*))
+                    uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->big_write.offset, je->big_write.len);
+                    void *dyn = NULL;
+                    void *dyn_from = (uint8_t*)je + sizeof(journal_entry_big_write);
+                    if (!bs->alloc_dyn_data)
                     {
-                        memcpy(&bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
+                        // Bitmap without checksum is only 4 bytes for 128k objects, save it inline
+                        memcpy(&dyn, dyn_from, dyn_size);
                     }
                     else
                     {
@@ -848,8 +1021,9 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         // allocations for entry bitmaps. This can only be fixed by using
                         // a patched map with dynamic entry size, but not the btree_map,
                         // because it doesn't keep iterators valid all the time.
-                        bmp = malloc_or_die(bs->dsk.clean_entry_bitmap_size);
-                        memcpy(bmp, bmp_from, bs->dsk.clean_entry_bitmap_size);
+                        dyn = malloc_or_die(dyn_size+sizeof(int));
+                        *((int*)dyn) = 1;
+                        memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size);
                     }
                     auto dirty_it = bs->dirty_db.emplace(ov, (dirty_entry){
                         .state = (BS_ST_BIG_WRITE | BS_ST_SYNCED),
@@ -858,7 +1032,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .offset = je->big_write.offset,
                         .len = je->big_write.len,
                         .journal_sector = proc_pos,
-                        .bitmap = bmp,
+                        .dyn_data = dyn,
                     }).first;
                     if (bs->data_alloc->get(je->big_write.location >> bs->dsk.block_order))
                     {

src/blockstore_init.h

@@ -50,6 +50,7 @@ class blockstore_init_journal
     uint64_t next_free;
     std::vector<bs_init_journal_done> done;
     std::vector<obj_ver_id> double_allocs;
+    std::vector<iovec> small_write_data;
     uint64_t journal_pos = 0;
     uint64_t continue_pos = 0;
     void *init_write_buf = NULL;

src/blockstore_journal.cpp

@@ -17,6 +17,7 @@ blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
 // Check if we can write <required> entries of <size> bytes and <data_after> data bytes after them to the journal
 int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries_required, int size, int data_after)
 {
+    uint64_t prev_next = next_sector;
     int required = entries_required;
     while (1)
     {
@@ -35,11 +36,19 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
             }
             required -= fits;
             next_in_pos += fits * size;
-            sectors_to_write++;
+            if (next_sector != prev_next || !sectors_to_write)
+            {
+                // Except the previous call to this function
+                sectors_to_write++;
+            }
         }
         else if (bs->journal.sector_info[next_sector].dirty)
         {
-            sectors_to_write++;
+            if (next_sector != prev_next || !sectors_to_write)
+            {
+                // Except the previous call to this function
+                sectors_to_write++;
+            }
         }
         if (required <= 0)
         {
@@ -289,3 +298,31 @@ void journal_t::dump_diagnostics()
         journal_used_it == used_sectors.end() ? 0 : journal_used_it->second
     );
 }
+
+static uint64_t zero_page[4096];
+
+uint32_t crc32c_pad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad)
+{
+    uint32_t r = prev_crc;
+    while (left_pad >= 4096)
+    {
+        r = crc32c(r, zero_page, 4096);
+        left_pad -= 4096;
+    }
+    if (left_pad > 0)
+        r = crc32c(r, zero_page, left_pad);
+    r = crc32c(r, buf, len);
+    while (right_pad >= 4096)
+    {
+        r = crc32c(r, zero_page, 4096);
+        right_pad -= 4096;
+    }
+    if (left_pad > 0)
+        r = crc32c(r, zero_page, right_pad);
+    return r;
+}
+
+uint32_t crc32c_nopad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad)
+{
+    return crc32c(0, buf, len);
+}

src/blockstore_journal.h

@@ -8,7 +8,8 @@
 
 #define MIN_JOURNAL_SIZE 4*1024*1024
 #define JOURNAL_MAGIC 0x4A33
-#define JOURNAL_VERSION 1
+#define JOURNAL_VERSION_V1 1
+#define JOURNAL_VERSION_V2 2
 #define JOURNAL_BUFFER_SIZE 4*1024*1024
 #define JOURNAL_ENTRY_HEADER_SIZE 16
 
@@ -32,7 +33,7 @@
 #define JE_BIG_WRITE_INSTANT   0x08
 #define JE_MAX         0x08
 
-// crc32c comes first to ease calculation and is equal to crc32()
+// crc32c comes first to ease calculation
 struct __attribute__((__packed__)) journal_entry_start
 {
     uint32_t crc32;
@@ -42,8 +43,12 @@ struct __attribute__((__packed__)) journal_entry_start
     uint32_t reserved;
     uint64_t journal_start;
     uint64_t version;
+    uint32_t data_csum_type;
+    uint32_t csum_block_size;
 };
-#define JE_START_LEGACY_SIZE 24
+#define JE_START_V0_SIZE 24
+#define JE_START_V1_SIZE 32
+#define JE_START_V2_SIZE 40
 
 struct __attribute__((__packed__)) journal_entry_small_write
 {
@@ -59,10 +64,12 @@ struct __attribute__((__packed__)) journal_entry_small_write
     // small_write entries contain <len> bytes of data which is stored in next sectors
     // data_offset is its offset within journal
     uint64_t data_offset;
-    uint32_t crc32_data;
+    uint32_t crc32_data; // zero when data_csum_type != 0
     // small_write and big_write entries are followed by the "external" bitmap
     // its size is dynamic and included in journal entry's <size> field
     uint8_t bitmap[];
+    // and then data checksums if data_csum_type != 0
+    // uint32_t data_crc32c[];
 };
 
 struct __attribute__((__packed__)) journal_entry_big_write
@@ -80,6 +87,8 @@ struct __attribute__((__packed__)) journal_entry_big_write
     // small_write and big_write entries are followed by the "external" bitmap
     // its size is dynamic and included in journal entry's <size> field
     uint8_t bitmap[];
+    // and then data checksums if data_csum_type != 0
+    // uint32_t data_crc32c[];
 };
 
 struct __attribute__((__packed__)) journal_entry_stable
@@ -218,3 +227,6 @@ struct blockstore_journal_check_t
 };
 
 journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size);
+
+uint32_t crc32c_pad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad);
+uint32_t crc32c_nopad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad);

src/blockstore_open.cpp

@@ -133,19 +133,24 @@ void blockstore_impl_t::calc_lengths()
     {
         metadata_buffer = memalign(MEM_ALIGNMENT, dsk.meta_len);
         if (!metadata_buffer)
-            throw std::runtime_error("Failed to allocate memory for the metadata");
+            throw std::runtime_error("Failed to allocate memory for the metadata ("+std::to_string(dsk.meta_len/1024/1024)+" MB)");
     }
-    else if (dsk.clean_entry_bitmap_size)
+    else if (dsk.clean_entry_bitmap_size || dsk.data_csum_type)
     {
-        clean_bitmap = (uint8_t*)malloc(dsk.block_count * 2*dsk.clean_entry_bitmap_size);
-        if (!clean_bitmap)
-            throw std::runtime_error("Failed to allocate memory for the metadata sparse write bitmap");
+        clean_bitmaps = (uint8_t*)malloc(dsk.block_count * 2 * dsk.clean_entry_bitmap_size);
+        if (!clean_bitmaps)
+        {
+            throw std::runtime_error(
+                "Failed to allocate memory for the metadata sparse write bitmap ("+
+                std::to_string(dsk.block_count * 2 * dsk.clean_entry_bitmap_size / 1024 / 1024)+" MB)"
+            );
+        }
     }
     if (journal.inmemory)
     {
         journal.buffer = memalign(MEM_ALIGNMENT, journal.len);
         if (!journal.buffer)
-            throw std::runtime_error("Failed to allocate memory for journal");
+            throw std::runtime_error("Failed to allocate memory for journal ("+std::to_string(journal.len/1024/1024)+" MB)");
     }
     else
     {

src/blockstore_read.cpp

@@ -1,6 +1,7 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
+#include <limits.h>
 #include "blockstore_impl.h"
 
 int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
@@ -8,12 +9,7 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
 {
     if (!len)
     {
-        // Zero-length version - skip
-        return 1;
-    }
-    else if (IS_IN_FLIGHT(item_state))
-    {
-        // Write not finished yet - skip
+        // Zero-length read
         return 1;
     }
     else if (IS_DELETE(item_state))
@@ -22,6 +18,7 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
         memset(buf, 0, len);
         return 1;
     }
+    assert(!IS_IN_FLIGHT(item_state));
     if (journal.inmemory && IS_JOURNAL(item_state))
     {
         memcpy(buf, (uint8_t*)journal.buffer + offset, len);
@@ -32,7 +29,7 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
     PRIV(op)->pending_ops++;
     my_uring_prep_readv(
         sqe,
-        IS_JOURNAL(item_state) ? dsk.journal_fd : dsk.data_fd,
+        IS_JOURNAL(item_state) ? dsk.read_journal_fd : dsk.read_data_fd,
         &data->iov, 1,
         (IS_JOURNAL(item_state) ? dsk.journal_offset : dsk.data_offset) + offset
     );
@@ -40,59 +37,115 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
     return 1;
 }
 
-// FIXME I've seen a bug here so I want some tests
-int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-    uint32_t item_state, uint64_t item_version, uint64_t item_location, uint64_t journal_sector)
+void blockstore_impl_t::find_holes(std::vector<copy_buffer_t> & read_vec,
+    uint32_t item_start, uint32_t item_end,
+    std::function<int(int, bool, uint32_t, uint32_t)> callback)
 {
-    uint32_t cur_start = item_start;
-    if (cur_start < read_op->offset + read_op->len && item_end > read_op->offset)
+    auto cur_start = item_start;
+    int i = 0;
+    while (cur_start < item_end)
     {
-        cur_start = cur_start < read_op->offset ? read_op->offset : cur_start;
-        item_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end;
-        auto it = PRIV(read_op)->read_vec.begin();
-        while (1)
+        // COPY_BUF_CSUM_FILL items are fake items inserted in the end, their offsets aren't in order
+        if (i >= read_vec.size() || read_vec[i].copy_flags & COPY_BUF_CSUM_FILL || read_vec[i].offset >= item_end)
         {
-            for (; it != PRIV(read_op)->read_vec.end(); it++)
-            {
-                if (it->offset >= cur_start)
-                {
-                    break;
-                }
-                else if (it->offset + it->len > cur_start)
-                {
-                    cur_start = it->offset + it->len;
-                    if (cur_start >= item_end)
-                    {
-                        goto endwhile;
-                    }
-                }
-            }
-            if (it == PRIV(read_op)->read_vec.end() || it->offset > cur_start)
-            {
-                fulfill_read_t el = {
-                    .offset = cur_start,
-                    .len = it == PRIV(read_op)->read_vec.end() || it->offset >= item_end ? item_end-cur_start : it->offset-cur_start,
-                    .journal_sector = journal_sector,
-                };
-                it = PRIV(read_op)->read_vec.insert(it, el);
-                if (!fulfill_read_push(read_op,
-                    (uint8_t*)read_op->buf + el.offset - read_op->offset,
-                    item_location + el.offset - item_start,
-                    el.len, item_state, item_version))
-                {
-                    return 0;
-                }
-                fulfilled += el.len;
-            }
-            cur_start = it->offset + it->len;
-            if (it == PRIV(read_op)->read_vec.end() || cur_start >= item_end)
-            {
-                break;
-            }
+            // Hole (at end): cur_start .. item_end
+            i += callback(i, false, cur_start, item_end);
+            break;
         }
+        else if (read_vec[i].offset > cur_start)
+        {
+            // Hole: cur_start .. min(read_vec[i].offset, item_end)
+            auto cur_end = read_vec[i].offset > item_end ? item_end : read_vec[i].offset;
+            i += callback(i, false, cur_start, cur_end);
+            cur_start = cur_end;
+        }
+        else if (read_vec[i].offset + read_vec[i].len > cur_start)
+        {
+            // Allocated: cur_start .. min(read_vec[i].offset + read_vec[i].len, item_end)
+            auto cur_end = read_vec[i].offset + read_vec[i].len;
+            cur_end = cur_end > item_end ? item_end : cur_end;
+            i += callback(i, true, cur_start, cur_end);
+            cur_start = cur_end;
+            i++;
+        }
+        else
+            i++;
     }
-endwhile:
-    return 1;
+}
+
+int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op,
+    uint64_t &fulfilled, uint32_t item_start, uint32_t item_end, // FIXME: Rename item_* to dirty_*
+    uint32_t item_state, uint64_t item_version, uint64_t item_location,
+    uint64_t journal_sector, uint8_t *csum, int *dyn_data)
+{
+    int r = 1;
+    if (item_start < read_op->offset + read_op->len && item_end > read_op->offset)
+    {
+        auto & rv = PRIV(read_op)->read_vec;
+        auto rd_start = item_start < read_op->offset ? read_op->offset : item_start;
+        auto rd_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end;
+        find_holes(rv, rd_start, rd_end, [&](int pos, bool alloc, uint32_t start, uint32_t end)
+        {
+            if (!r || alloc)
+                return 0;
+            if (!journal.inmemory && dsk.csum_block_size > dsk.bitmap_granularity && IS_JOURNAL(item_state) && !IS_DELETE(item_state))
+            {
+                uint32_t blk_begin = (start/dsk.csum_block_size) * dsk.csum_block_size;
+                blk_begin = blk_begin < item_start ? item_start : blk_begin;
+                uint32_t blk_end = ((end-1) / dsk.csum_block_size + 1) * dsk.csum_block_size;
+                blk_end = blk_end > item_end ? item_end : blk_end;
+                rv.push_back((copy_buffer_t){
+                    .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL,
+                    .offset = blk_begin,
+                    .len = blk_end-blk_begin,
+                    .csum_buf = (csum + (blk_begin/dsk.csum_block_size -
+                        item_start/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)),
+                    .dyn_data = dyn_data,
+                });
+                if (dyn_data)
+                {
+                    (*dyn_data)++;
+                }
+                // Submit the journal checksum block read
+                if (!read_checksum_block(read_op, 1, fulfilled, item_location - item_start))
+                {
+                    r = 0;
+                }
+                return 0;
+            }
+            copy_buffer_t el = {
+                .copy_flags = (IS_JOURNAL(item_state) ? COPY_BUF_JOURNAL : COPY_BUF_DATA),
+                .offset = start,
+                .len = end-start,
+                .disk_offset = item_location + start - item_start,
+                .journal_sector = (IS_JOURNAL(item_state) ? journal_sector : 0),
+                .csum_buf = !csum ? NULL : (csum + (start - item_start) / dsk.csum_block_size * (dsk.data_csum_type & 0xFF)),
+                .dyn_data = dyn_data,
+            };
+            if (dyn_data)
+            {
+                (*dyn_data)++;
+            }
+            if (IS_BIG_WRITE(item_state))
+            {
+                // If we don't track it then we may IN THEORY read another object's data:
+                // submit read -> remove the object -> flush remove -> overwrite with another object -> finish read
+                // Very improbable, but possible
+                PRIV(read_op)->clean_block_used = 1;
+            }
+            rv.insert(rv.begin() + pos, el);
+            fulfilled += el.len;
+            if (!fulfill_read_push(read_op,
+                (uint8_t*)read_op->buf + el.offset - read_op->offset,
+                item_location + el.offset - item_start,
+                el.len, item_state, item_version))
+            {
+                r = 0;
+            }
+            return 1;
+        });
+    }
+    return r;
 }
 
 uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset)
@@ -106,10 +159,225 @@ uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offse
         clean_entry_bitmap = ((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size + sizeof(clean_disk_entry) + offset);
     }
     else
-        clean_entry_bitmap = (uint8_t*)(clean_bitmap + meta_loc*2*dsk.clean_entry_bitmap_size + offset);
+        clean_entry_bitmap = (uint8_t*)(clean_bitmaps + meta_loc*2*dsk.clean_entry_bitmap_size + offset);
     return clean_entry_bitmap;
 }
 
+int blockstore_impl_t::fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
+    uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end)
+{
+    if (read_end == read_offset)
+        return 0;
+    int required = 0;
+    read_buf -= read_offset;
+    uint32_t last_block = (read_end-1)/dsk.csum_block_size;
+    uint32_t start_block = read_offset/dsk.csum_block_size;
+    uint32_t end_block = 0;
+    while (start_block <= last_block)
+    {
+        if (read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap,
+            start_block*dsk.csum_block_size < read_offset ? read_offset : start_block*dsk.csum_block_size,
+            (start_block+1)*dsk.csum_block_size > read_end ? read_end : (start_block+1)*dsk.csum_block_size))
+        {
+            // read_range_fulfilled() also adds zero-filled areas
+            start_block++;
+        }
+        else
+        {
+            // Find a sequence of checksum blocks required to be read
+            end_block = start_block;
+            while ((end_block+1)*dsk.csum_block_size < read_end &&
+                !read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap,
+                    (end_block+1)*dsk.csum_block_size < read_offset ? read_offset : (end_block+1)*dsk.csum_block_size,
+                    (end_block+2)*dsk.csum_block_size > read_end ? read_end : (end_block+2)*dsk.csum_block_size))
+            {
+                end_block++;
+            }
+            end_block++;
+            // OK, mark this range as required
+            rv.push_back((copy_buffer_t){
+                .copy_flags = COPY_BUF_CSUM_FILL | (from_journal ? COPY_BUF_JOURNALED_BIG : 0),
+                .offset = start_block*dsk.csum_block_size,
+                .len = (end_block-start_block)*dsk.csum_block_size,
+                // save clean_entry_bitmap if we're reading clean data from the journal
+                .csum_buf = from_journal ? clean_entry_bitmap : NULL,
+                .dyn_data = dyn_data,
+            });
+            if (dyn_data)
+            {
+                (*dyn_data)++;
+            }
+            start_block = end_block;
+            required++;
+        }
+    }
+    return required;
+}
+
+// read_buf should be == op->buf - op->offset
+bool blockstore_impl_t::read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
+    uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end)
+{
+    bool all_done = true;
+    find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
+    {
+        if (alloc)
+            return 0;
+        int diff = 0;
+        uint32_t bmp_start = cur_start/dsk.bitmap_granularity;
+        uint32_t bmp_end = cur_end/dsk.bitmap_granularity;
+        uint32_t bmp_pos = bmp_start;
+        while (bmp_pos < bmp_end)
+        {
+            while (bmp_pos < bmp_end && !(clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7))))
+                bmp_pos++;
+            if (bmp_pos > bmp_start)
+            {
+                // zero fill
+                copy_buffer_t el = {
+                    .copy_flags = COPY_BUF_ZERO,
+                    .offset = bmp_start*dsk.bitmap_granularity,
+                    .len = (bmp_pos-bmp_start)*dsk.bitmap_granularity,
+                };
+                rv.insert(rv.begin() + pos, el);
+                if (read_buf)
+                    memset(read_buf + el.offset, 0, el.len);
+                fulfilled += el.len;
+                diff++;
+            }
+            bmp_start = bmp_pos;
+            while (bmp_pos < bmp_end && (clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7))))
+                bmp_pos++;
+            if (bmp_pos > bmp_start)
+            {
+                // something is to be read
+                all_done = false;
+            }
+            bmp_start = bmp_pos;
+        }
+        return diff;
+    });
+    return all_done;
+}
+
+bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc)
+{
+    auto & rv = PRIV(op)->read_vec;
+    auto *vi = &rv[rv.size()-rv_pos];
+    uint32_t item_start = vi->offset, item_end = vi->offset+vi->len;
+    uint32_t fill_size = 0;
+    int n_iov = 0;
+    find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
+    {
+        if (alloc)
+        {
+            fill_size += cur_end-cur_start;
+            n_iov++;
+        }
+        else
+        {
+            if (cur_start < op->offset)
+            {
+                fill_size += op->offset-cur_start;
+                n_iov++;
+                cur_start = op->offset;
+            }
+            if (cur_end > op->offset+op->len)
+            {
+                fill_size += cur_end-(op->offset+op->len);
+                n_iov++;
+                cur_end = op->offset+op->len;
+            }
+            if (cur_end > cur_start)
+            {
+                n_iov++;
+            }
+        }
+        return 0;
+    });
+    void *buf = memalign_or_die(MEM_ALIGNMENT, fill_size + n_iov*sizeof(struct iovec));
+    iovec *iov = (struct iovec*)((uint8_t*)buf+fill_size);
+    n_iov = 0;
+    fill_size = 0;
+    find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
+    {
+        int res = 0;
+        if (alloc)
+        {
+            iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end-cur_start };
+            fill_size += cur_end-cur_start;
+        }
+        else
+        {
+            if (cur_start < op->offset)
+            {
+                iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, op->offset-cur_start };
+                fill_size += op->offset-cur_start;
+                cur_start = op->offset;
+            }
+            auto lim_end = cur_end > op->offset+op->len ? op->offset+op->len : cur_end;
+            if (lim_end > cur_start)
+            {
+                iov[n_iov++] = (struct iovec){ (uint8_t*)op->buf+cur_start-op->offset, lim_end-cur_start };
+                rv.insert(rv.begin() + pos, (copy_buffer_t){
+                    .copy_flags = COPY_BUF_DATA,
+                    .offset = cur_start,
+                    .len = lim_end-cur_start,
+                });
+                fulfilled += lim_end-cur_start;
+                res++;
+            }
+            if (cur_end > op->offset+op->len)
+            {
+                iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end - (op->offset+op->len) };
+                fill_size += cur_end - (op->offset+op->len);
+                cur_end = op->offset+op->len;
+            }
+        }
+        return res;
+    });
+    vi = &rv[rv.size()-rv_pos];
+    // Save buf into read_vec too but in a creepy way
+    // FIXME: Shit, something else should be invented %)
+    *vi = (copy_buffer_t){
+        .copy_flags = vi->copy_flags,
+        .offset = vi->offset,
+        .len = ((uint64_t)n_iov << 32) | fill_size,
+        .disk_offset = clean_loc + item_start,
+        .buf = (uint8_t*)buf,
+        .csum_buf = vi->csum_buf,
+        .dyn_data = vi->dyn_data,
+    };
+    int submit_fd = (vi->copy_flags & COPY_BUF_JOURNAL ? dsk.read_journal_fd : dsk.read_data_fd);
+    uint64_t submit_offset = (vi->copy_flags & COPY_BUF_JOURNAL ? journal.offset : dsk.data_offset);
+    uint32_t d_pos = 0;
+    for (int n_pos = 0; n_pos < n_iov; n_pos += IOV_MAX)
+    {
+        int n_cur = n_iov-n_pos < IOV_MAX ? n_iov-n_pos : IOV_MAX;
+        BS_SUBMIT_GET_SQE(sqe, data);
+        PRIV(op)->pending_ops++;
+        my_uring_prep_readv(sqe, submit_fd, iov + n_pos, n_cur, submit_offset + clean_loc + item_start + d_pos);
+        data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
+        if (n_pos > 0 || n_pos + IOV_MAX < n_iov)
+        {
+            uint32_t d_len = 0;
+            for (int i = 0; i < IOV_MAX; i++)
+                d_len += iov[n_pos+i].iov_len;
+            data->iov.iov_len = d_len;
+            d_pos += d_len;
+        }
+        else
+            data->iov.iov_len = item_end-item_start;
+    }
+    if (!(vi->copy_flags & COPY_BUF_JOURNAL))
+    {
+        // Reads running parallel to flushes of the same clean block may read
+        // a mixture of old and new data. So we don't verify checksums for such blocks.
+        PRIV(op)->clean_block_used = 1;
+    }
+    return true;
+}
+
 int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
 {
     auto & clean_db = clean_db_shard(read_op->oid);
@@ -131,6 +399,8 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     }
     uint64_t fulfilled = 0;
     PRIV(read_op)->pending_ops = 0;
+    PRIV(read_op)->clean_block_used = 0;
+    auto & rv = PRIV(read_op)->read_vec;
     uint64_t result_version = 0;
     if (dirty_found)
     {
@@ -148,23 +418,36 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
                     FINISH_OP(read_op);
                     return 2;
                 }
+                int *dyn_data = (int*)(dsk.csum_block_size > 0 && alloc_dyn_data ? dirty.dyn_data : NULL);
+                uint8_t *bmp_ptr = (alloc_dyn_data
+                    ? (uint8_t*)dirty.dyn_data + sizeof(int) : (uint8_t*)&dirty.dyn_data);
                 if (!result_version)
                 {
                     result_version = dirty_it->first.version;
                     if (read_op->bitmap)
                     {
-                        void *bmp_ptr = (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap);
                         memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
                     }
                 }
                 // If inmemory_journal is false, journal trim will have to wait until the read is completed
-                if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
-                    dirty.state, dirty_it->first.version, dirty.location + (IS_JOURNAL(dirty.state) ? 0 : dirty.offset),
-                    (IS_JOURNAL(dirty.state) ? dirty.journal_sector+1 : 0)))
+                if (!IS_JOURNAL(dirty.state))
                 {
-                    // need to wait. undo added requests, don't dequeue op
-                    PRIV(read_op)->read_vec.clear();
-                    return 0;
+                    // Read from data disk, possibly checking checksums
+                    if (!fulfill_clean_read(read_op, fulfilled, bmp_ptr, dyn_data,
+                        dirty.offset, dirty.offset+dirty.len, dirty.location, dirty_it->first.version))
+                    {
+                        goto undo_read;
+                    }
+                }
+                else
+                {
+                    // Copy from memory or read from journal, possibly checking checksums
+                    if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
+                        dirty.state, dirty_it->first.version, dirty.location, dirty.journal_sector+1,
+                        journal.inmemory ? NULL : bmp_ptr+dsk.clean_entry_bitmap_size, dyn_data))
+                    {
+                        goto undo_read;
+                    }
                 }
             }
             if (fulfilled == read_op->len || dirty_it == dirty_db.begin())
@@ -187,50 +470,10 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         }
         if (fulfilled < read_op->len)
         {
-            if (!dsk.clean_entry_bitmap_size)
+            if (!fulfill_clean_read(read_op, fulfilled, NULL, NULL, 0, dsk.data_block_size,
+                clean_it->second.location, clean_it->second.version))
             {
-                if (!fulfill_read(read_op, fulfilled, 0, dsk.data_block_size,
-                    (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_it->second.location, 0))
-                {
-                    // need to wait. undo added requests, don't dequeue op
-                    PRIV(read_op)->read_vec.clear();
-                    return 0;
-                }
-            }
-            else
-            {
-                uint8_t *clean_entry_bitmap = get_clean_entry_bitmap(clean_it->second.location, 0);
-                uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity;
-                while (bmp_start < bmp_size)
-                {
-                    while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size)
-                    {
-                        bmp_end++;
-                    }
-                    if (bmp_end > bmp_start)
-                    {
-                        // fill with zeroes
-                        assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
-                            bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0));
-                    }
-                    bmp_start = bmp_end;
-                    while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
-                    {
-                        bmp_end++;
-                    }
-                    if (bmp_end > bmp_start)
-                    {
-                        if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
-                            bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0,
-                            clean_it->second.location + bmp_start * dsk.bitmap_granularity, 0))
-                        {
-                            // need to wait. undo added requests, don't dequeue op
-                            PRIV(read_op)->read_vec.clear();
-                            return 0;
-                        }
-                        bmp_start = bmp_end;
-                    }
-                }
+                goto undo_read;
             }
         }
     }
@@ -242,11 +485,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         FINISH_OP(read_op);
         return 2;
     }
-    if (fulfilled < read_op->len)
-    {
-        assert(fulfill_read(read_op, fulfilled, 0, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0));
-        assert(fulfilled == read_op->len);
-    }
+    assert(fulfilled == read_op->len);
     read_op->version = result_version;
     if (!PRIV(read_op)->pending_ops)
     {
@@ -271,6 +510,309 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     }
     read_op->retval = 0;
     return 2;
+undo_read:
+    // need to wait. undo added requests, don't dequeue op
+    if (dsk.csum_block_size > dsk.bitmap_granularity)
+    {
+        for (auto & vec: rv)
+        {
+            if ((vec.copy_flags & COPY_BUF_CSUM_FILL) && vec.buf)
+            {
+                free(vec.buf);
+                vec.buf = NULL;
+            }
+            if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount
+            {
+                free(vec.dyn_data);
+                vec.dyn_data = NULL;
+            }
+        }
+    }
+    rv.clear();
+    return 0;
+}
+
+int blockstore_impl_t::pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
+    // FIXME Passing dirty_entry& would be nicer
+    uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
+    uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf)
+{
+    if (offset % dsk.csum_block_size || submit_len % dsk.csum_block_size)
+    {
+        if (offset < blk_end)
+        {
+            // Already being read as a part of the previous checksum block series
+            cp.buf = blk_buf + offset - blk_begin;
+            cp.copy_flags |= COPY_BUF_COALESCED;
+            if (offset+submit_len > blk_end)
+                cp.len = blk_end-offset;
+            return 2;
+        }
+        else
+        {
+            // We don't use fill_partial_checksum_blocks for journal because journal writes never have holes (internal bitmap)
+            blk_begin = (offset/dsk.csum_block_size) * dsk.csum_block_size;
+            blk_begin = blk_begin < dirty_offset ? dirty_offset : blk_begin;
+            blk_end = ((offset+submit_len-1)/dsk.csum_block_size + 1) * dsk.csum_block_size;
+            blk_end = blk_end > dirty_end ? dirty_end : blk_end;
+            if (blk_begin < offset || blk_end > offset+submit_len)
+            {
+                blk_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end-blk_begin);
+                cp.buf = blk_buf + offset - blk_begin;
+                cp.copy_flags |= COPY_BUF_COALESCED;
+                rv.push_back((copy_buffer_t){
+                    .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL,
+                    .offset = blk_begin,
+                    .len = blk_end-blk_begin,
+                    .disk_offset = dirty_loc + blk_begin - dirty_offset,
+                    .buf = blk_buf,
+                    .csum_buf = (csum_ptr + (blk_begin/dsk.csum_block_size -
+                        dirty_offset/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)),
+                    .dyn_data = dyn_data,
+                });
+                if (dyn_data)
+                {
+                    (*dyn_data)++;
+                }
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
+    uint8_t *clean_entry_bitmap, int *dyn_data, uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver)
+{
+    bool from_journal = clean_entry_bitmap != NULL;
+    if (!clean_entry_bitmap)
+    {
+        // NULL clean_entry_bitmap means we're reading from data, not from the journal,
+        // and the bitmap location is obvious
+        clean_entry_bitmap = get_clean_entry_bitmap(clean_loc, 0);
+    }
+    if (dsk.csum_block_size > dsk.bitmap_granularity)
+    {
+        auto & rv = PRIV(read_op)->read_vec;
+        int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, from_journal,
+            (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len);
+        if (!inmemory_meta && !from_journal && req > 0)
+        {
+            // Read checksums from disk
+            uint8_t *csum_buf = read_clean_meta_block(read_op, clean_loc, rv.size()-req);
+            for (int i = req; i > 0; i--)
+            {
+                rv[rv.size()-i].csum_buf = csum_buf;
+            }
+        }
+        for (int i = req; i > 0; i--)
+        {
+            if (!read_checksum_block(read_op, i, fulfilled, clean_loc))
+            {
+                return false;
+            }
+        }
+        PRIV(read_op)->clean_block_used = req > 0;
+    }
+    else if (from_journal)
+    {
+        // Don't scan bitmap - journal writes don't have holes (internal bitmap)!
+        uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size +
+            item_start/dsk.csum_block_size*(dsk.data_csum_type & 0xFF));
+        if (!fulfill_read(read_op, fulfilled, item_start, item_end,
+            (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_loc + item_start, 0, csum, dyn_data))
+        {
+            return false;
+        }
+        if (item_start > 0 && fulfilled < read_op->len)
+        {
+            // fill with zeroes
+            assert(fulfill_read(read_op, fulfilled, 0, item_start, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
+        }
+        if (item_end < dsk.data_block_size && fulfilled < read_op->len)
+        {
+            // fill with zeroes
+            assert(fulfill_read(read_op, fulfilled, item_end, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
+        }
+    }
+    else
+    {
+        bool csum_done = !dsk.csum_block_size || inmemory_meta;
+        uint8_t *csum_buf = clean_entry_bitmap;
+        uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity;
+        while (bmp_start < bmp_size)
+        {
+            while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size)
+            {
+                bmp_end++;
+            }
+            if (bmp_end > bmp_start)
+            {
+                // fill with zeroes
+                assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
+                    bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
+            }
+            bmp_start = bmp_end;
+            while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
+            {
+                bmp_end++;
+            }
+            if (bmp_end > bmp_start)
+            {
+                if (!csum_done)
+                {
+                    // Read checksums from disk
+                    csum_buf = read_clean_meta_block(read_op, clean_loc, PRIV(read_op)->read_vec.size());
+                    csum_done = true;
+                }
+                uint8_t *csum = !dsk.csum_block_size ? 0 : (csum_buf + 2*dsk.clean_entry_bitmap_size + bmp_start*(dsk.data_csum_type & 0xFF));
+                if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
+                    bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0,
+                    clean_loc + bmp_start * dsk.bitmap_granularity, 0, csum, dyn_data))
+                {
+                    return false;
+                }
+                bmp_start = bmp_end;
+            }
+        }
+    }
+    // Increment reference counter if clean data is being read from the disk
+    if (PRIV(read_op)->clean_block_used)
+    {
+        auto & uo = used_clean_objects[clean_loc];
+        uo.refs++;
+        if (dsk.csum_block_size && flusher->is_mutated(clean_loc))
+            uo.was_changed = true;
+        PRIV(read_op)->clean_block_used = clean_loc;
+    }
+    return true;
+}
+
+uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, uint64_t clean_loc, int rv_pos)
+{
+    auto & rv = PRIV(op)->read_vec;
+    auto sector = ((clean_loc >> dsk.block_order) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
+    auto pos = ((clean_loc >> dsk.block_order) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size;
+    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+    rv.insert(rv.begin()+rv_pos, (copy_buffer_t){
+        .copy_flags = COPY_BUF_META_BLOCK|COPY_BUF_CSUM_FILL,
+        .offset = pos,
+        .buf = buf,
+    });
+    BS_SUBMIT_GET_SQE(sqe, data);
+    data->iov = (struct iovec){ buf, dsk.meta_block_size };
+    PRIV(op)->pending_ops++;
+    my_uring_prep_readv(sqe, dsk.read_meta_fd, &data->iov, 1, dsk.meta_offset + dsk.meta_block_size + sector);
+    data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
+    // return pointer to checksums + bitmap
+    return buf + pos + sizeof(clean_disk_entry);
+}
+
+bool blockstore_impl_t::verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
+    iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb)
+{
+    assert(!(offset % dsk.csum_block_size));
+    uint32_t *csums = (uint32_t*)csum_buf;
+    uint32_t block_csum = 0;
+    uint32_t block_done = 0;
+    uint32_t block_num = clean_entry_bitmap ? offset/dsk.csum_block_size : 0;
+    uint32_t bmp_pos = offset/dsk.bitmap_granularity;
+    for (int i = 0; i < n_iov; i++)
+    {
+        uint32_t pos = 0;
+        while (pos < iov[i].iov_len)
+        {
+            uint32_t start = pos;
+            uint8_t bit = (clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1;
+            while (pos < iov[i].iov_len && ((clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1) == bit)
+            {
+                pos += dsk.bitmap_granularity;
+                bmp_pos++;
+            }
+            uint32_t len = pos-start;
+            auto buf = (uint8_t*)iov[i].iov_base+start;
+            while (block_done+len >= dsk.csum_block_size)
+            {
+                auto cur_len = dsk.csum_block_size-block_done;
+                block_csum = crc32c_pad(block_csum, buf, bit ? cur_len : 0, bit ? 0 : cur_len, 0);
+                if (block_csum != csums[block_num])
+                {
+                    if (bad_block_cb)
+                        bad_block_cb(block_num*dsk.csum_block_size, block_csum, csums[block_num]);
+                    else
+                        return false;
+                }
+                block_num++;
+                buf += cur_len;
+                len -= cur_len;
+                block_done = block_csum = 0;
+            }
+            if (len > 0)
+            {
+                block_csum = crc32c_pad(block_csum, buf, bit ? len : 0, bit ? 0 : len, 0);
+                block_done += len;
+            }
+        }
+    }
+    assert(!block_done);
+    return true;
+}
+
+bool blockstore_impl_t::verify_journal_checksums(uint8_t *csums, uint32_t offset,
+    iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb)
+{
+    uint32_t block_csum = 0;
+    uint32_t block_num = 0;
+    uint32_t block_done = offset%dsk.csum_block_size;
+    for (int i = 0; i < n_iov; i++)
+    {
+        uint32_t len = iov[i].iov_len;
+        auto buf = (uint8_t*)iov[i].iov_base;
+        while (block_done+len >= dsk.csum_block_size)
+        {
+            auto cur_len = dsk.csum_block_size-block_done;
+            block_csum = crc32c(block_csum, buf, cur_len);
+            if (block_csum != ((uint32_t*)csums)[block_num])
+            {
+                if (bad_block_cb)
+                    bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]);
+                else
+                    return false;
+            }
+            block_num++;
+            buf += cur_len;
+            len -= cur_len;
+            block_done = block_csum = 0;
+        }
+        if (len > 0)
+        {
+            block_csum = crc32c(block_csum, buf, len);
+            block_done += len;
+        }
+    }
+    if (block_done > 0 && block_csum != ((uint32_t*)csums)[block_num])
+    {
+        if (bad_block_cb)
+            bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]);
+        else
+            return false;
+    }
+    return true;
+}
+
+bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal,
+    iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb)
+{
+    uint32_t offset = clean_loc % dsk.data_block_size;
+    if (from_journal)
+        return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
+    clean_loc = (clean_loc >> dsk.block_order) << dsk.block_order;
+    if (!dyn_data)
+    {
+        assert(inmemory_meta);
+        dyn_data = get_clean_entry_bitmap(clean_loc, 0);
+    }
+    return verify_padded_checksums(dyn_data, dyn_data + 2*dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
 }
 
 void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op)
@@ -284,6 +826,139 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
     }
     if (PRIV(op)->pending_ops == 0)
     {
+        if (dsk.csum_block_size)
+        {
+            // verify checksums if required
+            auto & rv = PRIV(op)->read_vec;
+            void *meta_block = NULL;
+            if (dsk.csum_block_size > dsk.bitmap_granularity)
+            {
+                for (int i = rv.size()-1; i >= 0 && (rv[i].copy_flags & COPY_BUF_CSUM_FILL); i--)
+                {
+                    if (rv[i].copy_flags & COPY_BUF_META_BLOCK)
+                    {
+                        // Metadata read. Skip
+                        assert(!meta_block);
+                        meta_block = rv[i].buf;
+                        rv[i].buf = NULL;
+                        continue;
+                    }
+                    struct iovec *iov = (struct iovec*)((uint8_t*)rv[i].buf + (rv[i].len & 0xFFFFFFFF));
+                    int n_iov = rv[i].len >> 32;
+                    bool ok = true;
+                    if (rv[i].copy_flags & COPY_BUF_JOURNAL)
+                    {
+                        // SMALL_WRITE from journal
+                        verify_journal_checksums(
+                            rv[i].csum_buf, rv[i].offset, iov, n_iov,
+                            [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
+                            {
+                                ok = false;
+                                printf(
+                                    "Checksum mismatch in object %lx:%lx v%lu in journal at 0x%lx, checksum block #%u: got %08x, expected %08x\n",
+                                    op->oid.inode, op->oid.stripe, op->version,
+                                    rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum
+                                );
+                            }
+                        );
+                    }
+                    else
+                    {
+                        // BIG_WRITE from journal or clean data
+                        // Do not verify checksums if the data location is/was mutated by flushers
+                        auto & uo = used_clean_objects.at((rv[i].disk_offset >> dsk.block_order) << dsk.block_order);
+                        if (!uo.was_changed)
+                        {
+                            verify_clean_padded_checksums(
+                                op, rv[i].disk_offset, rv[i].csum_buf, (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG), iov, n_iov,
+                                [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
+                                {
+                                    ok = false;
+                                    printf(
+                                        "Checksum mismatch in object %lx:%lx v%lu in %s data at 0x%lx, checksum block #%u: got %08x, expected %08x\n",
+                                        op->oid.inode, op->oid.stripe, op->version,
+                                        (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG ? "redirect-write" : "clean"),
+                                        rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum
+                                    );
+                                }
+                            );
+                        }
+                    }
+                    if (!ok)
+                    {
+                        op->retval = -EDOM;
+                    }
+                    free(rv[i].buf);
+                    rv[i].buf = NULL;
+                    if (rv[i].dyn_data && --(*rv[i].dyn_data) == 0) // refcount
+                    {
+                        free(rv[i].dyn_data);
+                        rv[i].dyn_data = NULL;
+                    }
+                }
+            }
+            else
+            {
+                for (auto & vec: rv)
+                {
+                    if (vec.copy_flags & COPY_BUF_META_BLOCK)
+                    {
+                        // Metadata read. Skip
+                        assert(!meta_block);
+                        meta_block = vec.buf;
+                        vec.buf = NULL;
+                        continue;
+                    }
+                    if (vec.csum_buf)
+                    {
+                        uint32_t *csum = (uint32_t*)vec.csum_buf;
+                        for (size_t p = 0; p < vec.len; p += dsk.csum_block_size, csum++)
+                        {
+                            if (crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size) != *csum)
+                            {
+                                // checksum error
+                                printf(
+                                    "Checksum mismatch in object %lx:%lx v%lu in %s area at offset 0x%lx+0x%lx: %08x vs %08x\n",
+                                    op->oid.inode, op->oid.stripe, op->version,
+                                    (vec.copy_flags & COPY_BUF_JOURNAL) ? "journal" : "data", vec.disk_offset, p,
+                                    crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size), *csum
+                                );
+                                op->retval = -EDOM;
+                                break;
+                            }
+                        }
+                    }
+                    if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount
+                    {
+                        free(vec.dyn_data);
+                        vec.dyn_data = NULL;
+                    }
+                }
+            }
+            if (meta_block)
+            {
+                // Free after checking
+                free(meta_block);
+                meta_block = NULL;
+            }
+        }
+        if (PRIV(op)->clean_block_used)
+        {
+            // Release clean data block
+            auto uo_it = used_clean_objects.find(PRIV(op)->clean_block_used);
+            if (uo_it != used_clean_objects.end())
+            {
+                uo_it->second.refs--;
+                if (uo_it->second.refs <= 0)
+                {
+                    if (uo_it->second.was_freed)
+                    {
+                        data_alloc->set(PRIV(op)->clean_block_used, false);
+                    }
+                    used_clean_objects.erase(uo_it);
+                }
+            }
+        }
         if (!journal.inmemory)
         {
             // Release journal sector usage
@@ -324,8 +999,9 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
                     *result_version = dirty_it->first.version;
                 if (bitmap)
                 {
-                    void *bmp_ptr = (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap);
-                    memcpy(bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
+                    void *dyn_ptr = (alloc_dyn_data
+                        ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data);
+                    memcpy(bitmap, dyn_ptr, dsk.clean_entry_bitmap_size);
                 }
                 return 0;
             }

src/blockstore_rollback.cpp

@@ -218,7 +218,7 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
         auto used = --journal.used_sectors[dirty_it->second.journal_sector];
 #ifdef BLOCKSTORE_DEBUG
         printf(
-            "remove usage of journal offset %08lx by %lx:%lx v%lu (%d refs)\n", dirty_it->second.journal_sector,
+            "remove usage of journal offset %08lx by %lx:%lx v%lu (%lu refs)\n", dirty_it->second.journal_sector,
             dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, used
         );
 #endif
@@ -227,11 +227,7 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
             journal.used_sectors.erase(dirty_it->second.journal_sector);
             flusher->mark_trim_possible();
         }
-        if (dsk.clean_entry_bitmap_size > sizeof(void*))
-        {
-            free(dirty_it->second.bitmap);
-            dirty_it->second.bitmap = NULL;
-        }
+        free_dirty_dyn_data(dirty_it->second);
         if (dirty_it == dirty_start)
         {
             break;
@@ -240,3 +236,18 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
     }
     dirty_db.erase(dirty_start, dirty_end);
 }
+
+void blockstore_impl_t::free_dirty_dyn_data(dirty_entry & e)
+{
+    if (e.dyn_data)
+    {
+        if (alloc_dyn_data &&
+            --*((int*)e.dyn_data) == 0) // refcount
+        {
+            // dyn_data contains the bitmap and checksums
+            // free it if it doesn't refer to the in-memory journal
+            free(e.dyn_data);
+        }
+        e.dyn_data = NULL;
+    }
+}

src/blockstore_sync.cpp

@@ -78,7 +78,23 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
         // 2nd step: Data device is synced, prepare & write journal entries
         // Check space in the journal and journal memory buffers
         blockstore_journal_check_t space_check(this);
-        if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(),
+        if (dsk.csum_block_size)
+        {
+            // More complex check because all journal entries have different lengths
+            int left = PRIV(op)->sync_big_writes.size();
+            for (auto & sbw: PRIV(op)->sync_big_writes)
+            {
+                left--;
+                auto & dirty_entry = dirty_db.at(sbw);
+                uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
+                if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size,
+                    left == 0 ? JOURNAL_STABILIZE_RESERVATION : 0))
+                {
+                    return 0;
+                }
+            }
+        }
+        else if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(),
             sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, JOURNAL_STABILIZE_RESERVATION))
         {
             return 0;
@@ -90,16 +106,17 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
         int s = 0;
         while (it != PRIV(op)->sync_big_writes.end())
         {
-            if (!journal.entry_fits(sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size) &&
+            auto & dirty_entry = dirty_db.at(*it);
+            uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
+            if (!journal.entry_fits(sizeof(journal_entry_big_write) + dyn_size) &&
                 journal.sector_info[journal.cur_sector].dirty)
             {
                 prepare_journal_sector_write(journal.cur_sector, op);
                 s++;
             }
-            auto & dirty_entry = dirty_db.at(*it);
             journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
                 journal, (dirty_entry.state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
-                sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
+                sizeof(journal_entry_big_write) + dyn_size
             );
             dirty_entry.journal_sector = journal.sector_info[journal.cur_sector].offset;
             journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
@@ -115,8 +132,8 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
             je->offset = dirty_entry.offset;
             je->len = dirty_entry.len;
             je->location = dirty_entry.location;
-            memcpy((void*)(je+1), (dsk.clean_entry_bitmap_size > sizeof(void*)
-                ? dirty_entry.bitmap : &dirty_entry.bitmap), dsk.clean_entry_bitmap_size);
+            memcpy((void*)(je+1), (alloc_dyn_data
+                ? (uint8_t*)dirty_entry.dyn_data+sizeof(int) : (uint8_t*)&dirty_entry.dyn_data), dyn_size);
             je->crc32 = je_crc32((journal_entry*)je);
             journal.crc32_last = je->crc32;
             it++;

src/blockstore_write.cpp

@@ -8,12 +8,21 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
     // Check or assign version number
     bool found = false, deleted = false, unsynced = false, is_del = (op->opcode == BS_OP_DELETE);
     bool wait_big = false, wait_del = false;
-    void *bmp = NULL;
-    uint64_t version = 1;
-    if (!is_del && dsk.clean_entry_bitmap_size > sizeof(void*))
+    void *dyn = NULL;
+    if (is_del)
     {
-        bmp = calloc_or_die(1, dsk.clean_entry_bitmap_size);
+        op->len = 0;
     }
+    size_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
+    if (!is_del && alloc_dyn_data)
+    {
+        // FIXME: Working with `dyn_data` has to be refactored somehow but I first have to decide how :)
+        // +sizeof(int) = refcount
+        dyn = calloc_or_die(1, dyn_size+sizeof(int));
+        *((int*)dyn) = 1;
+    }
+    uint8_t *dyn_ptr = (uint8_t*)(alloc_dyn_data ? dyn+sizeof(int) : &dyn);
+    uint64_t version = 1;
     if (dirty_db.size() > 0)
     {
         auto dirty_it = dirty_db.upper_bound((obj_ver_id){
@@ -33,10 +42,9 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
                 : ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG);
             if (!is_del && !deleted)
             {
-                if (dsk.clean_entry_bitmap_size > sizeof(void*))
-                    memcpy(bmp, dirty_it->second.bitmap, dsk.clean_entry_bitmap_size);
-                else
-                    bmp = dirty_it->second.bitmap;
+                void *dyn_from = alloc_dyn_data
+                    ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data;
+                memcpy(dyn_ptr, dyn_from, dsk.clean_entry_bitmap_size);
             }
         }
     }
@@ -50,7 +58,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             if (!is_del)
             {
                 void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size);
-                memcpy((dsk.clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp), bmp_ptr, dsk.clean_entry_bitmap_size);
+                memcpy(dyn_ptr, bmp_ptr, dsk.clean_entry_bitmap_size);
             }
         }
         else
@@ -112,9 +120,9 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             printf("Write %lx:%lx v%lu requested, but we already have v%lu\n", op->oid.inode, op->oid.stripe, op->version, version);
 #endif
             op->retval = -EEXIST;
-            if (!is_del && dsk.clean_entry_bitmap_size > sizeof(void*))
+            if (!is_del && alloc_dyn_data)
             {
-                free(bmp);
+                free(dyn);
             }
             return false;
         }
@@ -158,26 +166,50 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         if (op->bitmap)
         {
             // Only allow to overwrite part of the object bitmap respective to the write's offset/len
-            uint8_t *bmp_ptr = (uint8_t*)(dsk.clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp);
             uint32_t bit = op->offset/dsk.bitmap_granularity;
             uint32_t bits_left = op->len/dsk.bitmap_granularity;
             while (!(bit % 8) && bits_left >= 8)
             {
                 // Copy bytes
-                bmp_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];
+                dyn_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];
                 bit += 8;
                 bits_left -= 8;
             }
             while (bits_left > 0)
             {
                 // Copy bits
-                bmp_ptr[bit/8] = (bmp_ptr[bit/8] & ~(1 << (bit%8)))
+                dyn_ptr[bit/8] = (dyn_ptr[bit/8] & ~(1 << (bit%8)))
                     | (((uint8_t*)op->bitmap)[bit/8] & (1 << bit%8));
                 bit++;
                 bits_left--;
             }
         }
     }
+    // Calculate checksums
+    // FIXME: Allow to receive checksums from outside?
+    if (!is_del && dsk.data_csum_type && op->len > 0)
+    {
+        uint32_t *data_csums = (uint32_t*)(dyn_ptr + dsk.clean_entry_bitmap_size);
+        uint32_t start = op->offset / dsk.csum_block_size;
+        uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
+        auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
+        if (start == end)
+            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
+        else
+        {
+            // First block
+            data_csums[0] = fn(0, op->buf, dsk.csum_block_size*(start+1)-op->offset, op->offset - start*dsk.csum_block_size, 0);
+            // Intermediate blocks
+            for (uint32_t i = start+1; i < end; i++)
+                data_csums[i-start] = crc32c(0, (uint8_t*)op->buf + dsk.csum_block_size*i-op->offset, dsk.csum_block_size);
+            // Last block
+            data_csums[end-start] = fn(
+                0, (uint8_t*)op->buf + end*dsk.csum_block_size - op->offset,
+                op->offset+op->len - end*dsk.csum_block_size,
+                0, (end+1)*dsk.csum_block_size - (op->offset+op->len)
+            );
+        }
+    }
     dirty_db.emplace((obj_ver_id){
         .oid = op->oid,
         .version = op->version,
@@ -188,7 +220,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         .offset = is_del ? 0 : op->offset,
         .len = is_del ? 0 : op->len,
         .journal_sector = 0,
-        .bitmap = bmp,
+        .dyn_data = dyn,
     });
     return true;
 }
@@ -197,8 +229,7 @@ void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_
 {
     while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
     {
-        if (dsk.clean_entry_bitmap_size > sizeof(void*))
-            free(dirty_it->second.bitmap);
+        free_dirty_dyn_data(dirty_it->second);
         dirty_db.erase(dirty_it++);
     }
     bool found = false;
@@ -280,7 +311,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         blockstore_journal_check_t space_check(this);
         if (!space_check.check_available(op, unsynced_big_write_count + 1,
-            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size,
+            sizeof(journal_entry_big_write) + dsk.clean_dyn_size,
             (dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION))
         {
             return 0;
@@ -363,12 +394,13 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         // Small (journaled) write
         // First check if the journal has sufficient space
+        uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
         blockstore_journal_check_t space_check(this);
         if (unsynced_big_write_count &&
             !space_check.check_available(op, unsynced_big_write_count,
-                sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0)
+                sizeof(journal_entry_big_write) + dsk.clean_dyn_size, 0)
             || !space_check.check_available(op, 1,
-                sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size,
+                sizeof(journal_entry_small_write) + dyn_size,
                 op->len + ((dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION)))
         {
             return 0;
@@ -377,7 +409,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         BS_SUBMIT_CHECK_SQES(
             // Write current journal sector only if it's dirty and full, or in the immediate_commit mode
             (immediate_commit != IMMEDIATE_NONE ||
-                !journal.entry_fits(sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size) ? 1 : 0) +
+                !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size) ? 1 : 0) +
             (op->len > 0 ? 1 : 0)
         );
         write_iodepth++;
@@ -385,7 +417,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         auto cb = [this, op](ring_data_t *data) { handle_write_event(data, op); };
         if (immediate_commit == IMMEDIATE_NONE)
         {
-            if (!journal.entry_fits(sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size))
+            if (!journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size))
             {
                 prepare_journal_sector_write(journal.cur_sector, op);
             }
@@ -397,7 +429,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         // Then pre-fill journal entry
         journal_entry_small_write *je = (journal_entry_small_write*)prefill_single_journal_entry(
             journal, op->opcode == BS_OP_WRITE_STABLE ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE,
-            sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size
+            sizeof(journal_entry_small_write) + dyn_size
         );
         dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
         journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
@@ -431,8 +463,9 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         je->offset = op->offset;
         je->len = op->len;
         je->data_offset = journal.next_free;
-        je->crc32_data = crc32c(0, op->buf, op->len);
-        memcpy((void*)(je+1), (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap), dsk.clean_entry_bitmap_size);
+        je->crc32_data = dsk.csum_block_size ? 0 : crc32c(0, op->buf, op->len);
+        memcpy((void*)(je+1), (alloc_dyn_data
+            ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size);
         je->crc32 = je_crc32((journal_entry*)je);
         journal.crc32_last = je->crc32;
         if (immediate_commit != IMMEDIATE_NONE)
@@ -501,9 +534,9 @@ resume_2:
             .version = op->version,
         });
         assert(dirty_it != dirty_db.end());
+        uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
         blockstore_journal_check_t space_check(this);
-        if (!space_check.check_available(op, 1,
-            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size,
+        if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size,
             ((dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION)))
         {
             return 0;
@@ -511,7 +544,7 @@ resume_2:
         BS_SUBMIT_CHECK_SQES(1);
         journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
             journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
-            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
+            sizeof(journal_entry_big_write) + dyn_size
         );
         dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
         journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
@@ -527,7 +560,8 @@ resume_2:
         je->offset = op->offset;
         je->len = op->len;
         je->location = dirty_it->second.location;
-        memcpy((void*)(je+1), (dsk.clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap), dsk.clean_entry_bitmap_size);
+        memcpy((void*)(je+1), (alloc_dyn_data
+            ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size);
         je->crc32 = je_crc32((journal_entry*)je);
         journal.crc32_last = je->crc32;
         prepare_journal_sector_write(journal.cur_sector, op);
@@ -661,8 +695,13 @@ void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
         uint64_t s = PRIV(op)->min_flushed_journal_sector;
         while (1)
         {
-            if (s != (1+journal.cur_sector) && journal.sector_info[s-1].flush_count == 0)
+            if (!journal.sector_info[s-1].dirty && journal.sector_info[s-1].flush_count == 0)
             {
+                if (s == (1+journal.cur_sector))
+                {
+                    // Forcibly move to the next sector and move dirty position
+                    journal.in_sector_pos = journal.block_size;
+                }
                 // We know for sure that we won't write into this sector anymore
                 uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size;
                 if (new_ds >= journal.len)

src/disk_simple_offsets.cpp

@@ -10,6 +10,7 @@
 #include "json11/json11.hpp"
 #include "str_util.h"
 #include "blockstore.h"
+#include "blockstore_disk.h"
 
 // Calculate offsets for a block device and print OSD command line parameters
 void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
@@ -20,23 +21,39 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
         fprintf(stderr, "Device path is missing\n");
         exit(1);
     }
-    uint64_t object_size = parse_size(cfg["object_size"].string_value());
+    uint64_t data_block_size = parse_size(cfg["object_size"].string_value());
     uint64_t bitmap_granularity = parse_size(cfg["bitmap_granularity"].string_value());
     uint64_t journal_size = parse_size(cfg["journal_size"].string_value());
     uint64_t device_block_size = parse_size(cfg["device_block_size"].string_value());
     uint64_t journal_offset = parse_size(cfg["journal_offset"].string_value());
     uint64_t device_size = parse_size(cfg["device_size"].string_value());
+    uint32_t csum_block_size = parse_size(cfg["csum_block_size"].string_value());
+    uint32_t data_csum_type = BLOCKSTORE_CSUM_NONE;
+    if (cfg["data_csum_type"] == "crc32c")
+        data_csum_type = BLOCKSTORE_CSUM_CRC32C;
+    else if (cfg["data_csum_type"].string_value() != "" && cfg["data_csum_type"].string_value() != "none")
+    {
+        fprintf(
+            stderr, "data_csum_type=%s is unsupported, only \"crc32c\" and \"none\" are supported",
+            cfg["data_csum_type"].string_value().c_str()
+        );
+        exit(1);
+    }
     std::string format = cfg["format"].string_value();
     if (json_output)
         format = "json";
-    if (!object_size)
-        object_size = 1 << DEFAULT_DATA_BLOCK_ORDER;
+    if (!data_block_size)
+        data_block_size = 1 << DEFAULT_DATA_BLOCK_ORDER;
     if (!bitmap_granularity)
         bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
     if (!journal_size)
         journal_size = 16*1024*1024;
     if (!device_block_size)
         device_block_size = 4096;
+    if (!data_csum_type)
+        csum_block_size = 0;
+    else if (!csum_block_size)
+        csum_block_size = bitmap_granularity;
     uint64_t orig_device_size = device_size;
     if (!device_size)
     {
@@ -85,22 +102,30 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
         fprintf(stderr, "Invalid device block size specified: %lu\n", device_block_size);
         exit(1);
     }
-    if (object_size < device_block_size || object_size > MAX_DATA_BLOCK_SIZE ||
-        object_size & (object_size-1) != 0)
+    if (data_block_size < device_block_size || data_block_size > MAX_DATA_BLOCK_SIZE ||
+        data_block_size & (data_block_size-1) != 0)
     {
-        fprintf(stderr, "Invalid object size specified: %lu\n", object_size);
+        fprintf(stderr, "Invalid object size specified: %lu\n", data_block_size);
         exit(1);
     }
-    if (bitmap_granularity < device_block_size || bitmap_granularity > object_size ||
+    if (bitmap_granularity < device_block_size || bitmap_granularity > data_block_size ||
         bitmap_granularity & (bitmap_granularity-1) != 0)
     {
         fprintf(stderr, "Invalid bitmap granularity specified: %lu\n", bitmap_granularity);
         exit(1);
     }
+    if (csum_block_size && (data_block_size % csum_block_size))
+    {
+        fprintf(stderr, "csum_block_size must be a divisor of data_block_size\n");
+        exit(1);
+    }
     journal_offset = ((journal_offset+device_block_size-1)/device_block_size)*device_block_size;
     uint64_t meta_offset = journal_offset + ((journal_size+device_block_size-1)/device_block_size)*device_block_size;
-    uint64_t entries_per_block = (device_block_size / (24 + 2*object_size/bitmap_granularity/8));
-    uint64_t object_count = ((device_size-meta_offset)/object_size);
+    uint64_t data_csum_size = (data_csum_type ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
+    uint64_t clean_entry_bitmap_size = data_block_size/bitmap_granularity/8;
+    uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size + data_csum_size + 4 /*entry_csum*/;
+    uint64_t entries_per_block = device_block_size / clean_entry_size;
+    uint64_t object_count = ((device_size-meta_offset)/data_block_size);
     uint64_t meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
     uint64_t data_offset = meta_offset + meta_size;
     if (format == "json")

src/disk_tool.cpp

@@ -59,6 +59,8 @@ static const char *help_text =
     "    --journal_size 32M/1G      Set journal size (area or partition size)\n"
     "    --block_size 128k/1M       Set blockstore object size\n"
     "    --bitmap_granularity 4k    Set bitmap granularity\n"
+    "    --data_csum_type none      Set data checksum type (crc32c or none)\n"
+    "    --csum_block_size 4k       Set data checksum block size\n"
     "    --data_device_block 4k     Override data device block size\n"
     "    --meta_device_block 4k     Override metadata device block size\n"
     "    --journal_device_block 4k  Override journal device block size\n"
@@ -72,8 +74,9 @@ static const char *help_text =
     "  If it doesn't succeed it issues a warning in the system log.\n"
     "  \n"
     "  You can also pass other OSD options here as arguments and they'll be persisted\n"
-    "  to the superblock: max_write_iodepth, max_write_iodepth, min_flusher_count,\n"
-    "  max_flusher_count, inmemory_metadata, inmemory_journal, journal_sector_buffer_count,\n"
+    "  in the superblock: cached_read_data, cached_read_meta, cached_read_journal,\n"
+    "  inmemory_metadata, inmemory_journal, max_write_iodepth,\n"
+    "  min_flusher_count, max_flusher_count, journal_sector_buffer_count,\n"
     "  journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,\n"
     "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.\n"
     "\n"
@@ -161,6 +164,8 @@ static const char *help_text =
     "    --object_size 128k       Set blockstore block size\n"
     "    --bitmap_granularity 4k  Set bitmap granularity\n"
     "    --journal_size 16M       Set journal size\n"
+    "    --data_csum_type none    Set data checksum type (crc32c or none)\n"
+    "    --csum_block_size 4k     Set data checksum block size\n"
     "    --device_block_size 4k   Set device block size\n"
     "    --journal_offset 0       Set journal offset\n"
     "    --device_size 0          Set device size\n"
@@ -270,6 +275,19 @@ int main(int argc, char *argv[])
             fprintf(stderr, "Invalid JSON: %s\n", json_err.c_str());
             return 1;
         }
+        if (entries[0]["type"] == "start")
+        {
+            self.dsk.data_csum_type = csum_type_from_str(entries[0]["data_csum_type"].string_value());
+            self.dsk.csum_block_size = entries[0]["csum_block_size"].uint64_value();
+        }
+        if (self.options["data_csum_type"] != "")
+        {
+            self.dsk.data_csum_type = csum_type_from_str(self.options["data_csum_type"]);
+        }
+        if (self.options["csum_block_size"] != "")
+        {
+            self.dsk.csum_block_size = stoull_full(self.options["csum_block_size"], 0);
+        }
         return self.write_json_journal(entries);
     }
     else if (!strcmp(cmd[0], "dump-meta"))

src/disk_tool.h

@@ -64,17 +64,19 @@ struct disk_tool_t
     ring_loop_t *ringloop;
     ring_consumer_t ring_consumer;
     int remap_active;
+    journal_entry_start je_start;
     uint8_t *new_journal_buf, *new_meta_buf, *new_journal_ptr, *new_journal_data;
     uint64_t new_journal_in_pos;
     int64_t data_idx_diff;
     uint64_t total_blocks, free_first, free_last;
-    uint64_t new_clean_entry_bitmap_size, new_clean_entry_size, new_entries_per_block;
+    uint64_t new_clean_entry_bitmap_size, new_data_csum_size, new_clean_entry_size, new_entries_per_block;
     int new_journal_fd, new_meta_fd;
     resizer_data_moving_t *moving_blocks;
 
     bool started;
     void *small_write_data;
     uint32_t data_crc32;
+    bool data_csum_valid;
     uint32_t crc32_last;
     uint32_t new_crc32_prev;
 
@@ -84,11 +86,11 @@ struct disk_tool_t
     void dump_journal_entry(int num, journal_entry *je, bool json);
     int process_journal(std::function<int(void*)> block_fn);
     int process_journal_block(void *buf, std::function<void(int, journal_entry*)> iter_fn);
-    int process_meta(std::function<void(blockstore_meta_header_v1_t *)> hdr_fn,
+    int process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
         std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn);
 
     int dump_meta();
-    void dump_meta_header(blockstore_meta_header_v1_t *hdr);
+    void dump_meta_header(blockstore_meta_header_v2_t *hdr);
     void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
 
     int write_json_journal(json11::Json entries);
@@ -96,7 +98,7 @@ struct disk_tool_t
 
     int resize_data();
     int resize_parse_params();
-    void resize_init(blockstore_meta_header_v1_t *hdr);
+    void resize_init(blockstore_meta_header_v2_t *hdr);
     int resize_remap_blocks();
     int resize_copy_data();
     int resize_rewrite_journal();
@@ -141,3 +143,5 @@ json11::Json read_parttable(std::string dev);
 uint64_t dev_size_from_parttable(json11::Json pt);
 uint64_t free_from_parttable(json11::Json pt);
 int fix_partition_type(std::string dev_by_uuid);
+std::string csum_type_str(uint32_t data_csum_type);
+uint32_t csum_type_from_str(std::string data_csum_type);

src/disk_tool_journal.cpp

@@ -55,6 +55,23 @@ int disk_tool_t::dump_journal()
                     printf("offset %08lx:\n", journal_pos);
                 else
                     printf(",\"entries\":[\n");
+                if (journal_pos == 0)
+                {
+                    // Fill journal header to know checksum type & size
+                    journal_entry *je = (journal_entry*)journal_buf;
+                    if (je->magic == JOURNAL_MAGIC && je->type == JE_START &&
+                        (je->start.version == JOURNAL_VERSION_V1 || je->start.version == JOURNAL_VERSION_V2))
+                    {
+                        memcpy(&je_start, je, sizeof(je_start));
+                        if (je_start.size == JE_START_V0_SIZE)
+                            je_start.version = 0;
+                        if (je_start.version < JOURNAL_VERSION_V2)
+                        {
+                            je_start.data_csum_type = 0;
+                            je_start.csum_block_size = 0;
+                        }
+                    }
+                }
                 first_entry = true;
                 process_journal_block(journal_buf, [this](int num, journal_entry *je) { dump_journal_entry(num, je, json); });
                 if (json)
@@ -120,8 +137,22 @@ int disk_tool_t::process_journal(std::function<int(void*)> block_fn)
         fprintf(stderr, "offset %08lx: journal superblock is invalid\n", journal_pos);
         r = 1;
     }
+    else if (je->start.size != JE_START_V0_SIZE && je->start.version != JOURNAL_VERSION_V1 && je->start.version != JOURNAL_VERSION_V2)
+    {
+        fprintf(stderr, "offset %08lx: journal superblock contains version %lu, but I only understand 0, 1 and 2\n",
+            journal_pos, je->start.size == JE_START_V0_SIZE ? 0 : je->start.version);
+        r = 1;
+    }
     else
     {
+        memcpy(&je_start, je, sizeof(je_start));
+        if (je_start.size == JE_START_V0_SIZE)
+            je_start.version = 0;
+        if (je_start.version < JOURNAL_VERSION_V2)
+        {
+            je_start.data_csum_type = 0;
+            je_start.csum_block_size = 0;
+        }
         started = false;
         crc32_last = 0;
         block_fn(data);
@@ -183,7 +214,49 @@ int disk_tool_t::process_journal_block(void *buf, std::function<void(int, journa
             }
             small_write_data = memalign_or_die(MEM_ALIGNMENT, je->small_write.len);
             assert(pread(dsk.journal_fd, small_write_data, je->small_write.len, dsk.journal_offset+je->small_write.data_offset) == je->small_write.len);
-            data_crc32 = crc32c(0, small_write_data, je->small_write.len);
+            data_crc32 = je_start.csum_block_size ? 0 : crc32c(0, small_write_data, je->small_write.len);
+            data_csum_valid = (data_crc32 == je->small_write.crc32_data);
+            if (je_start.csum_block_size && je->small_write.len > 0)
+            {
+                // like in enqueue_write()
+                uint32_t start = je->small_write.offset / je_start.csum_block_size;
+                uint32_t end = (je->small_write.offset+je->small_write.len-1) / je_start.csum_block_size;
+                uint32_t data_csum_size = (end-start+1) * (je_start.data_csum_type & 0xFF);
+                if (je->size < sizeof(journal_entry_small_write) + data_csum_size)
+                {
+                    data_csum_valid = false;
+                }
+                else
+                {
+                    uint32_t calc_csum = 0;
+                    uint32_t *block_csums = (uint32_t*)((uint8_t*)je + je->size - data_csum_size);
+                    if (start == end)
+                    {
+                        calc_csum = crc32c(0, (uint8_t*)small_write_data, je->small_write.len);
+                        data_csum_valid = data_csum_valid && (calc_csum == *block_csums++);
+                    }
+                    else
+                    {
+                        // First block
+                        calc_csum = crc32c(0, (uint8_t*)small_write_data,
+                            je_start.csum_block_size*(start+1)-je->small_write.offset);
+                        data_csum_valid = data_csum_valid && (calc_csum == *block_csums++);
+                        // Intermediate blocks
+                        for (uint32_t i = start+1; i < end; i++)
+                        {
+                            calc_csum = crc32c(0, (uint8_t*)small_write_data +
+                                je_start.csum_block_size*i-je->small_write.offset, je_start.csum_block_size);
+                            data_csum_valid = data_csum_valid && (calc_csum == *block_csums++);
+                        }
+                        // Last block
+                        calc_csum = crc32c(
+                            0, (uint8_t*)small_write_data + end*je_start.csum_block_size - je->small_write.offset,
+                            je->small_write.offset+je->small_write.len - end*je_start.csum_block_size
+                        );
+                        data_csum_valid = data_csum_valid && (calc_csum == *block_csums++);
+                    }
+                }
+            }
         }
         iter_fn(entry, je);
         if (je->type == JE_SMALL_WRITE || je->type == JE_SMALL_WRITE_INSTANT)
@@ -223,29 +296,40 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
     if (je->type == JE_START)
     {
         printf(
-            json ? ",\"type\":\"start\",\"start\":\"0x%lx\"}" : "je_start start=%08lx\n",
+            json ? ",\"type\":\"start\",\"start\":\"0x%lx\"" : "je_start start=%08lx",
             je->start.journal_start
         );
+        if (je->start.data_csum_type)
+        {
+            printf(
+                json ? ",\"data_csum_type\":\"%s\",\"csum_block_size\":%u" : " data_csum_type=%s csum_block_size=%u",
+                csum_type_str(je->start.data_csum_type).c_str(), je->start.csum_block_size
+            );
+        }
+        printf(json ? "}" : "\n");
     }
     else if (je->type == JE_SMALL_WRITE || je->type == JE_SMALL_WRITE_INSTANT)
     {
+        auto & sw = je->small_write;
         printf(
             json ? ",\"type\":\"small_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"offset\":%u,\"len\":%u,\"loc\":\"0x%lx\""
                 : "je_small_write%s oid=%lx:%lx ver=%lu offset=%u len=%u loc=%08lx",
             je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "",
-            je->small_write.oid.inode, je->small_write.oid.stripe,
-            je->small_write.version, je->small_write.offset, je->small_write.len,
-            je->small_write.data_offset
+            sw.oid.inode, sw.oid.stripe, sw.version, sw.offset, sw.len, sw.data_offset
         );
-        if (journal_calc_data_pos != je->small_write.data_offset)
+        if (journal_calc_data_pos != sw.data_offset)
         {
             printf(json ? ",\"bad_loc\":true,\"calc_loc\":\"0x%lx\""
                 : " (mismatched, calculated = %lu)", journal_pos);
         }
-        if (je->small_write.size > sizeof(journal_entry_small_write))
+        uint32_t data_csum_size = (!je_start.csum_block_size
+            ? 0
+            : ((sw.offset + sw.len - 1)/je_start.csum_block_size - sw.offset/je_start.csum_block_size + 1)
+                *(je_start.data_csum_type & 0xFF));
+        if (je->size > sizeof(journal_entry_small_write) + data_csum_size)
         {
             printf(json ? ",\"bitmap\":\"" : " (bitmap: ");
-            for (int i = sizeof(journal_entry_small_write); i < je->small_write.size; i++)
+            for (int i = sizeof(journal_entry_small_write); i < je->size - data_csum_size; i++)
             {
                 printf("%02x", ((uint8_t*)je)[i]);
             }
@@ -254,34 +338,56 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
         if (dump_with_data)
         {
             printf(json ? ",\"data\":\"" : " (data: ");
-            for (int i = 0; i < je->small_write.len; i++)
+            for (int i = 0; i < sw.len; i++)
             {
                 printf("%02x", ((uint8_t*)small_write_data)[i]);
             }
             printf(json ? "\"" : ")");
         }
+        if (data_csum_size > 0 && je->size >= sizeof(journal_entry_small_write) + data_csum_size)
+        {
+            printf(json ? ",\"block_csums\":\"" : " block_csums=");
+            uint8_t *block_csums = (uint8_t*)je + je->size - data_csum_size;
+            for (int i = 0; i < data_csum_size; i++)
+                printf("%02x", block_csums[i]);
+            printf(json ? "\"" : "");
+        }
+        else
+        {
+            printf(json ? ",\"data_crc32\":\"%08x\"" : " data_crc32=%08x", sw.crc32_data);
+        }
         printf(
-            json ? ",\"data_crc32\":\"%08x\",\"data_valid\":%s}" : " data_crc32=%08x%s\n",
-            je->small_write.crc32_data,
-            (data_crc32 != je->small_write.crc32_data
-                ? (json ? "false" : " (invalid)")
-                : (json ? "true" : " (valid)"))
+            json ? ",\"data_valid\":%s}" : "%s\n",
+            (data_csum_valid
+                ? (json ? "true" : " (valid)")
+                : (json ? "false" : " (invalid)"))
         );
     }
     else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
     {
+        auto & bw = je->big_write;
         printf(
             json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"offset\":%u,\"len\":%u,\"loc\":\"0x%lx\""
                 : "je_big_write%s oid=%lx:%lx ver=%lu offset=%u len=%u loc=%08lx",
             je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
-            je->big_write.oid.inode, je->big_write.oid.stripe,
-            je->big_write.version, je->big_write.offset, je->big_write.len,
-            je->big_write.location
+            bw.oid.inode, bw.oid.stripe, bw.version, bw.offset, bw.len, bw.location
         );
-        if (je->big_write.size > sizeof(journal_entry_big_write))
+        uint32_t data_csum_size = (!je_start.csum_block_size
+            ? 0
+            : ((bw.offset + bw.len - 1)/je_start.csum_block_size - bw.offset/je_start.csum_block_size + 1)
+                *(je_start.data_csum_type & 0xFF));
+        if (data_csum_size > 0 && je->size >= sizeof(journal_entry_big_write) + data_csum_size)
+        {
+            printf(json ? ",\"block_csums\":\"" : " block_csums=");
+            uint8_t *block_csums = (uint8_t*)je + je->size - data_csum_size;
+            for (int i = 0; i < data_csum_size; i++)
+                printf("%02x", block_csums[i]);
+            printf(json ? "\"" : "");
+        }
+        if (bw.size > sizeof(journal_entry_big_write) + data_csum_size)
         {
             printf(json ? ",\"bitmap\":\"" : " (bitmap: ");
-            for (int i = sizeof(journal_entry_big_write); i < je->big_write.size; i++)
+            for (int i = sizeof(journal_entry_big_write); i < bw.size - data_csum_size; i++)
             {
                 printf("%02x", ((uint8_t*)je)[i]);
             }
@@ -338,7 +444,9 @@ int disk_tool_t::write_json_journal(json11::Json entries)
         .type = JE_START,
         .size = sizeof(journal_entry_start),
         .journal_start = dsk.journal_block_size,
-        .version = JOURNAL_VERSION,
+        .version = JOURNAL_VERSION_V2,
+        .data_csum_type = dsk.data_csum_type,
+        .csum_block_size = dsk.csum_block_size,
     };
     ((journal_entry*)new_journal_buf)->crc32 = je_crc32((journal_entry*)new_journal_buf);
     new_journal_ptr += dsk.journal_block_size;
@@ -358,9 +466,11 @@ int disk_tool_t::write_json_journal(json11::Json entries)
         uint32_t entry_size = (type == JE_START
             ? sizeof(journal_entry_start)
             : (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT
-                ? sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size
+                ? sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size +
+                    (dsk.data_csum_type ? rec["len"].uint64_value()/dsk.csum_block_size*(dsk.data_csum_type & 0xFF) : 0)
                 : (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT
-                    ? sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
+                    ? sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size +
+                        (dsk.data_csum_type ? rec["len"].uint64_value()/dsk.csum_block_size*(dsk.data_csum_type & 0xFF) : 0)
                     : sizeof(journal_entry_del))));
         if (dsk.journal_block_size < new_journal_in_pos + entry_size)
         {
@@ -402,12 +512,24 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                 .offset = (uint32_t)rec["offset"].uint64_value(),
                 .len = (uint32_t)rec["len"].uint64_value(),
                 .data_offset = (uint64_t)(new_journal_data-new_journal_buf),
-                .crc32_data = (uint32_t)sscanf_json("%x", rec["data_crc32"]),
+                .crc32_data = !dsk.data_csum_type ? 0 : (uint32_t)sscanf_json("%x", rec["data_crc32"]),
             };
-            fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
+            uint32_t data_csum_size = !dsk.data_csum_type ? 0 : ne->small_write.len/dsk.csum_block_size*(dsk.data_csum_type & 0xFF);
+            fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write) + data_csum_size);
             fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
+            if (dsk.data_csum_type)
+                fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
             if (rec["data"].is_string())
-                ne->small_write.crc32_data = crc32c(0, new_journal_data, ne->small_write.len);
+            {
+                if (!dsk.data_csum_type)
+                    ne->small_write.crc32_data = crc32c(0, new_journal_data, ne->small_write.len);
+                else if (dsk.data_csum_type == BLOCKSTORE_CSUM_CRC32C)
+                {
+                    uint32_t *block_csums = (uint32_t*)(((uint8_t*)ne) + sizeof(journal_entry_small_write));
+                    for (uint32_t i = 0; i < ne->small_write.len; i += dsk.csum_block_size, block_csums++)
+                        *block_csums = crc32c(0, new_journal_data+i, dsk.csum_block_size);
+                }
+            }
             new_journal_data += ne->small_write.len;
         }
         else if (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT)
@@ -426,7 +548,10 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                 .len = (uint32_t)rec["len"].uint64_value(),
                 .location = sscanf_json(NULL, rec["loc"]),
             };
-            fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
+            uint32_t data_csum_size = !dsk.data_csum_type ? 0 : ne->big_write.len/dsk.csum_block_size*(dsk.data_csum_type & 0xFF);
+            fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write) + data_csum_size);
+            if (dsk.data_csum_type)
+                fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
         }
         else if (type == JE_STABLE || type == JE_ROLLBACK || type == JE_DELETE)
         {

src/disk_tool_meta.cpp

@@ -5,7 +5,7 @@
 #include "rw_blocking.h"
 #include "osd_id.h"
 
-int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)> hdr_fn,
+int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
     std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn)
 {
     if (dsk.meta_block_size % DIRECT_IO_ALIGNMENT)
@@ -28,12 +28,38 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)>
     lseek64(dsk.meta_fd, dsk.meta_offset, 0);
     read_blocking(dsk.meta_fd, data, dsk.meta_block_size);
     // Check superblock
-    blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)data;
-    if (hdr->zero == 0 &&
-        hdr->magic == BLOCKSTORE_META_MAGIC_V1 &&
-        hdr->version == BLOCKSTORE_META_VERSION_V1)
+    blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)data;
+    if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1)
     {
-        // Vitastor 0.6-0.7 - static array of clean_disk_entry with bitmaps
+        if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
+        {
+            // Vitastor 0.6-0.8 - static array of clean_disk_entry with bitmaps
+            hdr->data_csum_type = 0;
+            hdr->csum_block_size = 0;
+            hdr->header_csum = 0;
+        }
+        else if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
+        {
+            // Vitastor 0.9 - static array of clean_disk_entry with bitmaps and checksums
+            if (hdr->data_csum_type != 0 &&
+                hdr->data_csum_type != BLOCKSTORE_CSUM_CRC32C)
+            {
+                fprintf(stderr, "I don't know checksum format %u, the only supported format is crc32c = %u.\n", hdr->data_csum_type, BLOCKSTORE_CSUM_CRC32C);
+                free(data);
+                close(dsk.meta_fd);
+                dsk.meta_fd = -1;
+                return 1;
+            }
+        }
+        else
+        {
+            // Unsupported version
+            fprintf(stderr, "Metadata format is too new for me (stored version is %lu, max supported %u).\n", hdr->version, BLOCKSTORE_META_FORMAT_V2);
+            free(data);
+            close(dsk.meta_fd);
+            dsk.meta_fd = -1;
+            return 1;
+        }
         if (hdr->meta_block_size != dsk.meta_block_size)
         {
             fprintf(stderr, "Using block size of %u bytes based on information from the superblock\n", hdr->meta_block_size);
@@ -45,14 +71,24 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)>
                 memcpy(new_data, data, dsk.meta_block_size);
                 free(data);
                 data = new_data;
-                hdr = (blockstore_meta_header_v1_t *)data;
+                hdr = (blockstore_meta_header_v2_t *)data;
             }
         }
+        dsk.meta_format = hdr->version;
+        dsk.data_block_size = hdr->data_block_size;
+        dsk.csum_block_size = hdr->csum_block_size;
+        dsk.data_csum_type = hdr->data_csum_type;
         dsk.bitmap_granularity = hdr->bitmap_granularity;
-        dsk.clean_entry_bitmap_size = hdr->data_block_size / hdr->bitmap_granularity / 8;
-        dsk.clean_entry_size = sizeof(clean_disk_entry) + 2*dsk.clean_entry_bitmap_size;
+        dsk.clean_entry_bitmap_size = (hdr->data_block_size / hdr->bitmap_granularity + 7) / 8;
+        dsk.clean_entry_size = sizeof(clean_disk_entry) + 2*dsk.clean_entry_bitmap_size
+            + (hdr->data_csum_type
+                ? ((hdr->data_block_size+hdr->csum_block_size-1)/hdr->csum_block_size
+                    *(hdr->data_csum_type & 0xff))
+                : 0)
+            + (dsk.meta_format == BLOCKSTORE_META_FORMAT_V2 ? 4 /*entry_csum*/ : 0);
         uint64_t block_num = 0;
         hdr_fn(hdr);
+        hdr = NULL;
         meta_pos = dsk.meta_block_size;
         lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
         while (meta_pos < dsk.meta_len)
@@ -67,6 +103,15 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)>
                     clean_disk_entry *entry = (clean_disk_entry*)((uint8_t*)data + blk + ioff);
                     if (entry->oid.inode)
                     {
+                        if (dsk.data_csum_type)
+                        {
+                            uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + dsk.clean_entry_size - 4);
+                            if (*entry_csum != crc32c(0, entry, dsk.clean_entry_size - 4))
+                            {
+                                fprintf(stderr, "Metadata entry %lu is corrupt (checksum mismatch), skipping\n", block_num);
+                                continue;
+                            }
+                        }
                         record_fn(block_num, entry, entry->bitmap);
                     }
                 }
@@ -107,21 +152,35 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)>
 int disk_tool_t::dump_meta()
 {
     int r = process_meta(
-        [this](blockstore_meta_header_v1_t *hdr) { dump_meta_header(hdr); },
+        [this](blockstore_meta_header_v2_t *hdr) { dump_meta_header(hdr); },
         [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap) { dump_meta_entry(block_num, entry, bitmap); }
     );
-    printf("\n]}\n");
+    if (r == 0)
+        printf("\n]}\n");
     return r;
 }
 
-void disk_tool_t::dump_meta_header(blockstore_meta_header_v1_t *hdr)
+void disk_tool_t::dump_meta_header(blockstore_meta_header_v2_t *hdr)
 {
     if (hdr)
     {
-        printf(
-            "{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,\"entries\":[\n",
-            hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
-        );
+        if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
+        {
+            printf(
+                "{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "\"entries\":[\n",
+                hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
+            );
+        }
+        else if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
+        {
+            printf(
+                "{\"version\":\"0.9\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "\"data_csum_type\":%s,\"csum_block_size\":%u,\"entries\":[\n",
+                hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
+                csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
+            );
+        }
     }
     else
     {
@@ -151,6 +210,15 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
         {
             printf("%02x", bitmap[dsk.clean_entry_bitmap_size + i]);
         }
+        if (dsk.csum_block_size && dsk.data_csum_type)
+        {
+            uint8_t *csums = bitmap + dsk.clean_entry_bitmap_size*2;
+            printf("\",\"block_csums\":\"");
+            for (uint64_t i = 0; i < (dsk.data_block_size+dsk.csum_block_size-1)/dsk.csum_block_size*(dsk.data_csum_type & 0xFF); i++)
+            {
+                printf("%02x", csums[i]);
+            }
+        }
         printf("\"}");
     }
     else
@@ -164,18 +232,30 @@ int disk_tool_t::write_json_meta(json11::Json meta)
 {
     new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
-    blockstore_meta_header_v1_t *new_hdr = (blockstore_meta_header_v1_t *)new_meta_buf;
+    blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
     new_hdr->zero = 0;
     new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-    new_hdr->version = BLOCKSTORE_META_VERSION_V1;
+    new_hdr->version = meta["version"].uint64_value() == BLOCKSTORE_META_FORMAT_V1
+        ? BLOCKSTORE_META_FORMAT_V1 : BLOCKSTORE_META_FORMAT_V2;
     new_hdr->meta_block_size = meta["meta_block_size"].uint64_value()
         ? meta["meta_block_size"].uint64_value() : 4096;
     new_hdr->data_block_size = meta["data_block_size"].uint64_value()
         ? meta["data_block_size"].uint64_value() : 131072;
     new_hdr->bitmap_granularity = meta["bitmap_granularity"].uint64_value()
         ? meta["bitmap_granularity"].uint64_value() : 4096;
-    new_clean_entry_bitmap_size = new_hdr->data_block_size / new_hdr->bitmap_granularity / 8;
-    new_clean_entry_size = sizeof(clean_disk_entry) + 2*new_clean_entry_bitmap_size;
+    new_hdr->data_csum_type = meta["data_csum_type"].is_number()
+        ? meta["data_csum_type"].uint64_value()
+        : (meta["data_csum_type"].string_value() == "crc32c"
+            ? BLOCKSTORE_CSUM_CRC32C
+            : BLOCKSTORE_CSUM_NONE);
+    new_hdr->csum_block_size = meta["csum_block_size"].uint64_value();
+    uint32_t new_clean_entry_header_size = (new_hdr->version == BLOCKSTORE_META_FORMAT_V1
+        ? sizeof(clean_disk_entry) : sizeof(clean_disk_entry) + 4 /*entry_csum*/);
+    new_clean_entry_bitmap_size = (new_hdr->data_block_size / new_hdr->bitmap_granularity + 7) / 8;
+    new_data_csum_size = (new_hdr->data_csum_type
+        ? ((new_hdr->data_block_size+new_hdr->csum_block_size-1)/new_hdr->csum_block_size*(new_hdr->data_csum_type & 0xFF))
+        : 0);
+    new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
     new_entries_per_block = new_hdr->meta_block_size / new_clean_entry_size;
     for (const auto & e: meta["entries"].array_items())
     {
@@ -194,8 +274,21 @@ int disk_tool_t::write_json_meta(json11::Json meta)
         new_entry->oid.inode = (sscanf_json(NULL, e["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, e["inode"]);
         new_entry->oid.stripe = sscanf_json(NULL, e["stripe"]);
         new_entry->version = sscanf_json(NULL, e["version"]);
-        fromhexstr(e["bitmap"].string_value(), new_clean_entry_bitmap_size, ((uint8_t*)new_entry) + sizeof(clean_disk_entry));
-        fromhexstr(e["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, ((uint8_t*)new_entry) + sizeof(clean_disk_entry) + new_clean_entry_bitmap_size);
+        fromhexstr(e["bitmap"].string_value(), new_clean_entry_bitmap_size,
+            ((uint8_t*)new_entry) + sizeof(clean_disk_entry));
+        fromhexstr(e["ext_bitmap"].string_value(), new_clean_entry_bitmap_size,
+            ((uint8_t*)new_entry) + sizeof(clean_disk_entry) + new_clean_entry_bitmap_size);
+        if (new_hdr->version == BLOCKSTORE_META_FORMAT_V2)
+        {
+            if (new_hdr->data_csum_type != 0)
+            {
+                fromhexstr(e["data_csum"].string_value(), new_data_csum_size,
+                    ((uint8_t*)new_entry) + sizeof(clean_disk_entry) + 2*new_clean_entry_bitmap_size);
+            }
+            uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + sizeof(clean_disk_entry) +
+                2*new_clean_entry_bitmap_size + new_data_csum_size);
+            *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
+        }
     }
     int r = resize_write_new_meta();
     free(new_meta_buf);

src/disk_tool_prepare.cpp

@@ -8,6 +8,9 @@
 int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd)
 {
     static const char *allow_additional_params[] = {
+        "cached_read_data",
+        "cached_read_meta",
+        "cached_read_journal",
         "max_write_iodepth",
         "max_write_iodepth",
         "min_flusher_count",
@@ -99,15 +102,16 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     if (options["journal_size"] == "")
     {
         if (options["journal_device"] == "")
-            options["journal_size"] = "32M";
+            options["journal_size"] = is_hdd ? "128M" : "32M";
         else if (is_hdd)
             options["journal_size"] = DEFAULT_HYBRID_JOURNAL;
     }
+    bool is_hybrid = is_hdd && options["journal_device"] != "" && options["journal_device"] != options["data_device"];
     if (is_hdd)
     {
         if (options["block_size"] == "")
             options["block_size"] = "1M";
-        if (options["throttle_small_writes"] == "")
+        if (is_hybrid && options["throttle_small_writes"] == "")
             options["throttle_small_writes"] = "1";
     }
     json11::Json::object sb;
@@ -115,6 +119,7 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     try
     {
         dsk.parse_config(options);
+        dsk.cached_read_data = dsk.cached_read_meta = dsk.cached_read_journal = false;
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
@@ -134,7 +139,7 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
             { "meta_offset", 4096 + (dsk.meta_device == dsk.journal_device ? dsk.journal_len : 0) },
             { "data_offset", 4096 + (dsk.data_device == dsk.meta_device ? dsk.meta_len : 0) +
                 (dsk.data_device == dsk.journal_device ? dsk.journal_len : 0) },
-            { "journal_no_same_sector_overwrites", true },
+            { "journal_no_same_sector_overwrites", !is_hdd || is_hybrid },
             { "journal_sector_buffer_count", 1024 },
             { "disable_data_fsync", json_is_true(options["disable_data_fsync"]) },
             { "disable_meta_fsync", json_is_true(options["disable_meta_fsync"]) },
@@ -478,6 +483,7 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
     {
         blockstore_disk_t dsk;
         dsk.parse_config(options);
+        dsk.cached_read_data = dsk.cached_read_meta = dsk.cached_read_journal = false;
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();

src/disk_tool_resize.cpp

@@ -29,7 +29,7 @@ int disk_tool_t::resize_data()
     fprintf(stderr, "Reading metadata\n");
     data_alloc = new allocator((new_data_len < dsk.data_len ? dsk.data_len : new_data_len) / dsk.data_block_size);
     r = process_meta(
-        [this](blockstore_meta_header_v1_t *hdr)
+        [this](blockstore_meta_header_v2_t *hdr)
         {
             resize_init(hdr);
         },
@@ -91,6 +91,7 @@ int disk_tool_t::resize_parse_params()
     try
     {
         dsk.parse_config(options);
+        dsk.cached_read_data = dsk.cached_read_meta = dsk.cached_read_journal = false;
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
@@ -139,7 +140,7 @@ int disk_tool_t::resize_parse_params()
     return 0;
 }
 
-void disk_tool_t::resize_init(blockstore_meta_header_v1_t *hdr)
+void disk_tool_t::resize_init(blockstore_meta_header_v2_t *hdr)
 {
     if (hdr && dsk.data_block_size != hdr->data_block_size)
     {
@@ -149,6 +150,15 @@ void disk_tool_t::resize_init(blockstore_meta_header_v1_t *hdr)
         }
         dsk.data_block_size = hdr->data_block_size;
     }
+    if (hdr && (dsk.data_csum_type != hdr->data_csum_type || dsk.csum_block_size != hdr->csum_block_size))
+    {
+        if (dsk.data_csum_type)
+        {
+            fprintf(stderr, "Using data checksum type %s from metadata superblock\n", csum_type_str(hdr->data_csum_type).c_str());
+        }
+        dsk.data_csum_type = hdr->data_csum_type;
+        dsk.csum_block_size = hdr->csum_block_size;
+    }
     if (((new_data_len-dsk.data_len) % dsk.data_block_size) ||
         ((new_data_offset-dsk.data_offset) % dsk.data_block_size))
     {
@@ -160,8 +170,12 @@ void disk_tool_t::resize_init(blockstore_meta_header_v1_t *hdr)
     free_last = (new_data_offset+new_data_len < dsk.data_offset+dsk.data_len)
         ? (dsk.data_offset+dsk.data_len-new_data_offset-new_data_len) / dsk.data_block_size
         : 0;
+    uint32_t new_clean_entry_header_size = sizeof(clean_disk_entry) + 4 /*entry_csum*/;
     new_clean_entry_bitmap_size = dsk.data_block_size / (hdr ? hdr->bitmap_granularity : 4096) / 8;
-    new_clean_entry_size = sizeof(clean_disk_entry) + 2 * new_clean_entry_bitmap_size;
+    new_data_csum_size = (dsk.data_csum_type
+        ? ((dsk.data_block_size+dsk.csum_block_size-1)/dsk.csum_block_size*(dsk.data_csum_type & 0xFF))
+        : 0);
+    new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
     new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
     uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
     if (!new_meta_len)
@@ -349,13 +363,25 @@ int disk_tool_t::resize_rewrite_journal()
         {
             if (je->type == JE_START)
             {
+                if (je_start.data_csum_type != dsk.data_csum_type ||
+                    je_start.csum_block_size != dsk.csum_block_size)
+                {
+                    fprintf(
+                        stderr, "Error: journal header has different checksum parameters: %s/%u vs %s/%u\n",
+                        csum_type_str(je_start.data_csum_type).c_str(), je_start.csum_block_size,
+                        csum_type_str(dsk.data_csum_type).c_str(), dsk.csum_block_size
+                    );
+                    exit(1);
+                }
                 journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
                 *((journal_entry_start*)ne) = (journal_entry_start){
                     .magic = JOURNAL_MAGIC,
                     .type = JE_START,
                     .size = sizeof(journal_entry_start),
                     .journal_start = dsk.journal_block_size,
-                    .version = JOURNAL_VERSION,
+                    .version = JOURNAL_VERSION_V2,
+                    .data_csum_type = dsk.data_csum_type,
+                    .csum_block_size = dsk.csum_block_size,
                 };
                 ne->crc32 = je_crc32(ne);
                 new_journal_ptr += dsk.journal_block_size;
@@ -436,15 +462,17 @@ int disk_tool_t::resize_rewrite_meta()
     new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
     int r = process_meta(
-        [this](blockstore_meta_header_v1_t *hdr)
+        [this](blockstore_meta_header_v2_t *hdr)
         {
-            blockstore_meta_header_v1_t *new_hdr = (blockstore_meta_header_v1_t *)new_meta_buf;
+            blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
             new_hdr->zero = 0;
             new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-            new_hdr->version = BLOCKSTORE_META_VERSION_V1;
+            new_hdr->version = BLOCKSTORE_META_FORMAT_V1;
             new_hdr->meta_block_size = dsk.meta_block_size;
             new_hdr->data_block_size = dsk.data_block_size;
             new_hdr->bitmap_granularity = dsk.bitmap_granularity ? dsk.bitmap_granularity : 4096;
+            new_hdr->data_csum_type = dsk.data_csum_type;
+            new_hdr->csum_block_size = dsk.csum_block_size;
         },
         [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
         {
@@ -463,7 +491,7 @@ int disk_tool_t::resize_rewrite_meta()
             new_entry->oid = entry->oid;
             new_entry->version = entry->version;
             if (bitmap)
-                memcpy(new_entry->bitmap, bitmap, 2*new_clean_entry_bitmap_size);
+                memcpy(new_entry->bitmap, bitmap, 2*new_clean_entry_bitmap_size + new_data_csum_size);
             else
                 memset(new_entry->bitmap, 0xff, 2*new_clean_entry_bitmap_size);
         }

src/disk_tool_utils.cpp

@@ -373,3 +373,22 @@ int fix_partition_type(std::string dev_by_uuid)
     std::string out;
     return shell_exec({ "sfdisk", "--no-reread", "--force", "/dev/"+parent_dev }, script, &out, NULL);
 }
+
+std::string csum_type_str(uint32_t data_csum_type)
+{
+    std::string csum_type;
+    if (data_csum_type == BLOCKSTORE_CSUM_NONE)
+        csum_type = "none";
+    else if (data_csum_type == BLOCKSTORE_CSUM_CRC32C)
+        csum_type = "crc32c";
+    else
+        csum_type = std::to_string(data_csum_type);
+    return csum_type;
+}
+
+uint32_t csum_type_from_str(std::string data_csum_type)
+{
+    if (data_csum_type == "crc32c")
+        return BLOCKSTORE_CSUM_CRC32C;
+    return stoull_full(data_csum_type, 0);
+}

src/etcd_state_client.cpp

@@ -187,22 +187,30 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
             check_addr = addr;
         if (pos == std::string::npos)
             addr += "/v3";
+        bool local = false;
         int i;
         for (i = 0; i < local_ips.size(); i++)
         {
             if (local_ips[i] == check_addr)
             {
-                this->etcd_local.push_back(addr);
+                local = true;
                 break;
             }
         }
-        if (i >= local_ips.size())
-            this->etcd_addresses.push_back(addr);
+        auto & to = local ? this->etcd_local : this->etcd_addresses;
+        for (i = 0; i < to.size(); i++)
+        {
+            if (to[i] == addr)
+                break;
+        }
+        if (i >= to.size())
+            to.push_back(addr);
     }
 }
 
 void etcd_state_client_t::parse_config(const json11::Json & config)
 {
+    this->etcd_local.clear();
     this->etcd_addresses.clear();
     if (config["etcd_address"].is_string())
     {
@@ -349,7 +357,7 @@ void etcd_state_client_t::start_etcd_watcher()
                         watch_id == ETCD_OSD_STATE_WATCH_ID)
                         etcd_watches_initialised++;
                     if (etcd_watches_initialised == 4 && this->log_level > 0)
-                        fprintf(stderr, "Successfully subscribed to etcd at %s\n", selected_etcd_address.c_str());
+                        fprintf(stderr, "Successfully subscribed to etcd at %s\n", cur_addr.c_str());
                 }
                 if (data["result"]["canceled"].bool_value())
                 {
@@ -360,15 +368,17 @@ void etcd_state_client_t::start_etcd_watcher()
                         // so we should restart from the beginning if we can
                         if (on_reload_hook != NULL)
                         {
-                            fprintf(stderr, "Revisions before %lu were compacted by etcd, reloading state\n",
-                                data["result"]["compact_revision"].uint64_value());
-                            if (etcd_watch_ws)
+                            // check to not trigger on_reload_hook multiple times
+                            if (etcd_watch_ws != NULL)
                             {
+                                fprintf(stderr, "Revisions before %lu were compacted by etcd, reloading state\n",
+                                    data["result"]["compact_revision"].uint64_value());
                                 http_close(etcd_watch_ws);
                                 etcd_watch_ws = NULL;
+                                etcd_watch_revision = 0;
+                                on_reload_hook();
                             }
-                            etcd_watch_revision = 0;
-                            on_reload_hook();
+                            return;
                         }
                         else
                         {
@@ -415,13 +425,9 @@ void etcd_state_client_t::start_etcd_watcher()
         }
         if (msg->eof)
         {
+            fprintf(stderr, "Disconnected from etcd %s\n", cur_addr.c_str());
             if (cur_addr == selected_etcd_address)
-            {
-                fprintf(stderr, "Disconnected from etcd %s\n", selected_etcd_address.c_str());
                 selected_etcd_address = "";
-            }
-            else
-                fprintf(stderr, "Disconnected from etcd\n");
             if (etcd_watch_ws)
             {
                 http_close(etcd_watch_ws);
@@ -438,6 +444,7 @@ void etcd_state_client_t::start_etcd_watcher()
             else if (etcd_watches_initialised > 0)
             {
                 // Connection was live, retry immediately
+                etcd_watches_initialised = 0;
                 start_etcd_watcher();
             }
         }

src/messenger.h

@@ -132,7 +132,6 @@ protected:
     uint64_t rdma_max_msg = 0;
 #endif
 
-    bool has_send_loop = false;
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
@@ -160,7 +159,6 @@ public:
     std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
     void read_requests();
     void send_replies();
-    bool can_send();
     void accept_connections(int listen_fd);
     ~osd_messenger_t();
 

src/msgr_send.cpp

@@ -111,28 +111,22 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         return;
     }
 #endif
-    if (!ringloop && !has_send_loop)
+    if (!ringloop)
     {
-        // "Send loop" should be used, like in QEMU driver, or performance will suffer
-        // due to a large number of sendmsg() syscalls where they can be avoided.
-        // "Send loop" increases 4k T1Q128 randread from ~55k iops to ~90k on a 1 OSD.
-        // The difference is smaller when there are more OSDs though...
+        // FIXME: It's worse because it doesn't allow batching
         while (cl->outbox.size())
         {
             try_send(cl);
         }
     }
-    else if (cl->write_msg.msg_iovlen > 0 || has_send_loop || !try_send(cl))
+    else if (cl->write_msg.msg_iovlen > 0 || !try_send(cl))
     {
         if (cl->write_state == 0)
         {
             cl->write_state = CL_WRITE_READY;
             write_ready_clients.push_back(cur_op->peer_fd);
         }
-        if (ringloop)
-        {
-            ringloop->wakeup();
-        }
+        ringloop->wakeup();
     }
 }
 
@@ -224,12 +218,6 @@ void osd_messenger_t::send_replies()
     write_ready_clients.clear();
 }
 
-bool osd_messenger_t::can_send()
-{
-    has_send_loop = true;
-    return write_ready_clients.size() > 0;
-}
-
 void osd_messenger_t::handle_send(int result, osd_client_t *cl)
 {
     cl->write_msg.msg_iovlen = 0;

src/osd.h

@@ -198,13 +198,14 @@ class osd_t
     void on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num);
     void on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes);
     void on_load_config_hook(json11::Json::object & changes);
+    void on_reload_config_hook(json11::Json::object & changes);
     json11::Json on_load_pgs_checks_hook();
     void on_load_pgs_hook(bool success);
     void bind_socket();
     void acquire_lease();
     json11::Json get_osd_state();
     void create_osd_state();
-    void renew_lease();
+    void renew_lease(bool reload);
     void print_stats();
     void print_slow();
     void reset_stats();

src/osd_cluster.cpp

@@ -70,6 +70,7 @@ void osd_t::init_cluster()
         st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
         st_cli.load_pgs_checks_hook = [this]() { return on_load_pgs_checks_hook(); };
         st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
+        st_cli.on_reload_hook = [this]() { st_cli.load_global_config(); };
         peering_state = OSD_LOADING_PGS;
         st_cli.load_global_config();
     }
@@ -395,6 +396,14 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config)
     parse_config(true);
     bind_socket();
     acquire_lease();
+    st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_reload_config_hook(cfg); };
+}
+
+void osd_t::on_reload_config_hook(json11::Json::object & global_config)
+{
+    etcd_global_config = global_config;
+    parse_config(false);
+    renew_lease(true);
 }
 
 // Acquire lease
@@ -424,7 +433,7 @@ void osd_t::acquire_lease()
     );
     tfd->set_timer(etcd_report_interval*1000, true, [this](int timer_id)
     {
-        renew_lease();
+        renew_lease(false);
     });
 }
 
@@ -499,11 +508,11 @@ void osd_t::create_osd_state()
 }
 
 // Renew lease
-void osd_t::renew_lease()
+void osd_t::renew_lease(bool reload)
 {
     st_cli.etcd_call("/lease/keepalive", json11::Json::object {
         { "ID", etcd_lease_id }
-    }, st_cli.etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data)
+    }, st_cli.etcd_quick_timeout, 0, 0, [this, reload](std::string err, json11::Json data)
     {
         if (err == "" && data["result"]["TTL"].string_value() == "")
         {
@@ -522,15 +531,20 @@ void osd_t::renew_lease()
                 force_stop(1);
             }
             // Retry
-            tfd->set_timer(st_cli.etcd_quick_timeout, false, [this](int timer_id)
+            tfd->set_timer(st_cli.etcd_quick_timeout, false, [this, reload](int timer_id)
             {
-                renew_lease();
+                renew_lease(reload);
             });
         }
         else
         {
             etcd_failed_attempts = 0;
             report_statistics();
+            // Reload PGs
+            if (reload && run_primary)
+            {
+                st_cli.load_pgs();
+            }
         }
     });
 }
@@ -560,7 +574,6 @@ void osd_t::force_stop(int exitcode)
 
 json11::Json osd_t::on_load_pgs_checks_hook()
 {
-    assert(this->pgs.size() == 0);
     json11::Json::array checks = {
         json11::Json::object {
             { "target", "LEASE" },

src/osd_primary_write.cpp

@@ -145,6 +145,11 @@ resume_3:
             // Mark object corrupted and retry
             op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true, false);
             op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
+            if (cur_op->rmw_buf)
+            {
+                free(cur_op->rmw_buf);
+                cur_op->rmw_buf = NULL;
+            }
             goto retry_1;
         }
         deref_object_state(pg, &op_data->object_state, true);

src/qemu_driver.c

@@ -58,6 +58,8 @@ typedef struct VitastorFdData VitastorFdData;
 typedef struct VitastorClient
 {
     void *proxy;
+    int uring_eventfd;
+
     void *watch;
     char *config_path;
     char *etcd_host;
@@ -77,7 +79,7 @@ typedef struct VitastorClient
     AioContext *ctx;
     VitastorFdData **fds;
     int fd_count, fd_alloc;
-    int bh_send_all_scheduled;
+    int bh_uring_scheduled;
 
     uint64_t last_bitmap_inode, last_bitmap_offset, last_bitmap_len;
     uint32_t last_bitmap_granularity;
@@ -232,47 +234,51 @@ out:
     return;
 }
 
-#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
-static void vitastor_bh_send_all(void *opaque)
+static void vitastor_uring_handler(void *opaque)
 {
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
-    VitastorBH *vbh = opaque;
-    VitastorClient *client = vbh->cli;
-#else
-    VitastorClient *client = opaque;
-#endif
+    VitastorClient *client = (VitastorClient*)opaque;
     qemu_mutex_lock(&client->mutex);
-    client->bh_send_all_scheduled = 0;
-    vitastor_c_qemu_send_all(client->proxy);
+    client->bh_uring_scheduled = 0;
+    do
+    {
+        vitastor_c_uring_handle_events(client->proxy);
+    } while (vitastor_c_uring_has_work(client->proxy));
     qemu_mutex_unlock(&client->mutex);
+}
+
 #if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
+static void vitastor_bh_uring_handler(void *opaque)
+{
+    VitastorBH *vbh = opaque;
+    vitastor_bh_handler(vbh->cli);
     qemu_bh_delete(vbh->bh);
     free(vbh);
-#endif
 }
 #endif
 
-static void vitastor_schedule_send_all(VitastorClient *client)
+static void vitastor_schedule_uring_handler(VitastorClient *client)
 {
-#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
-    if (!client->bh_send_all_scheduled)
+    void *opaque = client;
+    if (client->uring_eventfd >= 0 && !client->bh_uring_scheduled)
     {
-        client->bh_send_all_scheduled = 1;
+        client->bh_uring_scheduled = 1;
 #if QEMU_VERSION_MAJOR > 4 || QEMU_VERSION_MAJOR == 4 && QEMU_VERSION_MINOR >= 2
-        replay_bh_schedule_oneshot_event(client->ctx, vitastor_bh_send_all, client);
+        replay_bh_schedule_oneshot_event(client->ctx, vitastor_uring_handler, opaque);
 #elif QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 8
-        aio_bh_schedule_oneshot(client->ctx, vitastor_bh_send_all, client);
+        aio_bh_schedule_oneshot(client->ctx, vitastor_uring_handler, opaque);
 #elif QEMU_VERSION_MAJOR >= 2
         VitastorBH *vbh = (VitastorBH*)malloc(sizeof(VitastorBH));
         vbh->cli = client;
-        vbh->bh = aio_bh_new(bdrv_get_aio_context(task->bs), vitastor_bh_send_all, vbh);
+        vbh->bh = aio_bh_new(bdrv_get_aio_context(task->bs), vitastor_bh_uring_handler, vbh);
         qemu_bh_schedule(vbh->bh);
 #else
-        client->bh_send_all_scheduled = 0;
-        vitastor_c_qemu_send_all(client->proxy);
+        client->bh_uring_scheduled = 0;
+        do
+        {
+            vitastor_c_uring_handle_events(client->proxy);
+        } while (vitastor_c_uring_has_work(client->proxy));
 #endif
     }
-#endif
 }
 
 static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
@@ -283,7 +289,7 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
-    vitastor_schedule_send_all(client);
+    vitastor_schedule_uring_handler(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task->complete)
@@ -297,7 +303,7 @@ static void vitastor_aio_fd_read(void *fddv)
     VitastorFdData *fdd = (VitastorFdData*)fddv;
     qemu_mutex_lock(&fdd->cli->mutex);
     fdd->fd_read(fdd->opaque);
-    vitastor_schedule_send_all(fdd->cli);
+    vitastor_schedule_uring_handler(fdd->cli);
     qemu_mutex_unlock(&fdd->cli->mutex);
 }
 
@@ -306,10 +312,30 @@ static void vitastor_aio_fd_write(void *fddv)
     VitastorFdData *fdd = (VitastorFdData*)fddv;
     qemu_mutex_lock(&fdd->cli->mutex);
     fdd->fd_write(fdd->opaque);
-    vitastor_schedule_send_all(fdd->cli);
+    vitastor_schedule_uring_handler(fdd->cli);
     qemu_mutex_unlock(&fdd->cli->mutex);
 }
 
+static void universal_aio_set_fd_handler(AioContext *ctx, int fd, IOHandler *fd_read, IOHandler *fd_write, void *opaque)
+{
+    aio_set_fd_handler(ctx, fd,
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 5 || QEMU_VERSION_MAJOR >= 3
+        0 /*is_external*/,
+#endif
+        fd_read,
+        fd_write,
+#if QEMU_VERSION_MAJOR == 1 && QEMU_VERSION_MINOR <= 6 || QEMU_VERSION_MAJOR < 1
+        NULL /*io_flush*/,
+#endif
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 9 || QEMU_VERSION_MAJOR >= 3
+        NULL /*io_poll*/,
+#endif
+#if QEMU_VERSION_MAJOR >= 7
+        NULL /*io_poll_ready*/,
+#endif
+        opaque);
+}
+
 static void vitastor_aio_set_fd_handler(void *vcli, int fd, int unused1, IOHandler *fd_read, IOHandler *fd_write, void *unused2, void *opaque)
 {
     VitastorClient *client = (VitastorClient*)vcli;
@@ -352,22 +378,9 @@ static void vitastor_aio_set_fd_handler(void *vcli, int fd, int unused1, IOHandl
         }
         client->fds[client->fd_count++] = fdd;
     }
-    aio_set_fd_handler(client->ctx, fd,
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 5 || QEMU_VERSION_MAJOR >= 3
-        0 /*is_external*/,
-#endif
-        fd_read ? vitastor_aio_fd_read : NULL,
-        fd_write ? vitastor_aio_fd_write : NULL,
-#if QEMU_VERSION_MAJOR == 1 && QEMU_VERSION_MINOR <= 6 || QEMU_VERSION_MAJOR < 1
-        NULL /*io_flush*/,
-#endif
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 9 || QEMU_VERSION_MAJOR >= 3
-        NULL /*io_poll*/,
-#endif
-#if QEMU_VERSION_MAJOR >= 7
-        NULL /*io_poll_ready*/,
-#endif
-        fdd);
+    universal_aio_set_fd_handler(
+        client->ctx, fd, fd_read ? vitastor_aio_fd_read : NULL, fd_write ? vitastor_aio_fd_write : NULL, fdd
+    );
 }
 
 static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, Error **errp)
@@ -388,10 +401,38 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
     client->rdma_gid_index = qdict_get_try_int(options, "rdma-gid-index", 0);
     client->rdma_mtu = qdict_get_try_int(options, "rdma-mtu", 0);
     client->ctx = bdrv_get_aio_context(bs);
-    client->proxy = vitastor_c_create_qemu(
+#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
+    client->proxy = vitastor_c_create_qemu_uring(
         vitastor_aio_set_fd_handler, client, client->config_path, client->etcd_host, client->etcd_prefix,
         client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0
     );
+#else
+    client->proxy = vitastor_c_create_uring(
+        client->config_path, client->etcd_host, client->etcd_prefix,
+        client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0
+    );
+#endif
+    if (!client->proxy)
+    {
+        fprintf(stderr, "vitastor: failed to create io_uring: %s - I/O will be slower\n", strerror(errno));
+        client->uring_eventfd = -1;
+        client->proxy = vitastor_c_create_qemu(
+            vitastor_aio_set_fd_handler, client, client->config_path, client->etcd_host, client->etcd_prefix,
+            client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0
+        );
+    }
+    else
+    {
+        client->uring_eventfd = vitastor_c_uring_register_eventfd(client->proxy);
+        if (client->uring_eventfd < 0)
+        {
+            fprintf(stderr, "vitastor: failed to create io_uring eventfd: %s\n", strerror(errno));
+            error_setg(errp, "failed to create io_uring eventfd");
+            vitastor_close(bs);
+            return -1;
+        }
+        universal_aio_set_fd_handler(client->ctx, client->uring_eventfd, vitastor_uring_handler, NULL, client);
+    }
     image = client->image = g_strdup(qdict_get_try_str(options, "image"));
     client->readonly = (flags & BDRV_O_RDWR) ? 1 : 0;
     // Get image metadata (size and readonly flag) or just wait until the client is ready
@@ -648,7 +689,7 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
-    vitastor_schedule_send_all(client);
+    vitastor_schedule_uring_handler(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -682,7 +723,7 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
-    vitastor_schedule_send_all(client);
+    vitastor_schedule_uring_handler(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -761,7 +802,7 @@ static int coroutine_fn vitastor_co_block_status(
         task.bitmap = client->last_bitmap = NULL;
         qemu_mutex_lock(&client->mutex);
         vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
-        vitastor_schedule_send_all(client);
+        vitastor_schedule_uring_handler(client);
         qemu_mutex_unlock(&client->mutex);
         while (!task.complete)
         {
@@ -848,7 +889,7 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
-    vitastor_schedule_send_all(client);
+    vitastor_schedule_uring_handler(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)

src/ringloop.cpp

@@ -2,9 +2,12 @@
 // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <stdlib.h>
+#include <unistd.h>
 
 #include <stdexcept>
 
+#include <sys/eventfd.h>
+
 #include "ringloop.h"
 
 ring_loop_t::ring_loop_t(int qd)
@@ -32,6 +35,10 @@ ring_loop_t::~ring_loop_t()
     free(free_ring_data);
     free(ring_datas);
     io_uring_queue_exit(&ring);
+    if (ring_eventfd)
+    {
+        close(ring_eventfd);
+    }
 }
 
 void ring_loop_t::register_consumer(ring_consumer_t *consumer)
@@ -127,3 +134,24 @@ int ring_loop_t::sqes_left()
     }
     return left;
 }
+
+int ring_loop_t::register_eventfd()
+{
+    if (ring_eventfd >= 0)
+    {
+        return ring_eventfd;
+    }
+    ring_eventfd = eventfd(0, EFD_CLOEXEC|EFD_NONBLOCK);
+    if (ring_eventfd < 0)
+    {
+        return -errno;
+    }
+    int r = io_uring_register_eventfd(&ring, ring_eventfd);
+    if (r < 0)
+    {
+        close(ring_eventfd);
+        ring_eventfd = -1;
+        return r;
+    }
+    return ring_eventfd;
+}

src/ringloop.h

@@ -126,11 +126,13 @@ class ring_loop_t
     unsigned free_ring_data_ptr;
     bool loop_again;
     struct io_uring ring;
+    int ring_eventfd = -1;
 public:
     ring_loop_t(int qd);
     ~ring_loop_t();
     void register_consumer(ring_consumer_t *consumer);
     void unregister_consumer(ring_consumer_t *consumer);
+    int register_eventfd();
 
     inline struct io_uring_sqe* get_sqe()
     {

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.9.2
+Version: 0.9.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/vitastor_c.cpp

@@ -5,6 +5,7 @@
 // Also acts as a C-C++ proxy for the QEMU driver (QEMU headers don't compile with g++)
 
 #include <sys/epoll.h>
+#include <sys/eventfd.h>
 
 #include "ringloop.h"
 #include "epoll_manager.h"
@@ -25,6 +26,7 @@ struct vitastor_c
     epoll_manager_t *epmgr = NULL;
     timerfd_manager_t *tfd = NULL;
     cluster_client_t *cli = NULL;
+    int uring_eventfd = -1;
 
     QEMUSetFDHandler *aio_set_fd_handler = NULL;
     void *aio_ctx = NULL;
@@ -70,14 +72,8 @@ static void vitastor_c_write_handler(void *opaque)
     data->callback(data->fd, EPOLLOUT);
 }
 
-vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
-    const char *config_path, const char *etcd_host, const char *etcd_prefix,
-    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+static vitastor_c *vitastor_c_create_qemu_common(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context)
 {
-    json11::Json cfg_json = vitastor_c_common_config(
-        config_path, etcd_host, etcd_prefix, use_rdma,
-        rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
-    );
     vitastor_c *self = new vitastor_c;
     self->aio_set_fd_handler = aio_set_fd_handler;
     self->aio_ctx = aio_context;
@@ -95,24 +91,77 @@ vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *a
             self->aio_set_fd_handler(self->aio_ctx, fd, false, NULL, NULL, NULL, NULL);
         }
     });
-    self->cli = new cluster_client_t(NULL, self->tfd, cfg_json);
     return self;
 }
 
-vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix,
+vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
     int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
 {
+    json11::Json cfg_json = vitastor_c_common_config(
+        config_path, etcd_host, etcd_prefix, use_rdma,
+        rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+    );
+    auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
+    self->cli = new cluster_client_t(NULL, self->tfd, cfg_json);
+    return self;
+}
+
+vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+{
+    ring_loop_t *ringloop = NULL;
+    try
+    {
+        ringloop = new ring_loop_t(512);
+    }
+    catch (std::exception & e)
+    {
+        return NULL;
+    }
+    json11::Json cfg_json = vitastor_c_common_config(
+        config_path, etcd_host, etcd_prefix, use_rdma,
+        rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+    );
+    auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
+    self->ringloop = ringloop;
+    self->cli = new cluster_client_t(self->ringloop, self->tfd, cfg_json);
+    return self;
+}
+
+vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+{
+    ring_loop_t *ringloop = NULL;
+    try
+    {
+        ringloop = new ring_loop_t(512);
+    }
+    catch (std::exception & e)
+    {
+        return NULL;
+    }
     json11::Json cfg_json = vitastor_c_common_config(
         config_path, etcd_host, etcd_prefix, use_rdma,
         rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
     );
     vitastor_c *self = new vitastor_c;
-    self->ringloop = new ring_loop_t(512);
+    self->ringloop = ringloop;
     self->epmgr = new epoll_manager_t(self->ringloop);
     self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
     return self;
 }
 
+int vitastor_c_uring_register_eventfd(vitastor_c *client)
+{
+    if (!client->ringloop)
+    {
+        return -EINVAL;
+    }
+    return client->ringloop->register_eventfd();
+}
+
 vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
 {
     json11::Json::object cfg;
@@ -166,12 +215,9 @@ void vitastor_c_uring_wait_events(vitastor_c *client)
     client->ringloop->wait();
 }
 
-void vitastor_c_qemu_send_all(vitastor_c *client)
+bool vitastor_c_uring_has_work(vitastor_c *client)
 {
-    while (client->cli->msgr.can_send())
-    {
-        client->cli->msgr.send_replies();
-    }
+    return client->ringloop->has_work();
 }
 
 void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,

src/vitastor_c.h

@@ -34,15 +34,19 @@ typedef void QEMUSetFDHandler(void *ctx, int fd, int is_external, IOHandler *fd_
 vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
     const char *config_path, const char *etcd_host, const char *etcd_prefix,
     int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
+vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
 vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix,
     int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
 vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len);
 void vitastor_c_destroy(vitastor_c *client);
 int vitastor_c_is_ready(vitastor_c *client);
+int vitastor_c_uring_register_eventfd(vitastor_c *client);
 void vitastor_c_uring_wait_ready(vitastor_c *client);
 void vitastor_c_uring_handle_events(vitastor_c *client);
 void vitastor_c_uring_wait_events(vitastor_c *client);
-void vitastor_c_qemu_send_all(vitastor_c *client);
+bool vitastor_c_uring_has_work(vitastor_c *client);
 void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
     struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,

tests/run_3osds.sh

@@ -27,7 +27,9 @@ fi
 start_osd()
 {
     local i=$1
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-disk simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
+    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
+        $(build/src/vitastor-disk simple-offsets --format options ./testdata/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) \
+        >>./testdata/osd$i.log 2>&1 &
     eval OSD${i}_PID=$!
 }
 
@@ -82,7 +84,9 @@ wait_up()
     done
 }
 
-wait_up 60
+if [[ $OSD_COUNT -gt 0 ]]; then
+    wait_up 60
+fi
 
 try_reweight()
 {

tests/run_tests.sh

@@ -53,6 +53,13 @@ SCHEME=xor ./test_write.sh
 PG_SIZE=2 ./test_heal.sh
 SCHEME=ec ./test_heal.sh
 
+TEST_NAME=csum_32k_dmj OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
+TEST_NAME=csum_32k_dj  OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
+TEST_NAME=csum_32k     OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
+TEST_NAME=csum_4k_dmj  OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
+TEST_NAME=csum_4k_dj   OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
+TEST_NAME=csum_4k      OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
+
 ./test_scrub.sh
 ZERO_OSD=2 ./test_scrub.sh
 SCHEME=xor ./test_scrub.sh

tests/test_checksum.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -ex
+
+OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false $OSD_ARGS"
+OFFSET_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false $OFFSET_ARGS"
+PG_COUNT=${PG_COUNT:-64}
+. `dirname $0`/run_3osds.sh
+check_qemu
+
+IMG_SIZE=128
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
+
+# Write
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
+        -mirror_file=./testdata/mirror.bin -end_fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg -runtime=10
+
+# Intentionally corrupt OSD data and restart it
+kill $OSD1_PID
+data_offset=$(build/src/vitastor-disk simple-offsets ./testdata/test_osd1.bin $OFFSET_ARGS | grep data_offset | awk '{print $2}')
+truncate -s $data_offset ./testdata/test_osd1.bin
+dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+start_osd 1
+
+# FIXME: corrupt the journal WHEN OSD IS RUNNING and check reads too
+
+# Wait until start
+wait_up 10
+
+# Read everything back
+qemu-img convert -S 4096 -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
+    -O raw ./testdata/read.bin
+
+diff ./testdata/read.bin ./testdata/mirror.bin
+
+format_green OK

tests/test_heal.sh

@@ -12,6 +12,7 @@ PG_COUNT=32
 . `dirname $0`/run_3osds.sh
 check_qemu
 
+# FIXME: Fix space rebalance priorities :)
 IMG_SIZE=960
 
 $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
@@ -24,6 +25,7 @@ kill_osds()
 {
     sleep 5
 
+    echo Killing OSD 1
     kill -9 $OSD1_PID
     $ETCDCTL del /vitastor/osd/state/1
 
@@ -38,6 +40,7 @@ kill_osds()
     done
 
     sleep 5
+    echo Starting OSD 7
     start_osd 7
 
     sleep 5
@@ -46,13 +49,19 @@ kill_osds()
 kill_osds &
 
 LD_PRELOAD="build/src/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bsrange=4k-128k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
         -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
 
 qemu-img convert -S 4096 -p \
     -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
     -O raw ./testdata/read.bin
 
-diff ./testdata/read.bin ./testdata/mirror.bin
+if ! diff -q ./testdata/read.bin ./testdata/mirror.bin; then
+    format_error Data lost during self-heal
+fi
+
+if grep -qP 'Checksum mismatch|BUG' ./testdata/osd*.log; then
+    format_error Checksum mismatches or BUGs detected during test
+fi
 
 format_green OK

tests/test_move_reappear.sh

@@ -4,7 +4,7 @@
 
 OSD_SIZE=1024
 OSD_COUNT=5
-OSD_ARGS=
+OSD_ARGS="$OSD_ARGS"
 for i in $(seq 1 $OSD_COUNT); do
     dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
     build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-disk simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) >>./testdata/osd$i.log 2>&1 &

tests/test_vm_cont.sh

@@ -8,7 +8,8 @@ etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/pg/state
 etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/osd/state
 
 OSD_COUNT=3
-OSD_ARGS=
+OSD_ARGS="$OSD_ARGS"
+OFFSET_ARGS="$OFFSET_ARGS"
 for i in $(seq 1 $OSD_COUNT); do
     build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-disk simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
     eval OSD${i}_PID=$!

tests/test_write_no_same.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 # Test the `no_same_sector_overwrites` mode
 
-OSD_ARGS="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all"
+OSD_ARGS="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all $OSD_ARGS"
 GLOBAL_CONF='{"immediate_commit":"all"}'
 
 . `dirname $0`/run_3osds.sh