Do not save watch revision from initial created: true messages

.gitea/workflows/test.yml

@@ -1206,6 +1206,78 @@ jobs:
           echo ""
         done
 
+  test_checksum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_checksum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_checksum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_checksum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_corrupt_all:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_corrupt_all.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_corrupt_all:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_reweight_half:
     runs-on: ubuntu-latest
     needs: build

csi/Dockerfile

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:bookworm AS build
+FROM golang:trixie AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:bookworm
+FROM debian:trixie
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -32,13 +32,13 @@ RUN apt-get update && \
 
 COPY --from=build /app/vitastor-csi /bin/
 
-RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
+RUN (echo deb http://vitastor.io/debian trixie main > /etc/apt/sources.list.d/vitastor.list) && \
     ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
     apt-get install -y vitastor-client ibverbs-providers && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-utils_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-block-extra_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

docker/Dockerfile

@@ -1,6 +1,6 @@
 # Build Docker image with Vitastor packages
 
-FROM debian:bookworm
+FROM debian:trixie
 
 ADD etc/apt /etc/apt/
 RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean

docs/config/network.en.md

@@ -22,7 +22,6 @@ between clients, OSDs and etcd.
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
-- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -102,11 +101,6 @@ found or if `osd_network` is not specified. Auto-selection is also
 unsupported with old libibverbs < v32, like in Debian 10 Buster or
 CentOS 7.
 
-Vitastor supports all adapters, even ones without ODP support, like
-Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
-1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
-See also [rdma_odp](#rdma_odp).
-
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
 
@@ -116,6 +110,23 @@ the manual of your network vendor for details about setting up the switch
 for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
 PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
 
+Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
+support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
+in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
+for RDMA and thus, in theory, avoid memory copying.
+
+Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
+but it was still supported up to 3.0.3. Now ODP support is removed because it
+actually only hurts performance: an example 3-node cluster with 8 NVMe in each
+node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
+ODP, but only 239000 iops with ODP.
+
+This happens because Mellanox ODP implementation seems to be based on
+message retransmissions when the adapter doesn't know about the buffer yet -
+it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+which is generally slow in RDMA/RoCE networks. Here's a presentation about
+it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
 ## rdma_port_num
 
 - Type: integer
@@ -187,28 +198,6 @@ less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
 Doesn't affect memory usage - additional memory isn't allocated for send
 operations.
 
-## rdma_odp
-
-- Type: boolean
-- Default: false
-
-Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
-ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
-for RDMA adapter to be able to use it. This, in turn, allows to skip memory
-copying during sending. One would think this should improve performance, but
-**in reality** RDMA performance with ODP is **drastically** worse. Example
-3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
-without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
-
-This happens because Mellanox ODP implementation seems to be based on
-message retransmissions when the adapter doesn't know about the buffer yet -
-it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-which is generally slow in RDMA/RoCE networks. Here's a presentation about
-it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-ODP support is retained in the code just in case a good ODP implementation
-appears one day.
-
 ## peer_connect_interval
 
 - Type: seconds

docs/config/network.ru.md

@@ -22,7 +22,6 @@
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
-- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -101,12 +100,6 @@ RoCEv1/RoCEv2, и даже позволяет полностью отключи
 не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
 libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-Vitastor поддерживает все модели адаптеров, включая те, у которых
-нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
-требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
-См. также [rdma_odp](#rdma_odp).
-
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
 
@@ -117,6 +110,24 @@ Vitastor поддерживает все модели адаптеров, вкл
 подразумевает настройку сети без потерь на основе PFC (Priority Flow
 Control) и ECN (Explicit Congestion Notification).
 
+Vitastor поддерживает все модели адаптеров, включая те, у которых нет
+поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
+не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
+более новых и позволяет не регистрировать память для её использования RDMA-картой,
+благодаря чему в теории можно избежать лишних копирований памяти.
+
+Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
+по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
+полностью удалена, так как на самом деле она только портит производительность:
+например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
+чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
+
+Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+основана на повторной передаче сообщений, когда карте не известен буфер -
+вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
 ## rdma_port_num
 
 - Тип: целое число
@@ -192,29 +203,6 @@ OSD в любом случае согласовывают реальное зн
 Не влияет на потребление памяти - дополнительная память на операции отправки
 не выделяется.
 
-## rdma_odp
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
-исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
-не регистрировать память для её использования RDMA-картой. Благодаря этому
-можно не копировать данные при отправке их в сеть и, казалось бы, это должно
-улучшать производительность - но **по факту** получается так, что
-производительность только ухудшается, причём сильно. Пример - на 3-узловом
-кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
-удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
-
-Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-основана на повторной передаче сообщений, когда карте не известен буфер -
-вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-Возможность использования ODP сохранена в коде на случай, если вдруг в один
-прекрасный день появится хорошая реализация ODP.
-
 ## peer_connect_interval
 
 - Тип: секунды

docs/config/src/network.yml

@@ -84,11 +84,6 @@
     unsupported with old libibverbs < v32, like in Debian 10 Buster or
     CentOS 7.
 
-    Vitastor supports all adapters, even ones without ODP support, like
-    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
-    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
-    See also [rdma_odp](#rdma_odp).
-
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
 
@@ -97,6 +92,23 @@
     the manual of your network vendor for details about setting up the switch
     for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
+
+    Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
+    support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
+    in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
+    for RDMA and thus, in theory, avoid memory copying.
+
+    Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
+    but it was still supported up to 3.0.3. Now ODP support is removed because it
+    actually only hurts performance: an example 3-node cluster with 8 NVMe in each
+    node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
+    ODP, but only 239000 iops with ODP.
+
+    This happens because Mellanox ODP implementation seems to be based on
+    message retransmissions when the adapter doesn't know about the buffer yet -
+    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+    which is generally slow in RDMA/RoCE networks. Here's a presentation about
+    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
     Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
@@ -105,12 +117,6 @@
     не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
     libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-    Vitastor поддерживает все модели адаптеров, включая те, у которых
-    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
-    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
-    См. также [rdma_odp](#rdma_odp).
-
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
 
@@ -120,6 +126,24 @@
     коммутатора для RoCEv2 ищите в документации производителя. Обычно это
     подразумевает настройку сети без потерь на основе PFC (Priority Flow
     Control) и ECN (Explicit Congestion Notification).
+
+    Vitastor поддерживает все модели адаптеров, включая те, у которых нет
+    поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
+    не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
+    более новых и позволяет не регистрировать память для её использования RDMA-картой,
+    благодаря чему в теории можно избежать лишних копирований памяти.
+
+    Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
+    по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
+    полностью удалена, так как на самом деле она только портит производительность:
+    например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
+    чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
+
+    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+    основана на повторной передаче сообщений, когда карте не известен буфер -
+    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
 - name: rdma_port_num
   type: int
   info: |
@@ -218,45 +242,6 @@
     у принимающей стороны в процессе работы не заканчивались буферы на приём.
     Не влияет на потребление памяти - дополнительная память на операции отправки
     не выделяется.
-- name: rdma_odp
-  type: bool
-  default: false
-  online: false
-  info: |
-    Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
-    ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
-    for RDMA adapter to be able to use it. This, in turn, allows to skip memory
-    copying during sending. One would think this should improve performance, but
-    **in reality** RDMA performance with ODP is **drastically** worse. Example
-    3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
-    without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
-
-    This happens because Mellanox ODP implementation seems to be based on
-    message retransmissions when the adapter doesn't know about the buffer yet -
-    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-    which is generally slow in RDMA/RoCE networks. Here's a presentation about
-    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-    ODP support is retained in the code just in case a good ODP implementation
-    appears one day.
-  info_ru: |
-    Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
-    исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
-    не регистрировать память для её использования RDMA-картой. Благодаря этому
-    можно не копировать данные при отправке их в сеть и, казалось бы, это должно
-    улучшать производительность - но **по факту** получается так, что
-    производительность только ухудшается, причём сильно. Пример - на 3-узловом
-    кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
-    удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
-
-    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-    основана на повторной передаче сообщений, когда карте не известен буфер -
-    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-    Возможность использования ODP сохранена в коде на случай, если вдруг в один
-    прекрасный день появится хорошая реализация ODP.
 - name: peer_connect_interval
   type: sec
   min: 1

rpm/build-tarball.sh

@@ -21,7 +21,7 @@ rpmbuild -bp fio.spec
 cd $VITASTOR
 VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
 rm -rf fio
-ln -s ~/rpmbuild/BUILD/fio*/ fio
+ln -s $(ls -d ~/rpmbuild/BUILD/fio*/ | grep -v SPECPARTS) fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio

rpm/vitastor-el10.Dockerfile

@@ -0,0 +1,17 @@
+# Build packages for AlmaLinux 10 inside a container
+# cd ..
+# docker pull --platform=linux/amd64/v2 quay.io/almalinuxorg/almalinux:10
+# docker build -t vitastor-buildenv:el10 -f rpm/vitastor-el10.Dockerfile .
+# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el10 /root/vitastor/rpm/vitastor-build.sh
+
+FROM quay.io/almalinuxorg/almalinux:10
+
+WORKDIR /root
+
+RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
+RUN dnf -y install epel-release dnf-plugins-core
+RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
+RUN dnf download --source fio
+RUN rpm --nomd5 -i fio*.src.rpm
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el10.spec

@@ -0,0 +1,198 @@
+Name:           vitastor
+Version:        3.0.3
+Release:        1%{?dist}
+Summary:        Vitastor, a fast software-defined clustered block storage
+
+License:        Vitastor Network Public License 1.1
+URL:            https://vitastor.io/
+Source0:        vitastor-3.0.3.el10.tar.gz
+
+BuildRequires:  gperftools-devel
+BuildRequires:  gcc-c++
+BuildRequires:  nodejs >= 10
+BuildRequires:  jerasure-devel
+BuildRequires:  isa-l-devel
+BuildRequires:  gf-complete-devel
+BuildRequires:  rdma-core-devel
+BuildRequires:  cmake
+BuildRequires:  libnl3-devel
+Requires:       vitastor-osd = %{version}-%{release}
+Requires:       vitastor-mon = %{version}-%{release}
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       vitastor-client-devel = %{version}-%{release}
+Requires:       vitastor-fio = %{version}-%{release}
+
+%description
+Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+architecturally similar to Ceph which means strong consistency, primary-replication,
+symmetric clustering and automatic data distribution over any number of drives of any
+size with configurable redundancy (replication or erasure codes/XOR).
+
+
+%package -n vitastor-osd
+Summary:        Vitastor - OSD
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       util-linux
+Requires:       parted
+
+
+%description -n vitastor-osd
+Vitastor object storage daemon, i.e. server program that stores data.
+
+
+%package -n vitastor-mon
+Summary:        Vitastor - monitor
+Requires:       nodejs >= 10
+Requires:       lpsolve
+
+
+%description -n vitastor-mon
+Vitastor monitor, i.e. server program responsible for watching cluster state and
+scheduling cluster-level operations.
+
+
+%package -n vitastor-client
+Summary:        Vitastor - client
+
+
+%description -n vitastor-client
+Vitastor client library and command-line interface.
+
+
+%package -n vitastor-client-devel
+Summary:        Vitastor - development files
+Group:          Development/Libraries
+Requires:       vitastor-client = %{version}-%{release}
+
+
+%description -n vitastor-client-devel
+Vitastor library headers for development.
+
+
+%package -n vitastor-fio
+Summary:        Vitastor - fio drivers
+Group:          Development/Libraries
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       fio = 3.36-5.el10
+
+
+%description -n vitastor-fio
+Vitastor fio drivers for benchmarking.
+
+
+%package -n vitastor-opennebula
+Summary:        Vitastor for OpenNebula
+Group:          Development/Libraries
+Requires:       vitastor-client
+Requires:       jq
+Requires:       python3-lxml
+Requires:       patch
+Requires:       qemu-kvm-block-vitastor
+
+
+%description -n vitastor-opennebula
+Vitastor storage plugin for OpenNebula.
+
+
+%prep
+%setup -q
+
+
+%build
+%cmake
+%cmake_build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%cmake_install
+cd mon
+npm install --production
+cd ..
+mkdir -p %buildroot/usr/lib/vitastor
+cp -r mon %buildroot/usr/lib/vitastor
+mv %buildroot/usr/lib/vitastor/mon/scripts/make-etcd %buildroot/usr/lib/vitastor/mon/
+mkdir -p %buildroot/lib/systemd/system
+cp mon/scripts/vitastor.target mon/scripts/vitastor-mon.service mon/scripts/vitastor-osd@.service %buildroot/lib/systemd/system
+mkdir -p %buildroot/lib/udev/rules.d
+cp mon/scripts/90-vitastor.rules %buildroot/lib/udev/rules.d
+mkdir -p %buildroot/var/lib/one
+cp -r opennebula/remotes %buildroot/var/lib/one
+cp opennebula/install.sh %buildroot/var/lib/one/remotes/datastore/vitastor/
+mkdir -p %buildroot/etc/
+cp -r opennebula/sudoers.d %buildroot/etc/
+
+
+%files
+%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
+
+
+%files -n vitastor-osd
+%_bindir/vitastor-osd
+%_bindir/vitastor-disk
+%_bindir/vitastor-dump-journal
+/lib/systemd/system/vitastor-osd@.service
+/lib/systemd/system/vitastor.target
+/lib/udev/rules.d/90-vitastor.rules
+
+
+%pre -n vitastor-osd
+groupadd -r -f vitastor 2>/dev/null ||:
+useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
+install -o vitastor -g vitastor -d /var/log/vitastor
+mkdir -p /etc/vitastor
+
+
+%files -n vitastor-mon
+/usr/lib/vitastor/mon
+/lib/systemd/system/vitastor-mon.service
+
+
+%pre -n vitastor-mon
+groupadd -r -f vitastor 2>/dev/null ||:
+useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
+mkdir -p /etc/vitastor
+mkdir -p /var/lib/vitastor
+chown vitastor:vitastor /var/lib/vitastor
+
+
+%files -n vitastor-client
+%_bindir/vitastor-nbd
+%_bindir/vitastor-ublk
+%_bindir/vitastor-nfs
+%_bindir/vitastor-cli
+%_bindir/vitastor-rm
+%_bindir/vitastor-kv
+%_bindir/vitastor-kv-stress
+%_bindir/vita
+%_libdir/libvitastor_client.so*
+%_libdir/libvitastor_kv.so*
+
+
+%files -n vitastor-client-devel
+%_includedir/vitastor_c.h
+%_includedir/vitastor_kv.h
+%_libdir/pkgconfig
+
+
+%files -n vitastor-fio
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+
+
+%files -n vitastor-opennebula
+/var/lib/one
+/etc/sudoers.d/opennebula-vitastor
+
+
+%triggerin -n vitastor-opennebula -- opennebula
+[ $2 = 0 ] || exit 0
+/var/lib/one/remotes/datastore/vitastor/install.sh
+
+
+# Turn off the brp-python-bytecompile script
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+
+
+%changelog

src/blockstore/blockstore_heap.cpp

@@ -331,7 +331,19 @@ corrupted_block:
                     block_num, block_offset, wr->size, sizeof(heap_entry_t));
                 goto corrupted_block;
             }
-            wr->entry_type &= ~BS_HEAP_GARBAGE;
+            if (wr->is_garbage())
+            {
+                // Garbage collection is only performed when writing new entries into the block
+                // because it needs a fake LSN and modified blocks require consecutive modified LSNs
+                // That's why garbage entries may persist on disk
+                if (log_level > 5)
+                {
+                    fprintf(stderr, "Notice: skipping garbage entry %jx:%jx v%ju l%ju in metadata block %u at %u\n",
+                        wr->inode, wr->stripe, wr->version, wr->lsn, block_num, block_offset);
+                }
+                block_offset += wr->size;
+                continue;
+            }
             if ((wr->entry_type & BS_HEAP_TYPE) < BS_HEAP_BIG_WRITE ||
                 (wr->entry_type & BS_HEAP_TYPE) > BS_HEAP_ROLLBACK ||
                 (wr->entry_type & ~(BS_HEAP_TYPE|BS_HEAP_STABLE)) ||
@@ -360,7 +372,7 @@ corrupted_object:
             {
                 // Small writes require accessing offset & len to calculate correct length,
                 // so require at least sizeof(heap_small_write_t) for them
-                fprintf(stderr, "Error: entry %jx:%jx v%ju has invalid size in metadata block %u at %u (%u < min %zu bytes). Metadata is corrupted, aborting\n",
+                fprintf(stderr, "Error: entry %jx:%jx v%ju has invalid size in metadata block %u at %u (%u < min %zu bytes)\n",
                     wr->inode, wr->stripe, wr->version, block_num, block_offset, wr->size, sizeof(heap_small_write_t));
                 goto corrupted_object;
             }
@@ -374,8 +386,8 @@ corrupted_object:
             uint32_t expected_crc32c = wr->calc_crc32c();
             if (wr->crc32c != expected_crc32c)
             {
-                fprintf(stderr, "Error: entry %jx:%jx v%ju in metadata block %u at %u is corrupt (crc32c mismatch: expected %08x, got %08x). Metadata is corrupted, aborting\n",
-                    wr->inode, wr->stripe, wr->version,
+                fprintf(stderr, "Error: entry %jx:%jx v%ju l%ju in metadata block %u at %u is corrupt (crc32c mismatch: expected %08x, got %08x). ",
+                    wr->inode, wr->stripe, wr->version, wr->lsn,
                     block_num, block_offset, expected_crc32c, wr->crc32c);
                 goto corrupted_object;
             }
@@ -385,7 +397,7 @@ corrupted_object:
                 wr->small().offset % dsk->bitmap_granularity ||
                 wr->small().len % dsk->bitmap_granularity))
             {
-                fprintf(stderr, "Error: %s entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters, aborting\n",
+                fprintf(stderr, "Error: %s entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters. ",
                     wr->type() == BS_HEAP_SMALL_WRITE ? "small_write" : "intent_write",
                     wr->inode, wr->stripe, wr->version, wr->small().offset, wr->small().len);
                 goto corrupted_object;
@@ -395,7 +407,7 @@ corrupted_object:
                 wr->big_intent().offset % dsk->bitmap_granularity ||
                 wr->big_intent().len % dsk->bitmap_granularity))
             {
-                fprintf(stderr, "Error: big_intent entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters, aborting\n",
+                fprintf(stderr, "Error: big_intent entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters. ",
                     wr->inode, wr->stripe, wr->version, wr->big_intent().offset, wr->big_intent().len);
                 goto corrupted_object;
             }
@@ -422,7 +434,7 @@ int blockstore_heap_t::load_blocks(uint64_t disk_offset, uint64_t size, uint8_t
             next_lsn = wr->lsn;
         }
         entries_loaded++;
-        insert_list_item(li);
+        loaded_list_items.push_back(li);
         modify_alloc(block_num, [&](heap_block_info_t & inf)
         {
             if (!inf.entries.size())
@@ -526,6 +538,23 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
     return true;
 }
 
+void blockstore_heap_t::finish_load()
+{
+    if (loaded_list_items.size())
+    {
+        // Sort everything and load in correct order
+        std::sort(loaded_list_items.begin(), loaded_list_items.end(), [this](const heap_list_item_t* a, const heap_list_item_t* b)
+        {
+            return a->entry.lsn < b->entry.lsn;
+        });
+        for (auto & li: loaded_list_items)
+        {
+            insert_list_item(li);
+        }
+        loaded_list_items.clear();
+    }
+}
+
 void blockstore_heap_t::fill_recheck_queue()
 {
     for (auto & pgp: block_index)
@@ -712,6 +741,7 @@ bool blockstore_heap_t::recheck_small_writes(std::function<void(bool is_data, ui
     }
     if (!recheck_queue_filled)
     {
+        finish_load();
         fill_recheck_queue();
         recheck_queue_filled = true;
     }
@@ -803,7 +833,7 @@ std::vector<uint32_t> blockstore_heap_t::get_recheck_modified_blocks()
     return modified;
 }
 
-int blockstore_heap_t::finish_load(bool allow_corrupted)
+int blockstore_heap_t::finish_recheck()
 {
     if (!marked_used_blocks)
     {
@@ -1128,6 +1158,35 @@ heap_entry_t *blockstore_heap_t::read_entry(object_id oid)
     return &li->entry;
 }
 
+void blockstore_heap_t::gc_block(heap_block_info_t & inf)
+{
+    if (inf.has_garbage)
+    {
+        size_t i = 0, j = 0;
+        for (; i < inf.entries.size(); i++)
+        {
+            if (inf.entries[i]->entry.is_garbage())
+            {
+                // old entry invalidated by a newer one, mark it as freeable on block write
+                // assign a 'virtual' LSN to track GC completion
+                assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
+                uint64_t gc_lsn = ++next_lsn;
+                inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : gc_lsn;
+                inf.mod_lsn_to = gc_lsn;
+                push_inflight_lsn(gc_lsn, &inf.entries[i]->entry, HEAP_INFLIGHT_GC);
+            }
+            else
+            {
+                if (j != i)
+                    inf.entries[j] = inf.entries[i];
+                j++;
+            }
+        }
+        inf.entries.resize(j);
+        inf.has_garbage = false;
+    }
+}
+
 int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free)
 {
     if (last_allocated_block != UINT32_MAX)
@@ -1194,31 +1253,7 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
     }
     // Write into the same block
     auto & inf = block_info.at(last_allocated_block);
-    if (inf.has_garbage)
-    {
-        size_t i = 0, j = 0;
-        for (; i < inf.entries.size(); i++)
-        {
-            if (inf.entries[i]->entry.is_garbage())
-            {
-                // old entry invalidated by a newer one, mark it as freeable on block write
-                // assign a 'virtual' LSN to track GC completion
-                assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
-                uint64_t gc_lsn = ++next_lsn;
-                inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : gc_lsn;
-                inf.mod_lsn_to = gc_lsn;
-                push_inflight_lsn(gc_lsn, &inf.entries[i]->entry, HEAP_INFLIGHT_GC);
-            }
-            else
-            {
-                if (j != i)
-                    inf.entries[j] = inf.entries[i];
-                j++;
-            }
-        }
-        inf.entries.resize(j);
-        inf.has_garbage = false;
-    }
+    gc_block(inf);
     *block_num = last_allocated_block;
     modify_alloc(last_allocated_block, [&](heap_block_info_t & inf)
     {

src/blockstore/blockstore_heap.h

@@ -200,6 +200,7 @@ class blockstore_heap_t
 
     bool marked_used_blocks = false;
     bool recheck_queue_filled = false;
+    std::vector<heap_list_item_t*> loaded_list_items;
     std::set<uint32_t> recheck_modified_blocks;
     std::deque<heap_entry_t*> recheck_queue;
     int recheck_in_progress = 0;
@@ -215,6 +216,7 @@ class blockstore_heap_t
     void defragment_block(uint32_t block_num);
     void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
+    void gc_block(heap_block_info_t & inf);
     int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
     void insert_list_item(heap_list_item_t *li);
     int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
@@ -237,13 +239,14 @@ public:
         std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
     int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
         bool allow_corrupted, uint64_t &entries_loaded);
-    // finish loading
-    int finish_load(bool allow_corrupted = false);
+    // finish loading - should be called after load_blocks
+    void finish_load();
     // get blocks which are modified during loading and should be written to the disk
     // before finishing initialization if not R/O
     std::vector<uint32_t> get_recheck_modified_blocks();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
+    int finish_recheck();
     // reshard database according to the pool's PG count
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void* reshard_state, uint64_t chunk_limit);

src/blockstore/blockstore_init.cpp

@@ -239,6 +239,7 @@ resume_4:
         return 1;
     }
     // metadata read finished
+    bs->heap->finish_load();
     printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
     if (zero_on_init && !bs->dsk.disable_meta_fsync)
     {
@@ -284,7 +285,7 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    if (bs->heap->finish_load() != 0)
+    if (bs->heap->finish_recheck() != 0)
     {
         exit(1);
     }

src/client/etcd_state_client.cpp

@@ -414,7 +414,10 @@ void etcd_state_client_t::start_etcd_watcher()
                 }
                 // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
                 // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
-                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null())
+                // Also don't save revision from the initial created: true messages because they always contain the latest revision
+                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES &&
+                    !data["result"]["header"]["revision"].is_null() &&
+                    !data["result"]["created"].bool_value())
                 {
                     // Restart watchers from the same revision number as in the last received message,
                     // not from the next one to protect against revision being split into multiple messages,

src/client/messenger.cpp

@@ -145,7 +145,7 @@ void osd_messenger_t::init()
         rdma_contexts = msgr_rdma_context_t::create_all(
             osd_num && osd_cluster_network_masks.size() ? osd_cluster_network_masks : osd_network_masks,
             rdma_device != "" ? rdma_device.c_str() : NULL,
-            rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
+            rdma_port_num, rdma_gid_index, rdma_mtu, log_level
         );
         if (!rdma_contexts.size())
         {
@@ -322,7 +322,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
-    this->rdma_odp = config["rdma_odp"].bool_value();
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();

src/client/messenger.h

@@ -200,7 +200,6 @@ protected:
     std::vector<msgr_rdma_context_t *> rdma_contexts;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
-    bool rdma_odp = false;
     rdma_event_channel *rdmacm_evch = NULL;
     std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
     std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
@@ -287,8 +286,6 @@ protected:
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);
-    void try_send_rdma_odp(osd_client_t *cl);
-    void try_send_rdma_nodp(osd_client_t *cl);
     bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);

src/client/msgr_rdma.cpp

@@ -59,8 +59,6 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_destroy_cq(cq);
     if (channel)
         ibv_destroy_comp_channel(channel);
-    if (mr)
-        ibv_dereg_mr(mr);
     if (pd)
         ibv_dealloc_pd(pd);
     if (context && !is_cm)
@@ -182,7 +180,7 @@ static int match_port_gid(const std::vector<addr_mask_t> & osd_network_masks, ib
 #endif
 
 std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vector<addr_mask_t> & osd_network_masks,
-    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level)
+    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level)
 {
     int res;
     std::vector<msgr_rdma_context_t*> ret;
@@ -271,7 +269,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 {
                     if (log_level > 0)
                         log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, best_gidx);
-                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
                     if (ctx)
                     {
                         ctx->net_mask = osd_network_masks[net_num];
@@ -291,7 +289,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                     log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, gidx);
                 }
 #endif
-                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
                 if (ctx)
                     ret.push_back(ctx);
             }
@@ -306,7 +304,7 @@ cleanup:
     return ret;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
+msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, int log_level)
 {
     msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
     ibv_context *context = ibv_open_device(dev);
@@ -346,30 +344,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
 
-    ctx->odp = odp;
-    if (ctx->odp)
-    {
-        if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
-            !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV))
-        {
-            ctx->odp = false;
-            if (log_level > 0)
-                fprintf(stderr, "The RDMA device isn't implicit ODP (On-Demand Paging) capable, disabling it\n");
-        }
-    }
-
-    if (ctx->odp)
-    {
-        ctx->mr = ibv_reg_mr(ctx->pd, NULL, SIZE_MAX, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ON_DEMAND);
-        if (!ctx->mr)
-        {
-            fprintf(stderr, "Couldn't register RDMA memory region\n");
-            goto cleanup;
-        }
-    }
-
     ctx->channel = ibv_create_comp_channel(ctx->context);
     if (!ctx->channel)
     {
@@ -388,8 +362,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
     return ctx;
 
 cleanup:
-    if (context)
-        ibv_close_device(context);
     delete ctx;
     return NULL;
 }
@@ -605,52 +577,7 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
     return total_dst_len-dst_len;
 }
 
-void osd_messenger_t::try_send_rdma_odp(osd_client_t *cl)
-{
-    auto rc = cl->rdma_conn;
-    if (!cl->send_list.size() || rc->cur_send >= rc->max_send)
-    {
-        return;
-    }
-    uint64_t op_size = 0, op_sge = 0;
-    ibv_sge sge[rc->max_sge];
-    while (rc->send_pos < cl->send_list.size())
-    {
-        iovec & iov = cl->send_list[rc->send_pos];
-        if (op_size >= rc->max_msg || op_sge >= rc->max_sge)
-        {
-            rc->send_sizes.push_back(op_size);
-            try_send_rdma_wr(cl, sge, op_sge);
-            op_sge = 0;
-            op_size = 0;
-            if (rc->cur_send >= rc->max_send)
-            {
-                break;
-            }
-        }
-        uint32_t len = (uint32_t)(op_size+iov.iov_len-rc->send_buf_pos < rc->max_msg
-            ? iov.iov_len-rc->send_buf_pos : rc->max_msg-op_size);
-        sge[op_sge++] = {
-            .addr = (uintptr_t)((uint8_t*)iov.iov_base+rc->send_buf_pos),
-            .length = len,
-            .lkey = rc->ctx->mr->lkey,
-        };
-        op_size += len;
-        rc->send_buf_pos += len;
-        if (rc->send_buf_pos >= iov.iov_len)
-        {
-            rc->send_pos++;
-            rc->send_buf_pos = 0;
-        }
-    }
-    if (op_sge > 0)
-    {
-        rc->send_sizes.push_back(op_size);
-        try_send_rdma_wr(cl, sge, op_sge);
-    }
-}
-
-void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
+void osd_messenger_t::try_send_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
     if (!rc->send_out_size)
@@ -658,14 +585,11 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
         rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
-        if (!rc->ctx->odp)
+        rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
+        if (!rc->send_out.mr)
         {
-            rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
-            if (!rc->send_out.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
         }
     }
     // Copy data into the buffer and send it
@@ -690,7 +614,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
             ibv_sge sge = {
                 .addr = (uintptr_t)dst,
                 .length = (uint32_t)copied,
-                .lkey = rc->ctx->odp ? rc->ctx->mr->lkey : rc->send_out.mr->lkey,
+                .lkey = rc->send_out.mr->lkey,
             };
             try_send_rdma_wr(cl, &sge, 1);
             rc->send_sizes.push_back(copied);
@@ -698,20 +622,12 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     }
 }
 
-void osd_messenger_t::try_send_rdma(osd_client_t *cl)
-{
-    if (cl->rdma_conn->ctx->odp)
-        try_send_rdma_odp(cl);
-    else
-        try_send_rdma_nodp(cl);
-}
-
 static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
         .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
+        .lkey = cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -733,14 +649,11 @@ bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
     auto rc = cl->rdma_conn;
     assert(!rc->recv_buf.buf);
     rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
-    if (!rc->ctx->odp)
+    rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+    if (!rc->recv_buf.mr)
     {
-        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
-        if (!rc->recv_buf.mr)
-        {
-            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-            exit(1);
-        }
+        fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+        exit(1);
     }
     for (uint32_t i = 0; i < rc->max_recv; i++)
     {
@@ -816,14 +729,11 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 rc->cur_send--;
                 uint64_t sent_size = rc->send_sizes.at(0);
                 rc->send_sizes.erase(rc->send_sizes.begin(), rc->send_sizes.begin()+1);
-                if (!rdma_context->odp)
-                {
-                    rc->send_done_pos += sent_size;
-                    rc->send_out_full = false;
-                    if (rc->send_done_pos == rc->send_out_size)
-                        rc->send_done_pos = 0;
-                    assert(rc->send_done_pos < rc->send_out_size);
-                }
+                rc->send_done_pos += sent_size;
+                rc->send_out_full = false;
+                if (rc->send_done_pos == rc->send_out_size)
+                    rc->send_done_pos = 0;
+                assert(rc->send_done_pos < rc->send_out_size);
                 int send_pos = 0, send_buf_pos = 0;
                 while (sent_size > 0)
                 {

src/client/msgr_rdma.h

@@ -26,8 +26,6 @@ struct msgr_rdma_context_t
     ibv_context *context = NULL;
     ibv_device_attr_ex attrx;
     ibv_pd *pd = NULL;
-    bool odp = false;
-    ibv_mr *mr = NULL;
     ibv_comp_channel *channel = NULL;
     ibv_cq *cq = NULL;
     ibv_port_attr portinfo;
@@ -43,9 +41,9 @@ struct msgr_rdma_context_t
     int cm_refs = 0;
 
     static std::vector<msgr_rdma_context_t*> create_all(const std::vector<addr_mask_t> & osd_network_masks,
-        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level);
+        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level);
     static msgr_rdma_context_t *create(ibv_device *dev, ibv_port_attr & portinfo,
-        int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
+        int ib_port, int gid_index, uint32_t mtu, int log_level);
     static msgr_rdma_context_t* create_cm(ibv_context *ctx);
     bool reserve_cqe(int n);
 

src/client/msgr_rdmacm.cpp

@@ -178,7 +178,6 @@ msgr_rdma_context_t* msgr_rdma_context_t::create_cm(ibv_context *ctx)
         delete rdma_context;
         return NULL;
     }
-    rdma_context->odp = false;
     rdma_context->channel = ibv_create_comp_channel(rdma_context->context);
     if (!rdma_context->channel)
     {

src/cmd/cli_dd.cpp

@@ -442,7 +442,7 @@ struct cli_dd_t
                 }
                 delete cur_read;
             }
-            else if (!is_zero(read_op->bitmap_buf, read_op->len/iinfo.in_granularity/8))
+            else if (!is_zero(read_op->bitmap_buf, (read_op->len/iinfo.in_granularity+7)/8))
             {
                 vitastor_read(cur_read);
             }

src/disk_tool/disk_tool_meta.cpp

@@ -109,6 +109,7 @@ close_error:
             r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
             meta_pos += read_len;
         }
+        heap->finish_load();
         heap->iterate_objects([&](heap_entry_t* obj, uint32_t meta_block_num)
         {
             obj_fn(heap, obj, meta_block_num);

src/osd/osd_peering_pg.cpp

@@ -334,36 +334,44 @@ pg_osd_set_state_t* pg_t::add_object_to_state(const object_id oid, const uint64_
     if (it == state_dict.end())
     {
         std::vector<osd_num_t> read_target;
+        bool found = false;
+        uint32_t bad_mask = (LOC_OUTDATED | LOC_CORRUPTED);
+retry:
         if (scheme == POOL_SCHEME_REPLICATED)
         {
             for (auto & o: osd_set)
             {
-                if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED)))
+                if (!(o.loc_bad & bad_mask))
                 {
                     read_target.push_back(o.osd_num);
+                    found = true;
                 }
             }
-            while (read_target.size() < pg_size)
+            if (read_target.size() < pg_size)
             {
                 // FIXME: This is because we then use .data() and assume it's at least <pg_size> long
-                read_target.push_back(0);
+                read_target.resize(pg_size);
             }
         }
         else
         {
             read_target.resize(pg_size);
-            for (int i = 0; i < pg_size; i++)
-            {
-                read_target[i] = 0;
-            }
             for (auto & o: osd_set)
             {
-                if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED)))
+                if (!(o.loc_bad & bad_mask))
                 {
                     read_target[o.role] = o.osd_num;
+                    found = true;
                 }
             }
         }
+        if (!found && (bad_mask & LOC_CORRUPTED))
+        {
+            // Allow to try reading corrupted copies in rare cases when the object is corrupted on all OSDs
+            bad_mask = LOC_OUTDATED;
+            read_target.clear();
+            goto retry;
+        }
         state_dict[osd_set] = {
             .read_target = read_target,
             .osd_set = osd_set,

src/osd/osd_primary.cpp

@@ -271,7 +271,6 @@ resume_0:
             }
             cur_op->buf = alloc_read_buffer(op_data->stripes, pg ? pg->pg_data_size : 1, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
-            op_data->st = 1;
         }
         else
         {
@@ -284,11 +283,14 @@ resume_0:
             op_data->degraded = 1;
             cur_op->buf = alloc_read_buffer(op_data->stripes, pg->pg_size, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
-            op_data->st = 1;
         }
     }
 resume_1:
-    return;
+    if (op_data->n_subops > 0)
+    {
+        op_data->st = 1;
+        return;
+    }
 resume_2:
     if (op_data->errors > 0)
     {
@@ -296,8 +298,9 @@ resume_2:
         {
             // I/O or checksum error
             // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
-            goto resume_0;
+            auto new_object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
+            if (new_object_state != op_data->object_state)
+                goto resume_0;
         }
         finish_op(cur_op, op_data->errcode);
         return;
@@ -736,8 +739,11 @@ resume_1:
     submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
     op_data->prev_set = NULL;
 resume_2:
-    op_data->st = 2;
-    return;
+    if (op_data->n_subops > 0)
+    {
+        op_data->st = 2;
+        return;
+    }
 resume_3:
     if (op_data->errors > 0)
     {

src/osd/osd_primary_subops.cpp

@@ -134,11 +134,17 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
         n_subops = 1;
     else
         zero_read = -1;
-    osd_op_t *subops = new osd_op_t[n_subops];
     op_data->fact_ver = 0;
     op_data->done = op_data->errors = op_data->drops = op_data->errcode = 0;
     op_data->n_subops = n_subops;
-    op_data->subops = subops;
+    if (!n_subops)
+    {
+        op_data->errcode = -EIO;
+        op_data->subops = NULL;
+        op_data->errors = 1;
+        return;
+    }
+    op_data->subops = new osd_op_t[n_subops];
     int sent = submit_primary_subop_batch(submit_type, op_data->oid.inode, op_version, op_data->stripes, osd_set, cur_op, 0, zero_read);
     assert(sent == n_subops);
 }

src/osd/osd_primary_write.cpp

@@ -108,57 +108,31 @@ retry_1:
         }
     }
     // Read required blocks
-    {
-        if (op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
-        {
-            // Allow to read version number (just version number!) from corrupted chunks
-            // to allow full overwrite of a corrupted object
-            bool found = false;
-            for (int role = 0; role < pg.pg_size; role++)
-            {
-                if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start)
-                {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-            {
-                osd_num_t corrupted_target[pg.pg_size];
-                for (int role = 0; role < pg.pg_size; role++)
-                {
-                    corrupted_target[role] = 0;
-                }
-                for (auto & loc: op_data->object_state->osd_set)
-                {
-                    if (!(loc.loc_bad & LOC_OUTDATED) && !corrupted_target[loc.role])
-                    {
-                        corrupted_target[loc.role] = loc.osd_num;
-                    }
-                }
-                submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, corrupted_target, cur_op);
-                goto resume_2;
-            }
-        }
-        submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
-    }
+    submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
 resume_2:
-    op_data->st = 2;
-    return;
+    if (op_data->n_subops > 0)
+    {
+        op_data->st = 2;
+        return;
+    }
 resume_3:
     if (op_data->errors > 0)
     {
         if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
         {
             // Mark object corrupted and retry
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
-            op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
-            if (cur_op->rmw_buf)
+            pg_osd_set_state_t *new_object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
+            if (new_object_state != op_data->object_state)
             {
-                free(cur_op->rmw_buf);
-                cur_op->rmw_buf = NULL;
+                op_data->object_state = new_object_state;
+                op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
+                if (cur_op->rmw_buf)
+                {
+                    free(cur_op->rmw_buf);
+                    cur_op->rmw_buf = NULL;
+                }
+                goto retry_1;
             }
-            goto retry_1;
         }
         deref_object_state(pg, &op_data->object_state, true);
         pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);

src/test/test_heap.cpp

@@ -131,7 +131,7 @@ void test_mvcc(bool csum)
     _test_init(dsk, csum);
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     // write, read, modify, check basic mvcc
     {
@@ -186,7 +186,7 @@ void test_update(bool csum)
     _test_init(dsk, csum);
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     {
         _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 0, buffer_area.data());
@@ -206,7 +206,7 @@ void test_delete(bool csum)
     _test_init(dsk, csum);
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     {
         // Add 1:0 and 1:20000
@@ -302,7 +302,7 @@ void test_defrag_block()
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
     dsk.meta_area_size = 4096*3;
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     uint32_t big_write_size = heap.get_big_entry_size();
     uint32_t small_write_size = heap.get_small_entry_size(0, 4096);
@@ -377,7 +377,7 @@ void test_compact(bool csum, bool stable)
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
 
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     memset(buffer_area.data(), 0x19, 4096);
     _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, buffer_area.data());
@@ -508,7 +508,7 @@ void test_iterate_compaction()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Case: BIG_STABLE(v1 l1) SMALL(v2 l2) SMALL(v3 l3) SMALL(v4 l4) COMMIT(v2 l5) SMALL(v5 l6) COMMIT(v5 l7)
         uint32_t mblock = 0;
@@ -551,9 +551,10 @@ void test_iterate_compaction()
         blockstore_heap_t heap(&dsk, buffer_area.data());
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
         bool done = heap.recheck_small_writes([&](bool, uint64_t, uint64_t, uint8_t*, std::function<void()> cb) {}, 1);
         assert(done);
-        heap.finish_load();
+        heap.finish_recheck();
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
 
@@ -572,7 +573,7 @@ void test_iterate_compaction()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Case: BIG_STABLE(v1 l1) SMALL(v2 l2) COMMIT(v2 l3) SMALL(v3 l4) COMMIT(v3 l5) unfinished
         uint32_t mblock = 0;
@@ -613,9 +614,10 @@ void test_iterate_compaction()
         blockstore_heap_t heap(&dsk, buffer_area.data());
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
         bool done = heap.recheck_small_writes([&](bool, uint64_t, uint64_t, uint8_t*, std::function<void()> cb) {}, 1);
         assert(done);
-        heap.finish_load();
+        heap.finish_recheck();
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
 
@@ -634,7 +636,7 @@ void test_iterate_compaction()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Case: BIG_STABLE(v1 l1) SMALL(v2 l2) SMALL(v3 l3) SMALL(v4 l4) ROLLBACK(v3 l5) COMMIT(v2 l6)
         // -> compact by adding BIG_STABLE(v2 l2)
@@ -685,9 +687,10 @@ void test_iterate_compaction()
         blockstore_heap_t heap(&dsk, buffer_area.data());
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
         bool done = heap.recheck_small_writes([&](bool, uint64_t, uint64_t, uint8_t*, std::function<void()> cb) {}, 1);
         assert(done);
-        heap.finish_load();
+        heap.finish_recheck();
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
 
@@ -708,7 +711,7 @@ void test_iterate_compaction()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Case: BIG_STABLE(v1 l1) DELETE(l2) BIG_UNSTABLE(v1 l3) ROLLBACK(v0 l4)
         // -> compact by adding DELETE(l4)
@@ -760,9 +763,10 @@ void test_iterate_compaction()
         blockstore_heap_t heap(&dsk, buffer_area.data());
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
         bool done = heap.recheck_small_writes([&](bool, uint64_t, uint64_t, uint8_t*, std::function<void()> cb) {}, 1);
         assert(done);
-        heap.finish_load();
+        heap.finish_recheck();
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
 
@@ -782,7 +786,7 @@ void test_iterate_compaction()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Case: BIG_STABLE(v1 l1) SMALL(v2 l2) SMALL(v3 l3) ROLLBACK(v2 l4) SMALL(v3 l5) COMMIT(v3 l6)
         // -> compact by adding BIG_STABLE(v3 l6) and skip l3
@@ -836,9 +840,10 @@ void test_iterate_compaction()
         blockstore_heap_t heap(&dsk, buffer_area.data());
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
         bool done = heap.recheck_small_writes([&](bool, uint64_t, uint64_t, uint8_t*, std::function<void()> cb) {}, 1);
         assert(done);
-        heap.finish_load();
+        heap.finish_recheck();
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
 
@@ -859,7 +864,7 @@ void test_iterate_compaction()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_UNSTABLE(v3 l3)
         // -> skip compaction of l2 into l1 if not under pressure
@@ -902,9 +907,10 @@ void test_iterate_compaction()
         blockstore_heap_t heap(&dsk, buffer_area.data());
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
         bool done = heap.recheck_small_writes([&](bool, uint64_t, uint64_t, uint8_t*, std::function<void()> cb) {}, 1);
         assert(done);
-        heap.finish_load();
+        heap.finish_recheck();
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
 
@@ -933,7 +939,7 @@ void test_modify_bitmap()
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
 
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     memset(buffer_area.data(), 0x19, 8192);
     _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 8192, buffer_area.data());
@@ -981,7 +987,7 @@ void test_recheck(bool async, bool csum, bool intent)
     // write
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // object 1
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 8192, buffer_area.data());
@@ -1007,6 +1013,7 @@ void test_recheck(bool async, bool csum, bool intent)
         blockstore_heap_t heap(&dsk, async ? NULL : buffer_area.data(), 10);
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
 
         int calls = 0;
         bool done = heap.recheck_small_writes([&](bool is_data, uint64_t offset, uint64_t len, uint8_t *buf, std::function<void()> cb)
@@ -1033,7 +1040,7 @@ void test_recheck(bool async, bool csum, bool intent)
         assert(done);
         assert(calls == (async || intent ? 3 : 1));
 
-        heap.finish_load();
+        heap.finish_recheck();
 
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 1);
@@ -1075,7 +1082,7 @@ void test_corruption()
     // write
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // big_write
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
@@ -1128,7 +1135,7 @@ void test_full_overwrite(bool stable)
     // write
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // big_write
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
@@ -1233,7 +1240,7 @@ void test_reshard_list()
     // write
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
         _test_big_write(heap, dsk, 1, 0x20000, 1, 0x40000, true, 0, 0, buffer_area.data());
@@ -1312,7 +1319,7 @@ void test_reshard_chunked()
     // write
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         for (int i = 0; i < 30; i++)
             _test_big_write(heap, dsk, 1, i*0x20000, 1, i*0x20000, true, 0, 0, buffer_area.data());
@@ -1380,7 +1387,7 @@ void test_destructor_mvcc()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // some writes
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
@@ -1406,7 +1413,7 @@ void test_rollback()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // some writes
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
@@ -1495,7 +1502,7 @@ void test_rollback()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // Remove a big write at all
         object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0x20000 };
@@ -1534,7 +1541,7 @@ void test_rollback()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // v1 unstable -> v2 unstable -> v3 unstable -> rollback v2 -> rollback v1
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, false, 0, 0, buffer_area.data());
@@ -1561,7 +1568,7 @@ void test_rollback()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // v1 unstable -> v2 unstable -> v3 unstable -> rollback v2 -> rollback v1
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, false, 0, 0, buffer_area.data());
@@ -1588,7 +1595,7 @@ void test_rollback()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // v1 unstable -> v2 unstable -> v3 unstable -> commit v1 -> commit v2 -> rollback v1
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, false, 0, 0, buffer_area.data());
@@ -1641,7 +1648,7 @@ void test_rollback()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         // v1 stable -> rollback v2
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
@@ -1676,7 +1683,7 @@ void test_alloc_buffer()
     }
 
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     uint64_t pos;
 
@@ -1739,7 +1746,7 @@ void test_full_alloc()
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
 
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
     assert(heap.get_meta_total_space() == 4*4096);
 
     uint32_t big_write_size = heap.get_big_entry_size();
@@ -1784,7 +1791,7 @@ void test_intent_write(bool csum)
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, buffer_area.data());
 
@@ -1817,7 +1824,7 @@ void test_big_intent_csums()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, buffer_area.data());
 
@@ -1865,6 +1872,7 @@ void test_big_intent_csums()
         blockstore_heap_t heap(&dsk, buffer_area.data(), 10);
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
 
         int calls = 0;
         bool done = heap.recheck_small_writes([&](bool is_data, uint64_t offset, uint64_t len, uint8_t *buf, std::function<void()> cb)
@@ -1882,7 +1890,7 @@ void test_big_intent_csums()
         assert(done);
         assert(calls == 2);
 
-        heap.finish_load();
+        heap.finish_recheck();
 
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
@@ -1891,7 +1899,7 @@ void test_big_intent_csums()
         object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
         heap_entry_t *obj = heap.read_entry(oid);
         assert(obj);
-        assert(count_writes(heap, obj) == 2);
+        assert(count_writes(heap, obj) == 1);
         assert(obj->lsn == 2);
         assert(obj->entry_type == BS_HEAP_BIG_INTENT|BS_HEAP_STABLE);
         assert(obj->version == 2);
@@ -1907,7 +1915,7 @@ void test_recalc_stats()
     _test_init(dsk, false);
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     {
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());
@@ -1959,7 +1967,7 @@ void test_redirect_intent_csums()
 
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_load();
+        heap.finish_recheck();
 
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, buffer_area.data());
 
@@ -1985,6 +1993,7 @@ void test_redirect_intent_csums()
         blockstore_heap_t heap(&dsk, buffer_area.data(), 10);
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
 
         int calls = 0;
         bool done = heap.recheck_small_writes([&](bool is_data, uint64_t offset, uint64_t len, uint8_t *buf, std::function<void()> cb)
@@ -2002,7 +2011,7 @@ void test_redirect_intent_csums()
         assert(done);
         assert(calls == 2);
 
-        heap.finish_load();
+        heap.finish_recheck();
 
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 0);
@@ -2039,6 +2048,7 @@ void test_redirect_intent_csums()
         blockstore_heap_t heap(&dsk, buffer_area.data(), 10);
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);
+        heap.finish_load();
 
         int calls = 0;
         bool done = heap.recheck_small_writes([&](bool is_data, uint64_t offset, uint64_t len, uint8_t *buf, std::function<void()> cb)
@@ -2054,7 +2064,7 @@ void test_redirect_intent_csums()
         assert(done);
         assert(calls == 2);
 
-        heap.finish_load();
+        heap.finish_recheck();
 
         auto mod = heap.get_recheck_modified_blocks();
         assert(mod.size() == 1);
@@ -2081,7 +2091,7 @@ void test_explicit_complete()
     dsk.disable_journal_fsync = dsk.disable_meta_fsync = false;
     std::vector<uint8_t> buffer_area(dsk.journal_device_size);
     blockstore_heap_t heap(&dsk, buffer_area.data());
-    heap.finish_load();
+    heap.finish_recheck();
 
     {
         _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 0, buffer_area.data());

tests/run_tests.sh

@@ -96,6 +96,11 @@ TEST_NAME=local_read POOLCFG='"local_reads":"random",' ./test_heal.sh
 SCHEME=ec ./test_heal.sh
 ANTIETCD=1 ./test_heal.sh
 
+./test_checksum.sh
+OLD=1 ./test_checksum.sh
+./test_corrupt_all.sh
+OLD=1 ./test_corrupt_all.sh
+
 ./test_reweight_half.sh
 ./test_snapshot_pool2.sh
 ./test_snapshot_read_bitmap.sh

tests/test_corrupt_all.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -ex
+
+SCHEME=replicated
+PG_SIZE=2
+OSD_COUNT=2
+IMG_SIZE=128
+OSD_ARGS="--data_csum_type crc32c --csum_block_size 4k --inmemory_journal false --journal_trim_interval $((IMG_SIZE*8)) $OSD_ARGS"
+OFFSET_ARGS="--data_csum_type crc32c --csum_block_size 4k $OFFSET_ARGS"
+GLOBAL_CONFIG=',"client_eio_retry_interval":0'
+. `dirname $0`/run_3osds.sh
+check_qemu
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
+
+# Write
+$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -end_fsync=1 -rw=write -image=testimg -runtime=10
+#$VITASTOR_FIO -bs=4k -direct=1 -iodepth=16 -end_fsync=1 -rw=randwrite -image=testimg -number_ios=10000
+
+# Intentionally corrupt OSD data and restart both of them
+kill $OSD1_PID $OSD2_PID
+data_offset=$(build/src/disk_tool/vitastor-disk simple-offsets ./testdata/bin/test_osd1.bin $OFFSET_ARGS | grep data_offset | awk '{print $2}')
+truncate -s $data_offset ./testdata/bin/test_osd1.bin
+dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+truncate -s $data_offset ./testdata/bin/test_osd2.bin
+dd if=/dev/zero of=./testdata/bin/test_osd2.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+start_osd 1
+start_osd 2
+
+# Wait until start
+wait_up 10
+
+# Trigger scrub
+$ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
+
+# Wait for scrub to finish
+wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
+
+# Verify that ALL objects are now corrupted+incomplete
+$VITASTOR_CLI describe --json &>./testdata/describe.json
+$VITASTOR_CLI describe --json | jq -e '[ .[] | select(.corrupted) ] | length == '$((IMG_SIZE * 8 * PG_SIZE))
+
+# Check that we can remove or overwrite them
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -end_fsync=1 -rw=write -offset=$((IMG_SIZE/2))M -image=testimg -runtime=10
+$VITASTOR_CLI rm-data --pool 1 --inode 1
+
+format_green OK