trace writes

- Fix a bug introduced in 2.2.0 - pg_locks weren't disabled for pools without local_reads
  correctly which could lead to inactive pools during various operations
- Fix an old bug where OSDs could send sub-operations to incorrect peer OSDs when their
  connections were stopped and reestablished quickly, in 2.2.0 it was usually leading
  to "sequencing broken" messages in OSD logs
- Fix debug use_sync_send_recv mode

.gitea/workflows/test.yml

@@ -684,6 +684,24 @@ jobs:
           echo ""
         done
 
+  test_write_iothreads:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: TEST_NAME=iothreads GLOBAL_CONFIG=',"client_iothread_count":4' /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_write_no_same:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.2.0")
+set(VITASTOR_VERSION "2.2.2")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.2.0
+VITASTOR_VERSION ?= v2.2.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.2.0
+          image: vitalif/vitastor-csi:v2.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.2.0
+          image: vitalif/vitastor-csi:v2.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.2.0"
+    vitastorCSIDriverVersion = "2.2.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.2.0-1) unstable; urgency=medium
+vitastor (2.2.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.2.0
+VITASTOR_VERSION ?= v2.2.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.2.0
+VITASTOR_VERSION=v2.2.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/pool.en.md

@@ -310,6 +310,10 @@ If you set this parameter to "random", clients will try to distribute read reque
 all available secondary OSDs. This mode is mainly useful for tests, but, probably, not
 really required in production setups.
 
+[PG locks](osd.en.md#enable_pg_locks) are required for local reads to function. However,
+PG locks are enabled automatically by default for pools with enabled local reads, so you
+don't have to enable them explicitly.
+
 ## max_osd_combinations
 
 - Type: integer

docs/config/pool.ru.md

@@ -309,6 +309,9 @@ meow недоступен".
 запросы чтения по всем доступным вторичным OSD. Этот режим в основном полезен для тестов,
 но, скорее всего, редко нужен в реальных инсталляциях.
 
+Для работы локальных чтений требуются [блокировки PG](osd.ru.md#enable_pg_locks). Включать
+их явно не нужно - они включаются автоматически для пулов с включёнными локальными чтениями.
+
 ## max_osd_combinations
 
 - Тип: целое число

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitastor:v2.2.0`
+   `docker pull vitastor:v2.2.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitastor:v2.2.0`
+   `docker pull vitastor:v2.2.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/proxmox.en.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.1 are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
-  bookworm for 8.1, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
+  bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
 - Install vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* or see note) packages from Vitastor repository
 - Define storage in `/etc/pve/storage.cfg` (see below)
 - Block network access from VMs to Vitastor network (to OSDs and etcd),

docs/installation/proxmox.ru.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.1):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
-  bookworm для 8.1, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
+  bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
 - Установите пакеты vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* или см. сноску) из репозитория Vitastor
 - Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже)
 - Обязательно заблокируйте доступ от виртуальных машин к сети Vitastor (OSD и etcd), т.к. Vitastor (пока) не поддерживает аутентификацию

docs/intro/author.en.md

@@ -10,8 +10,17 @@ Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
 
 Join Vitastor Telegram Chat: https://t.me/vitastor
 
-All server-side code (OSD, Monitor and so on) is licensed under the terms of
-Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
+License: VNPL 1.1 for server-side code and dual VNPL 1.1 + GPL 2.0+ for client tools.
+
+Server-side code is licensed only under the terms of VNPL.
+
+Client libraries (cluster_client and so on) are dual-licensed under the same
+VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+software like QEMU and fio.
+
+## VNPL
+
+Vitastor Network Public License 1.1 (VNPL 1.1) is a copyleft license based on
 GNU GPLv3.0 with the additional "Network Interaction" clause which requires
 opensourcing all programs directly or indirectly interacting with Vitastor
 through a computer network and expressly designed to be used in conjunction
@@ -20,18 +29,83 @@ the terms of the same license, but also under the terms of any GPL-Compatible
 Free Software License, as listed by the Free Software Foundation.
 This is a stricter copyleft license than the Affero GPL.
 
-Please note that VNPL doesn't require you to open the code of proprietary
-software running inside a VM if it's not specially designed to be used with
-Vitastor.
+The idea of VNPL is, in addition to modules linked to Vitastor code in a single
+binary file, to extend copyleft action to micro-service modules only interacting
+with it over the network.
 
 Basically, you can't use the software in a proprietary environment to provide
 its functionality to users without opensourcing all intermediary components
 standing between the user and Vitastor or purchasing a commercial license
 from the author 😀.
 
-Client libraries (cluster_client and so on) are dual-licensed under the same
-VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
-software like QEMU and fio.
+At the same time, VNPL doesn't impose any restrictions on software *not specially designed*
+to be used with Vitastor, for example, on Windows running inside a VM with a Vitastor disk.
 
-You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](../../VNPL-1.1.txt).
-GPL 2.0 is also included in this repository as [GPL-2.0.txt](../../GPL-2.0.txt).
+## Explanation
+
+Network copyleft is governed by the clause **13. Remote Network Interaction** of VNPL.
+
+A program is considered to be a "Proxy Program" if it meets both conditions:
+- It is specially designed to be used with Vitastor. Basically, it means that the program
+  has any functionality specific to Vitastor and thus "knows" that it works with Vitastor,
+  not with something random.
+- It interacts with Vitastor directly or indirectly through any programming interface,
+  including API, CLI, network or any wrapper (also considered a Proxy Program itself).
+
+If, in addition to that:
+- You give any user an apportunity to interact with Vitastor directly or indirectly through
+  any computer interface including the network or any number of wrappers (Proxy Programs).
+
+Then VNPL requires you to publish the code of all above Proxy Programs to all above users
+under the terms of any GPL-compatible license - that is, GPL, LGPL, MIT/BSD or Apache 2,
+because "GPL compatibility" is treated as an ability to legally include licensed code in
+a GPL application.
+
+So, if you have a "Proxy Program", but it's not open to the user who directly or indirectly
+interacts with Vitastor - you are forbidden to use Vitastor under the terms of VNPL and you
+need a commercial license which doesn't contain open-source requirements.
+
+## Examples
+
+- Vitastor Kubernetes CSI driver which creates PersistentVolumes by calling `vitastor-cli create`.
+  - Yes, it interacts with Vitastor through vitastor-cli.
+  - Yes, it is designed specially for use with Vitastor (it has no sense otherwise).
+  - So, CSI driver **definitely IS** a Proxy Program and must be published under the terms of
+    a free software license.
+- Windows, installed in a VM with the system disk on Vitastor storage.
+  - Yes, it interacts with Vitastor indirectly - it reads and writes data through the block
+    device interface, emulated by QEMU.
+  - No, it definitely isn't designed specially for use with Vitastor - Windows was created long
+    ago before Vitastor and doesn't know anything about it.
+  - So, Windows **definitely IS NOT** a Proxy Program and VNPL doesn't require to open it.
+- Cloud control panel which makes requests to Vitastor Kubernetes CSI driver.
+  - Yes, it interacts with Vitastor indirectly through the CSI driver, which is a Proxy Program.
+  - May or may not be designed specially for use with Vitastor. How to determine exactly?
+    Imagine that Vitastor is replaced with any other storage (for example, with a proprietary).
+    Do control panel functions change in any way? If they do (for example, if snapshots stop working),
+    then the panel contains specific functionality and thus is designed specially for use with Vitastor.
+    Otherwise, the panel is universal and isn't designed specially for Vitastor.
+  - So, whether you are required to open-source the panel also **depends** on whether it
+    contains specific functionality or not.
+
+## Why?
+
+Because I believe into the spirit of copyleft (Linux wouldn't become so popular without GPL!)
+and, at the same time, I want to have a way to monetize the product.
+
+Existing licenses including AGPL are useless for it with an SDS - SDS is a very deeply
+internal software which is almost definitely invisible to the user and thus AGPL doesn't
+require anyone to open the code even if they make a proprietary fork.
+
+And, in fact, the current situation in the world where GPL is though to only restrict direct
+linking of programs into a single executable file, isn't much correct. Nowadays, programs
+are more often linked with network API calls, not with /usr/bin/ld, and a software product
+may consist of dozens of microservices interacting with each other over the network.
+
+That's why we need VNPL to keep the license sufficiently copyleft.
+
+## License Texts
+
+- VNPL 1.1 in English: [VNPL-1.1.txt](../../VNPL-1.1.txt)
+- VNPL 1.1 in Russian: [VNPL-1.1-RU.txt](../../VNPL-1.1-RU.txt)
+- GPL 2.0: [GPL-2.0.txt](../../GPL-2.0.txt)

docs/intro/author.ru.md

@@ -12,6 +12,14 @@
 
 Лицензия: VNPL 1.1 на серверный код и двойная VNPL 1.1 + GPL 2.0+ на клиентский.
 
+Серверные компоненты распространяются только на условиях VNPL.
+
+Клиентские библиотеки распространяются на условиях двойной лицензии VNPL 1.0
+и также на условиях GNU GPL 2.0 или более поздней версии. Так сделано в целях
+совместимости с таким ПО, как QEMU и fio.
+
+## VNPL
+
 VNPL - "сетевой копилефт", собственная свободная копилефт-лицензия
 Vitastor Network Public License 1.1, основанная на GNU GPL 3.0 с дополнительным
 условием "Сетевого взаимодействия", требующим распространять все программы,
@@ -29,9 +37,70 @@ Vitastor Network Public License 1.1, основанная на GNU GPL 3.0 с д
 На Windows и любое другое ПО, не разработанное *специально* для использования
 вместе с Vitastor, никакие ограничения не накладываются.
 
-Клиентские библиотеки распространяются на условиях двойной лицензии VNPL 1.0
-и также на условиях GNU GPL 2.0 или более поздней версии. Так сделано в целях
-совместимости с таким ПО, как QEMU и fio.
+## Пояснение
 
-Вы можете найти полный текст VNPL 1.1 на английском языке в файле [VNPL-1.1.txt](../../VNPL-1.1.txt),
-VNPL 1.1 на русском языке в файле [VNPL-1.1-RU.txt](../../VNPL-1.1-RU.txt), а GPL 2.0 в файле [GPL-2.0.txt](../../GPL-2.0.txt).
+Сетевой копилефт регулируется пунктом лицензии **13. Удалённое сетевое взаимодействие**.
+
+Программа считается "прокси-программой", если верны оба условия:
+- Она создана специально для работы вместе с Vitastor. По сути это означает, что программа
+  должна иметь специфичный для Vitastor функционал, то есть, "знать", что она взаимодействует
+  именно с Vitastor.
+- Она прямо или косвенно взаимодействует с Vitastor через абсолютно любой программный
+  интерфейс, включая любые способы вызова: API, CLI, сеть или через какую-то обёртку (в
+  свою очередь тоже являющуюся прокси-программой).
+
+Если в дополнение к этому также:
+- Вы предоставляете любому пользователю возможность взаимодействовать с Vitastor по сети,
+  опять-таки, через любой интерфейс или любую серию "обёрток" (прокси-программ)
+
+То, согласно VNPL, вы должны открыть код "прокси-программ" **таким пользователям** на условиях
+любой GPL-совместимой лицензии - то есть, GPL, LGPL, MIT/BSD или Apache 2 - "совместимость с GPL"
+понимается как возможность включать лицензируемый код в GPL-приложение.
+
+Соответственно, если у вас есть "прокси-программа", но её код не открыт пользователю,
+который прямо или косвенно взаимодействует с Vitastor - вам запрещено использовать Vitastor
+на условиях VNPL и вам нужна коммерческая лицензия, не содержащая требований об открытии кода.
+
+## Примеры
+
+- Kubernetes CSI-драйвер Vitastor, создающий PersistentVolume с помощью вызова `vitastor-cli create`.
+  - Да, взаимодействует с Vitastor через vitastor-cli.
+  - Да, создавался специально для работы с Vitastor (иначе в чём же ещё его смысл).
+  - Значит, CSI-драйвер **точно считается** "прокси-программой" и должен быть открыт под свободной
+    лицензией.
+- Windows, установленный в виртуальную машину на диске Vitastor.
+  - Да, взаимодействует с Vitastor "прямо или косвенно" - пишет и читает данные через интерфейс
+    блочного устройства, эмулируемый QEMU.
+  - Нет, точно не создан *специально для работы с Vitastor* - когда его создавали, никакого
+    Vitastor ещё и в помине не было.
+  - Значит, Windows **точно не считается** "прокси-программой" и на него требования VNPL не распространяются.
+- Панель управления облака, делающая запросы к Kubernetes CSI-драйверу Vitastor.
+  - Да, взаимодействует с Vitastor косвенно через CSI-драйвер, являющийся "прокси-программой".
+  - Сходу не известно, создавалась ли конкретно для работы с Vitastor. Как понять, да или нет?
+    Представьте, что Vitastor заменён на любую другую систему хранения (например, на проприетарную).
+    Работа панели управления изменится? Если да (например, перестанут работать снапшоты) - значит,
+    панель содержит специфичный функционал и "создана специально для работы с Vitastor".
+    Если нет - значит, специфичного функционала панель не содержит и в принципе она универсальна.
+  - Нужно ли открывать панель - **зависит** от того, содержит она специфичный функционал или нет.
+
+## Почему так?
+
+Потому что я одновременно верю в дух копилефт-лицензий (Linux не стал бы так популярен,
+если бы не GPL!) и хочу иметь возможность монетизации продукта.
+
+При этом использовать даже AGPL для программной СХД бессмысленно - это глубоко внутреннее
+ПО, которое пользователь почти наверняка не увидит вообще, поэтому и открывать код никому
+никогда не придётся, даже при создании производного продукта.
+
+Да и в целом сложившаяся в мире ситуация, при которой действие GPL ограничивается только
+прямым связыванием в один исполняемый файл, не очень корректна. В настоящее время программы
+гораздо чаще интегрируют сетевыми вызовами, а не с помощью /usr/bin/ld, и общий программный
+продукт может состоять из нескольких десятков микросервисов, взаимодействующих по сети.
+
+Поэтому для сохранения достаточной "копилефтности" и придумана VNPL.
+
+## Тексты лицензий
+
+- VNPL 1.1 на английском языке: [VNPL-1.1.txt](../../VNPL-1.1.txt)
+- VNPL 1.1 на русском языке: [VNPL-1.1-RU.txt](../../VNPL-1.1-RU.txt)
+- GPL 2.0: [GPL-2.0.txt](../../GPL-2.0.txt)

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.2.0",
+  "version": "2.2.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.2.0",
+  "version": "2.2.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.2.0'
+VITASTOR_VERSION = '2.2.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.0
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.0.el7.tar.gz
+Source0:        vitastor-2.2.2.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.0
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.0.el8.tar.gz
+Source0:        vitastor-2.2.2.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.0
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.0.el9.tar.gz
+Source0:        vitastor-2.2.2.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.2.0")
+add_definitions(-DVITASTOR_VERSION="2.2.2")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/client/messenger.cpp

@@ -188,7 +188,7 @@ void osd_messenger_t::init()
             auto cl = cl_it->second;
             cl_it++;
             auto peer_fd = cl->peer_fd;
-            if (!cl->osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
+            if (!cl->osd_num && !cl->in_osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
             {
                 // Do not run keepalive on regular clients
                 continue;
@@ -199,7 +199,7 @@ void osd_messenger_t::init()
                 if (!cl->ping_time_remaining)
                 {
                     // Ping timed out, stop the client
-                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->osd_num, cl->peer_fd);
+                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->peer_fd);
                     stop_client(peer_fd, true);
                     // Restart iterator because it may be invalidated
                     cl_it = clients.upper_bound(peer_fd);
@@ -230,7 +230,7 @@ void osd_messenger_t::init()
                             return;
                         }
                         int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
-                        auto fail_osd_num = cl->osd_num;
+                        auto fail_osd_num = cl->in_osd_num ? cl->in_osd_num : cl->osd_num;
                         cl->ping_time_remaining = 0;
                         delete op;
                         if (fail_fd >= 0)

src/client/messenger.h

@@ -60,6 +60,7 @@ struct osd_client_t
     int ping_time_remaining = 0;
     int idle_time_remaining = 0;
     osd_num_t osd_num = 0;
+    osd_num_t in_osd_num = 0;
     bool is_incoming = false;
 
     void *in_buf = NULL;
@@ -98,6 +99,7 @@ struct osd_client_t
     std::vector<osd_op_t*> zc_free_list;
 
     ~osd_client_t();
+    void cancel_ops();
 };
 
 struct osd_wanted_peer_t
@@ -235,6 +237,7 @@ public:
     void outbox_push(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;
+    std::function<void(osd_num_t)> break_pg_locks;
     std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
     void read_requests();
     void send_replies();

src/client/msgr_op.h

@@ -173,6 +173,7 @@ struct osd_op_t
     osd_op_buf_list_t iov;
 
     ~osd_op_t();
+    void cancel();
 
     bool is_recovery_related();
 };

src/client/msgr_rdmacm.cpp

@@ -510,13 +510,12 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
     rc->qp = conn->cmid->qp;
     // And an osd_client_t
     auto cl = new osd_client_t();
-    cl->is_incoming = true;
     cl->peer_addr = conn->parsed_addr;
     cl->peer_port = conn->rdmacm_port;
     cl->peer_fd = conn->peer_fd;
     cl->peer_state = PEER_RDMA;
     cl->connect_timeout_id = -1;
-    cl->osd_num = peer_osd;
+    cl->in_osd_num = peer_osd;
     cl->in_buf = malloc_or_die(receive_buffer_size);
     cl->rdma_conn = rc;
     clients[conn->peer_fd] = cl;

src/client/msgr_receive.cpp

@@ -8,11 +8,12 @@ void osd_messenger_t::read_requests()
     for (int i = 0; i < read_ready_clients.size(); i++)
     {
         int peer_fd = read_ready_clients[i];
-        osd_client_t *cl = clients[peer_fd];
-        if (cl->read_msg.msg_iovlen)
+        auto cl_it = clients.find(peer_fd);
+        if (cl_it == clients.end() || !cl_it->second || cl_it->second->read_msg.msg_iovlen)
         {
             continue;
         }
+        auto cl = cl_it->second;
         if (cl->read_remaining < receive_buffer_size)
         {
             cl->read_iov.iov_base = cl->in_buf;
@@ -33,8 +34,12 @@ void osd_messenger_t::read_requests()
             auto iothread = iothreads.size() ? iothreads[peer_fd % iothreads.size()] : NULL;
             io_uring_sqe sqe_local;
             ring_data_t data_local;
-            sqe_local.user_data = (uint64_t)&data_local;
             io_uring_sqe* sqe = (iothread ? &sqe_local : ringloop->get_sqe());
+            if (iothread)
+            {
+                sqe_local = { .user_data = (uint64_t)&data_local };
+                data_local = {};
+            }
             if (!sqe)
             {
                 cl->read_msg.msg_iovlen = 0;
@@ -56,7 +61,8 @@ void osd_messenger_t::read_requests()
             {
                 result = -errno;
             }
-            handle_read(result, cl);
+            // like set_immediate
+            tfd->set_timer_us(0, false, [this, result, cl](int){ handle_read(result, cl); });
         }
     }
     read_ready_clients.clear();
@@ -228,7 +234,7 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
             {
                 if (cl->read_op->req.hdr.id != cl->read_op_id)
                 {
-                    fprintf(stderr, "Warning: operation sequencing is broken on client %d, stopping client\n", cl->peer_fd);
+                    fprintf(stderr, "Warning: operation sequencing is broken on client %d: expected num %ju, got %ju, stopping client\n", cl->peer_fd, cl->read_op_id, cl->read_op->req.hdr.id);
                     stop_client(cl->peer_fd);
                     return false;
                 }

src/client/msgr_send.cpp

@@ -194,12 +194,14 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         auto iothread = iothreads.size() ? iothreads[peer_fd % iothreads.size()] : NULL;
         io_uring_sqe sqe_local;
         ring_data_t data_local;
-        sqe_local.user_data = (uint64_t)&data_local;
         io_uring_sqe* sqe = (iothread ? &sqe_local : ringloop->get_sqe());
-        if (!sqe)
+        if (iothread)
         {
-            return false;
+            sqe_local = { .user_data = (uint64_t)&data_local };
+            data_local = {};
         }
+        if (!sqe)
+            return false;
         cl->write_msg.msg_iov = cl->send_list.data();
         cl->write_msg.msg_iovlen = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
         cl->refs++;
@@ -237,7 +239,8 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         {
             result = -errno;
         }
-        handle_send(result, false, false, cl);
+        // like set_immediate
+        tfd->set_timer_us(0, false, [this, result, cl](int){ handle_send(result, false, false, cl); });
     }
     return true;
 }

src/client/msgr_stop.cpp

@@ -9,38 +9,37 @@
 #include "msgr_rdma.h"
 #endif
 
-void osd_messenger_t::cancel_osd_ops(osd_client_t *cl)
+void osd_client_t::cancel_ops()
 {
     std::vector<osd_op_t*> cancel_ops;
-    cancel_ops.resize(cl->sent_ops.size());
+    cancel_ops.resize(sent_ops.size());
     int i = 0;
-    for (auto p: cl->sent_ops)
+    for (auto p: sent_ops)
     {
         cancel_ops[i++] = p.second;
     }
-    cl->sent_ops.clear();
-    cl->outbox.clear();
+    sent_ops.clear();
     for (auto op: cancel_ops)
     {
-        cancel_op(op);
+        op->cancel();
     }
 }
 
-void osd_messenger_t::cancel_op(osd_op_t *op)
+void osd_op_t::cancel()
 {
-    if (op->op_type == OSD_OP_OUT)
+    if (op_type == OSD_OP_OUT && callback)
     {
-        op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
-        op->reply.hdr.id = op->req.hdr.id;
-        op->reply.hdr.opcode = op->req.hdr.opcode;
-        op->reply.hdr.retval = -EPIPE;
-        // Copy lambda to be unaffected by `delete op`
-        std::function<void(osd_op_t*)>(op->callback)(op);
+        reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+        reply.hdr.id = req.hdr.id;
+        reply.hdr.opcode = req.hdr.opcode;
+        reply.hdr.retval = -EPIPE;
+        // Copy lambda to be unaffected by `delete this`
+        (std::function<void(osd_op_t*)>(callback))(this);
     }
     else
     {
         // This function is only called in stop_client(), so it's fine to destroy the operation
-        delete op;
+        delete this;
     }
 }
 
@@ -63,6 +62,10 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         {
             fprintf(stderr, "[OSD %ju] Stopping client %d (OSD peer %ju)\n", osd_num, peer_fd, cl->osd_num);
         }
+        else if (cl->in_osd_num)
+        {
+            fprintf(stderr, "[OSD %ju] Stopping client %d (incoming OSD peer %ju)\n", osd_num, peer_fd, cl->in_osd_num);
+        }
         else
         {
             fprintf(stderr, "[OSD %ju] Stopping client %d (regular client)\n", osd_num, peer_fd);
@@ -73,8 +76,12 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     cl->peer_state = PEER_STOPPED;
     if (cl->osd_num)
     {
-        // ...and forget OSD peer
-        osd_peer_fds.erase(cl->osd_num);
+        auto osd_it = osd_peer_fds.find(cl->osd_num);
+        if (osd_it != osd_peer_fds.end() && osd_it->second == cl->peer_fd)
+        {
+            // ...and forget OSD peer
+            osd_peer_fds.erase(osd_it);
+        }
     }
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
@@ -101,30 +108,17 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         }
     }
 #endif
+    if (cl->in_osd_num && break_pg_locks)
+    {
+        // Break PG locks
+        break_pg_locks(cl->in_osd_num);
+    }
     if (cl->osd_num)
     {
         // Then repeer PGs because cancel_op() callbacks can try to perform
         // some actions and we need correct PG states to not do something silly
         repeer_pgs(cl->osd_num);
     }
-    // Then cancel all operations
-    if (cl->read_op)
-    {
-        if (!cl->read_op->callback)
-        {
-            delete cl->read_op;
-        }
-        else
-        {
-            cancel_op(cl->read_op);
-        }
-        cl->read_op = NULL;
-    }
-    if (cl->osd_num)
-    {
-        // Cancel outbound operations
-        cancel_osd_ops(cl);
-    }
     // Find the item again because it can be invalidated at this point
     it = clients.find(peer_fd);
     if (it != clients.end())
@@ -149,6 +143,17 @@ osd_client_t::~osd_client_t()
         close(peer_fd);
         peer_fd = -1;
     }
+    // Then cancel all operations
+    // Operations have to be canceled only after clearing all references to osd_client_t
+    // because otherwise their buffers may be still present in io_uring asynchronous requests
+    if (read_op)
+    {
+        // read_op may be an incoming op or a continued response for an outbound op
+        read_op->cancel();
+        read_op = NULL;
+    }
+    // Cancel outbound ops
+    cancel_ops();
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.2.0
+Version: 2.2.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -283,6 +283,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
                 !strcmp(opt, "readonly") || !strcmp(opt, "readwrite") ||
                 !strcmp(opt, "force") || !strcmp(opt, "reverse") ||
                 !strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") ||
+                !strcmp(opt, "allow-up") || !strcmp(opt, "allow_up") ||
                 !strcmp(opt, "down-ok") || !strcmp(opt, "down_ok") ||
                 !strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
                 !strcmp(opt, "help") || !strcmp(opt, "all") ||

src/cmd/cli_rm_osd.cpp

@@ -15,7 +15,7 @@ struct rm_osd_t
 {
     cli_tool_t *parent;
 
-    bool dry_run, force_warning, force_dataloss;
+    bool dry_run, force_warning, force_dataloss, allow_up;
     uint64_t etcd_tx_retry_ms = 500;
     uint64_t etcd_tx_retries = 10000;
     std::vector<uint64_t> osd_ids;
@@ -70,7 +70,7 @@ struct rm_osd_t
         {
             if (parent->cli->st_cli.peer_states.find(osd_id) != parent->cli->st_cli.peer_states.end())
             {
-                is_warning = true;
+                is_warning = !allow_up;
                 still_up.push_back(osd_id);
             }
         }
@@ -168,7 +168,7 @@ struct rm_osd_t
                                 : strtoupper(e["effect"].string_value())+" PGs"))
                 )+" after deleting OSD(s).\n";
             }
-            if (still_up.size())
+            if (still_up.size() && !allow_up)
                 error += (still_up.size() == 1 ? "OSD " : "OSDs ") + implode(", ", still_up) +
                     (still_up.size() == 1 ? "is" : "are") + " still up. Use `vitastor-disk purge` to delete them.\n";
             if (is_dataloss && !force_dataloss && !dry_run)
@@ -476,6 +476,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm_osd(json11::Json cfg)
     auto rm_osd = new rm_osd_t();
     rm_osd->parent = this;
     rm_osd->dry_run = cfg["dry_run"].bool_value();
+    rm_osd->allow_up = cfg["allow_up"].bool_value();
     rm_osd->force_dataloss = cfg["allow_data_loss"].bool_value();
     rm_osd->force_warning = rm_osd->force_dataloss || cfg["force"].bool_value();
     if (!cfg["etcd_tx_retries"].is_null())

src/disk_tool/disk_tool_udev.cpp

@@ -435,7 +435,7 @@ int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
             printf("%s\n", json11::Json(result).dump().c_str());
         return 0;
     }
-    std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd" };
+    std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd", "--allow-up" };
     for (auto osd_num: osd_numbers)
     {
         rm_osd_cli.push_back(std::to_string(osd_num));

src/osd/osd.cpp

@@ -85,6 +85,7 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
     msgr.ringloop = this->ringloop;
     msgr.exec_op = [this](osd_op_t *op) { exec_op(op); };
     msgr.repeer_pgs = [this](osd_num_t peer_osd) { repeer_pgs(peer_osd); };
+    msgr.break_pg_locks = [this](osd_num_t peer_osd) { break_pg_locks(peer_osd); };
     msgr.check_config_hook = [this](osd_client_t *cl, json11::Json conf) { return check_peer_config(cl, conf); };
     msgr.init();
 

src/osd/osd.h

@@ -278,6 +278,8 @@ class osd_t
     void handle_peers();
     bool check_peer_config(osd_client_t *cl, json11::Json conf);
     void repeer_pgs(osd_num_t osd_num);
+    void repeer_pg(pg_t & pg);
+    void break_pg_locks(osd_num_t osd_num);
     void start_pg_peering(pg_t & pg);
     void drop_dirty_pg_connections(pool_pg_num_t pg);
     void record_pg_lock(pg_t & pg, osd_num_t peer_osd, uint64_t pg_state);

src/osd/osd_cluster.cpp

@@ -432,9 +432,16 @@ void osd_t::apply_pg_locks_localize_only()
         }
         auto & pool_cfg = pool_it->second;
         auto & pg = pp.second;
+        auto old_disable_pg_locks = pg.disable_pg_locks;
         pg.disable_pg_locks = pg_locks_localize_only &&
-            pool_cfg.scheme == POOL_SCHEME_REPLICATED &&
-            pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY;
+            (pool_cfg.scheme != POOL_SCHEME_REPLICATED ||
+            pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY);
+        if (!pg.disable_pg_locks && old_disable_pg_locks)
+        {
+            // Relock PG
+            printf("[PG %u/%u] Repeer to enable PG locks\n", pg.pool_id, pg.pg_num);
+            repeer_pg(pg);
+        }
     }
 }
 
@@ -877,8 +884,8 @@ void osd_t::apply_pg_config()
                 pg.next_scrub = pg_cfg.next_scrub;
                 pg.target_set = pg_cfg.target_set;
                 pg.disable_pg_locks = pg_locks_localize_only &&
-                    pool_item.second.scheme == POOL_SCHEME_REPLICATED &&
-                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY;
+                    (pool_item.second.scheme != POOL_SCHEME_REPLICATED ||
+                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY);
                 if (pg.scheme == POOL_SCHEME_EC)
                 {
                     use_ec(pg.pg_size, pg.pg_data_size, true);

src/osd/osd_peering.cpp

@@ -73,18 +73,25 @@ void osd_t::handle_peers()
     }
 }
 
+void osd_t::break_pg_locks(osd_num_t peer_osd)
+{
+    for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
+    {
+        if (lock_it->second.primary_osd == peer_osd)
+        {
+            if (log_level > 3)
+            {
+                printf("Break PG %u/%u lock on disconnection of OSD %ju\n", lock_it->first.pool_id, lock_it->first.pg_num, peer_osd);
+            }
+            pg_locks.erase(lock_it++);
+        }
+        else
+            lock_it++;
+    }
+}
+
 void osd_t::repeer_pgs(osd_num_t peer_osd)
 {
-    if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
-    {
-        for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
-        {
-            if (lock_it->second.primary_osd == peer_osd)
-                pg_locks.erase(lock_it++);
-            else
-                lock_it++;
-        }
-    }
     // Re-peer affected PGs
     for (auto & p: pgs)
     {
@@ -104,21 +111,26 @@ void osd_t::repeer_pgs(osd_num_t peer_osd)
             {
                 // Repeer this pg
                 printf("[PG %u/%u] Repeer because of OSD %ju\n", pg.pool_id, pg.pg_num, peer_osd);
-                if (!(pg.state & (PG_ACTIVE | PG_REPEERING)) || pg.can_repeer())
-                {
-                    start_pg_peering(pg);
-                }
-                else
-                {
-                    // Stop accepting new operations, wait for current ones to finish or fail
-                    pg.state = pg.state & ~PG_ACTIVE | PG_REPEERING;
-                    report_pg_state(pg);
-                }
+                repeer_pg(pg);
             }
         }
     }
 }
 
+void osd_t::repeer_pg(pg_t & pg)
+{
+    if (!(pg.state & (PG_ACTIVE | PG_REPEERING)) || pg.can_repeer())
+    {
+        start_pg_peering(pg);
+    }
+    else
+    {
+        // Stop accepting new operations, wait for current ones to finish or fail
+        pg.state = pg.state & ~PG_ACTIVE | PG_REPEERING;
+        report_pg_state(pg);
+    }
+}
+
 // Reset PG state (when peering or stopping)
 void osd_t::reset_pg(pg_t & pg)
 {
@@ -466,6 +478,7 @@ void osd_t::relock_pg(pg_t & pg)
             auto pg_it = pgs.find(pg_id);
             if (pg_it == pgs.end())
             {
+                printf("Warning: PG %u/%u is gone during lock attempt\n", pg_id.pool_id, pg_id.pg_num);
                 return;
             }
             auto & pg = pg_it->second;

src/osd/osd_primary_subops.cpp

@@ -46,6 +46,11 @@ void osd_t::autosync()
 void osd_t::finish_op(osd_op_t *cur_op, int retval)
 {
     inflight_ops--;
+    if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
+    {
+        printf("%jx %jx+%x p%jx v%jx r=%x\n", cur_op->req.rw.inode, cur_op->req.rw.offset, cur_op->req.rw.len,
+            cur_op->op_data ? cur_op->op_data->orig_ver : 0, cur_op->reply.rw.version, retval);
+    }
     if (cur_op->req.hdr.opcode == OSD_OP_READ ||
         cur_op->req.hdr.opcode == OSD_OP_WRITE ||
         cur_op->req.hdr.opcode == OSD_OP_DELETE)
@@ -417,15 +422,17 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
     if (retval != expected)
     {
         int64_t peer_osd = (msgr.clients.find(subop->peer_fd) != msgr.clients.end()
-            ? msgr.clients[subop->peer_fd]->osd_num : -subop->peer_fd);
+            ? msgr.clients[subop->peer_fd]->osd_num : 0);
         if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
         {
             printf(
                 subop->peer_fd >= 0
-                    ? "%1$s subop to %2$jx:%3$jx v%4$ju failed on osd %7$jd: retval = %5$d (expected %6$d)\n"
+                    ? (peer_osd > 0
+                        ? "%1$s subop to %2$jx:%3$jx v%4$ju failed on osd %7$ju: retval = %5$d (expected %6$d)\n"
+                        : "%1$s subop to %2$jx:%3$jx v%4$ju failed on peer %8$d: retval = %5$d (expected %6$d)\n")
                     : "%1$s subop to %2$jx:%3$jx v%4$ju failed locally: retval = %5$d (expected %6$d)\n",
                 osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version,
-                retval, expected, peer_osd
+                retval, expected, peer_osd, subop->peer_fd
             );
         }
         else if (opcode == OSD_OP_SEC_DELETE)

src/osd/osd_scrub.cpp

@@ -505,7 +505,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 if (votes[role] > 0 && votes[role] < votes[best])
                 {
                     printf(
-                        "[PG %u/%u] Object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
+                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[role].osd_num, votes[best],
@@ -528,7 +528,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 best = -1;
                 inconsistent = true;
                 printf(
-                    "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
+                    "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
                     INODE_POOL(op_data->oid.inode), op_data->pg_num,
                     op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
                 );
@@ -547,7 +547,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
         {
             inconsistent = true;
             printf(
-                "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
+                "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
                 INODE_POOL(op_data->oid.inode), op_data->pg_num,
                 op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
             );
@@ -574,7 +574,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 {
                     op_data->stripes[i].read_error = true;
                     printf(
-                        "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
+                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[i].role, op_data->stripes[i].osd_num,

src/osd/osd_secondary.cpp

@@ -91,16 +91,17 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
     {
         return false;
     }
+    auto & pool_cfg = pool_cfg_it->second;
+    if (pg_locks_localize_only && (pool_cfg.scheme != POOL_SCHEME_REPLICATED || pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY))
+    {
+        return true;
+    }
     auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) };
     auto pg_it = pgs.find(ppg);
     if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE)
     {
         return false;
     }
-    if (pg_it->second.disable_pg_locks)
-    {
-        return true;
-    }
     auto lock_it = pg_locks.find(ppg);
     return lock_it != pg_locks.end() && lock_it->second.primary_osd == primary_osd;
 }
@@ -140,7 +141,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {
         if (!(cur_op->req.sec_rw.flags & OSD_OP_IGNORE_PG_LOCK) &&
-            !sec_check_pg_lock(cl->osd_num, cur_op->req.sec_rw.oid))
+            !sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_rw.oid))
         {
             cur_op->bs_op->retval = -EPIPE;
             secondary_op_callback(cur_op);
@@ -169,7 +170,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
     else if (cur_op->req.hdr.opcode == OSD_OP_SEC_DELETE)
     {
         if (!(cur_op->req.sec_del.flags & OSD_OP_IGNORE_PG_LOCK) &&
-            !sec_check_pg_lock(cl->osd_num, cur_op->req.sec_del.oid))
+            !sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_del.oid))
         {
             cur_op->bs_op->retval = -EPIPE;
             secondary_op_callback(cur_op);
@@ -193,7 +194,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         {
             for (int i = 0; i < cur_op->bs_op->len; i++)
             {
-                if (!sec_check_pg_lock(cl->osd_num, ((obj_ver_id*)cur_op->buf)[i].oid))
+                if (!sec_check_pg_lock(cl->in_osd_num, ((obj_ver_id*)cur_op->buf)[i].oid))
                 {
                     cur_op->bs_op->retval = -EPIPE;
                     secondary_op_callback(cur_op);
@@ -247,7 +248,7 @@ void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
         void *cur_buf = reply_buf;
         for (int i = 0; i < n; i++)
         {
-            if (!sec_check_pg_lock(cl->osd_num, ov[i].oid) &&
+            if (!sec_check_pg_lock(cl->in_osd_num, ov[i].oid) &&
                 !(cur_op->req.sec_read_bmp.flags & OSD_OP_IGNORE_PG_LOCK))
             {
                 free(reply_buf);
@@ -269,7 +270,7 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
 {
     cur_op->reply.sec_lock.cur_primary = 0;
     auto cl = msgr.clients.at(cur_op->peer_fd);
-    if (!cl->osd_num ||
+    if (!cl->in_osd_num ||
         cur_op->req.sec_lock.flags != OSD_SEC_LOCK_PG &&
         cur_op->req.sec_lock.flags != OSD_SEC_UNLOCK_PG ||
         cur_op->req.sec_lock.pool_id > ((uint64_t)1<<POOL_ID_BITS) ||
@@ -290,7 +291,7 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
     auto lock_it = pg_locks.find(ppg);
     if (cur_op->req.sec_lock.flags == OSD_SEC_LOCK_PG)
     {
-        if (lock_it != pg_locks.end() && lock_it->second.primary_osd != cl->osd_num)
+        if (lock_it != pg_locks.end() && lock_it->second.primary_osd != cl->in_osd_num)
         {
             cur_op->reply.sec_lock.cur_primary = lock_it->second.primary_osd;
             finish_op(cur_op, -EBUSY);
@@ -303,13 +304,21 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
             finish_op(cur_op, -EBUSY);
             return;
         }
+        if (log_level > 3)
+        {
+            printf("Lock PG %u/%u for OSD %ju\n", ppg.pool_id, ppg.pg_num, cl->in_osd_num);
+        }
         pg_locks[ppg] = (osd_pg_lock_t){
-            .primary_osd = cl->osd_num,
+            .primary_osd = cl->in_osd_num,
             .state = cur_op->req.sec_lock.pg_state,
         };
     }
-    else if (lock_it != pg_locks.end() && lock_it->second.primary_osd == cl->osd_num)
+    else if (lock_it != pg_locks.end() && lock_it->second.primary_osd == cl->in_osd_num)
     {
+        if (log_level > 3)
+        {
+            printf("Unlock PG %u/%u by OSD %ju\n", ppg.pool_id, ppg.pg_num, cl->in_osd_num);
+        }
         pg_locks.erase(lock_it);
     }
     finish_op(cur_op, 0);
@@ -323,7 +332,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
         : json11::Json();
     auto peer_osd_num = req_json["osd_num"].uint64_value();
     auto cl = msgr.clients.at(cur_op->peer_fd);
-    cl->osd_num = peer_osd_num;
+    cl->in_osd_num = peer_osd_num;
     if (req_json["features"]["check_sequencing"].bool_value())
     {
         cl->check_sequencing = true;

src/test/test_cluster_client.cpp

@@ -121,6 +121,7 @@ void pretend_connected(cluster_client_t *cli, osd_num_t osd_num)
     cli->msgr.osd_peer_fds[osd_num] = peer_fd;
     cli->msgr.clients[peer_fd] = new osd_client_t();
     cli->msgr.clients[peer_fd]->osd_num = osd_num;
+    cli->msgr.clients[peer_fd]->peer_fd = peer_fd;
     cli->msgr.clients[peer_fd]->peer_state = PEER_CONNECTED;
     cli->msgr.wanted_peers.erase(osd_num);
     cli->msgr.repeer_pgs(osd_num);

src/util/ringloop.cpp

@@ -125,6 +125,8 @@ void ring_loop_t::loop()
         if (cqe->flags & IORING_CQE_F_MORE)
         {
             // There will be a second notification
+            if (mt)
+                mu.unlock();
             d->res = cqe->res;
             d->more = true;
             if (d->callback)

tests/run_tests.sh

@@ -59,6 +59,7 @@ SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 
 ./test_write.sh
 SCHEME=xor ./test_write.sh
+TEST_NAME=iothreads GLOBAL_CONFIG=',"client_iothread_count":4' ./test_write.sh
 
 ./test_write_no_same.sh
 

tests/test_heal.sh

@@ -55,6 +55,20 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
         -serialize_overlap=1 -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/bin/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
 
+# Wait for active/clean
+wait_up 120
+
+# Trigger scrub
+for i in $(seq 1 $PG_COUNT); do
+    $ETCDCTL put /vitastor/pg/history/1/$i `$ETCDCTL get --print-value-only /vitastor/pg/history/1/$i | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
+done
+
+# Wait for scrub to finish
+wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
+
+# Check that everything is ok
+wait_up 1
+
 qemu-img convert -S 4096 -p \
     -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
     -O raw ./testdata/bin/read.bin
@@ -64,7 +78,7 @@ if ! diff -q ./testdata/bin/read.bin ./testdata/bin/mirror.bin; then
     format_error Data lost during self-heal
 fi
 
-if grep -qP 'Checksum mismatch|BUG' ./testdata/osd*.log; then
+if grep -qP 'Checksum mismatch|BUG|Scrub error' ./testdata/osd*.log; then
     format_error Checksum mismatches or BUGs detected during test
 fi