trace writes

- Fix a bug introduced in 2.2.0 - pg_locks weren't disabled for pools without local_reads
  correctly which could lead to inactive pools during various operations
- Fix an old bug where OSDs could send sub-operations to incorrect peer OSDs when their
  connections were stopped and reestablished quickly, in 2.2.0 it was usually leading
  to "sequencing broken" messages in OSD logs
- Fix debug use_sync_send_recv mode

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.2.1")
+set(VITASTOR_VERSION "2.2.2")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.2.1
+VITASTOR_VERSION ?= v2.2.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.2.1
+          image: vitalif/vitastor-csi:v2.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.2.1
+          image: vitalif/vitastor-csi:v2.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.2.1"
+    vitastorCSIDriverVersion = "2.2.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.2.1-1) unstable; urgency=medium
+vitastor (2.2.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.2.1
+VITASTOR_VERSION ?= v2.2.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.2.1
+VITASTOR_VERSION=v2.2.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitastor:v2.2.1`
+   `docker pull vitastor:v2.2.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.1 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitastor:v2.2.1`
+   `docker pull vitastor:v2.2.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.1 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/proxmox.en.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.1 are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
-  bookworm for 8.1, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
+  bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
 - Install vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* or see note) packages from Vitastor repository
 - Define storage in `/etc/pve/storage.cfg` (see below)
 - Block network access from VMs to Vitastor network (to OSDs and etcd),

docs/installation/proxmox.ru.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.1):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
-  bookworm для 8.1, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
+  bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
 - Установите пакеты vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* или см. сноску) из репозитория Vitastor
 - Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже)
 - Обязательно заблокируйте доступ от виртуальных машин к сети Vitastor (OSD и etcd), т.к. Vitastor (пока) не поддерживает аутентификацию

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.2.1",
+  "version": "2.2.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.2.1",
+  "version": "2.2.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.2.1'
+VITASTOR_VERSION = '2.2.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.1
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.1.el7.tar.gz
+Source0:        vitastor-2.2.2.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.1
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.1.el8.tar.gz
+Source0:        vitastor-2.2.2.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.2.1
+Version:        2.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.2.1.el9.tar.gz
+Source0:        vitastor-2.2.2.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.2.1")
+add_definitions(-DVITASTOR_VERSION="2.2.2")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/client/messenger.cpp

@@ -188,7 +188,7 @@ void osd_messenger_t::init()
             auto cl = cl_it->second;
             cl_it++;
             auto peer_fd = cl->peer_fd;
-            if (!cl->osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
+            if (!cl->osd_num && !cl->in_osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
             {
                 // Do not run keepalive on regular clients
                 continue;
@@ -199,7 +199,7 @@ void osd_messenger_t::init()
                 if (!cl->ping_time_remaining)
                 {
                     // Ping timed out, stop the client
-                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->osd_num, cl->peer_fd);
+                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->peer_fd);
                     stop_client(peer_fd, true);
                     // Restart iterator because it may be invalidated
                     cl_it = clients.upper_bound(peer_fd);
@@ -230,7 +230,7 @@ void osd_messenger_t::init()
                             return;
                         }
                         int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
-                        auto fail_osd_num = cl->osd_num;
+                        auto fail_osd_num = cl->in_osd_num ? cl->in_osd_num : cl->osd_num;
                         cl->ping_time_remaining = 0;
                         delete op;
                         if (fail_fd >= 0)

src/client/messenger.h

@@ -60,6 +60,7 @@ struct osd_client_t
     int ping_time_remaining = 0;
     int idle_time_remaining = 0;
     osd_num_t osd_num = 0;
+    osd_num_t in_osd_num = 0;
     bool is_incoming = false;
 
     void *in_buf = NULL;
@@ -98,6 +99,7 @@ struct osd_client_t
     std::vector<osd_op_t*> zc_free_list;
 
     ~osd_client_t();
+    void cancel_ops();
 };
 
 struct osd_wanted_peer_t
@@ -235,6 +237,7 @@ public:
     void outbox_push(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;
+    std::function<void(osd_num_t)> break_pg_locks;
     std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
     void read_requests();
     void send_replies();

src/client/msgr_op.h

@@ -173,6 +173,7 @@ struct osd_op_t
     osd_op_buf_list_t iov;
 
     ~osd_op_t();
+    void cancel();
 
     bool is_recovery_related();
 };

src/client/msgr_rdmacm.cpp

@@ -510,13 +510,12 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
     rc->qp = conn->cmid->qp;
     // And an osd_client_t
     auto cl = new osd_client_t();
-    cl->is_incoming = true;
     cl->peer_addr = conn->parsed_addr;
     cl->peer_port = conn->rdmacm_port;
     cl->peer_fd = conn->peer_fd;
     cl->peer_state = PEER_RDMA;
     cl->connect_timeout_id = -1;
-    cl->osd_num = peer_osd;
+    cl->in_osd_num = peer_osd;
     cl->in_buf = malloc_or_die(receive_buffer_size);
     cl->rdma_conn = rc;
     clients[conn->peer_fd] = cl;

src/client/msgr_receive.cpp

@@ -8,11 +8,12 @@ void osd_messenger_t::read_requests()
     for (int i = 0; i < read_ready_clients.size(); i++)
     {
         int peer_fd = read_ready_clients[i];
-        osd_client_t *cl = clients[peer_fd];
-        if (cl->read_msg.msg_iovlen)
+        auto cl_it = clients.find(peer_fd);
+        if (cl_it == clients.end() || !cl_it->second || cl_it->second->read_msg.msg_iovlen)
         {
             continue;
         }
+        auto cl = cl_it->second;
         if (cl->read_remaining < receive_buffer_size)
         {
             cl->read_iov.iov_base = cl->in_buf;
@@ -60,7 +61,8 @@ void osd_messenger_t::read_requests()
             {
                 result = -errno;
             }
-            handle_read(result, cl);
+            // like set_immediate
+            tfd->set_timer_us(0, false, [this, result, cl](int){ handle_read(result, cl); });
         }
     }
     read_ready_clients.clear();
@@ -232,7 +234,7 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
             {
                 if (cl->read_op->req.hdr.id != cl->read_op_id)
                 {
-                    fprintf(stderr, "Warning: operation sequencing is broken on client %d, stopping client\n", cl->peer_fd);
+                    fprintf(stderr, "Warning: operation sequencing is broken on client %d: expected num %ju, got %ju, stopping client\n", cl->peer_fd, cl->read_op_id, cl->read_op->req.hdr.id);
                     stop_client(cl->peer_fd);
                     return false;
                 }

src/client/msgr_send.cpp

@@ -239,7 +239,8 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         {
             result = -errno;
         }
-        handle_send(result, false, false, cl);
+        // like set_immediate
+        tfd->set_timer_us(0, false, [this, result, cl](int){ handle_send(result, false, false, cl); });
     }
     return true;
 }

src/client/msgr_stop.cpp

@@ -9,38 +9,37 @@
 #include "msgr_rdma.h"
 #endif
 
-void osd_messenger_t::cancel_osd_ops(osd_client_t *cl)
+void osd_client_t::cancel_ops()
 {
     std::vector<osd_op_t*> cancel_ops;
-    cancel_ops.resize(cl->sent_ops.size());
+    cancel_ops.resize(sent_ops.size());
     int i = 0;
-    for (auto p: cl->sent_ops)
+    for (auto p: sent_ops)
     {
         cancel_ops[i++] = p.second;
     }
-    cl->sent_ops.clear();
-    cl->outbox.clear();
+    sent_ops.clear();
     for (auto op: cancel_ops)
     {
-        cancel_op(op);
+        op->cancel();
     }
 }
 
-void osd_messenger_t::cancel_op(osd_op_t *op)
+void osd_op_t::cancel()
 {
-    if (op->op_type == OSD_OP_OUT)
+    if (op_type == OSD_OP_OUT && callback)
     {
-        op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
-        op->reply.hdr.id = op->req.hdr.id;
-        op->reply.hdr.opcode = op->req.hdr.opcode;
-        op->reply.hdr.retval = -EPIPE;
-        // Copy lambda to be unaffected by `delete op`
-        std::function<void(osd_op_t*)>(op->callback)(op);
+        reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+        reply.hdr.id = req.hdr.id;
+        reply.hdr.opcode = req.hdr.opcode;
+        reply.hdr.retval = -EPIPE;
+        // Copy lambda to be unaffected by `delete this`
+        (std::function<void(osd_op_t*)>(callback))(this);
     }
     else
     {
         // This function is only called in stop_client(), so it's fine to destroy the operation
-        delete op;
+        delete this;
     }
 }
 
@@ -63,6 +62,10 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         {
             fprintf(stderr, "[OSD %ju] Stopping client %d (OSD peer %ju)\n", osd_num, peer_fd, cl->osd_num);
         }
+        else if (cl->in_osd_num)
+        {
+            fprintf(stderr, "[OSD %ju] Stopping client %d (incoming OSD peer %ju)\n", osd_num, peer_fd, cl->in_osd_num);
+        }
         else
         {
             fprintf(stderr, "[OSD %ju] Stopping client %d (regular client)\n", osd_num, peer_fd);
@@ -73,8 +76,12 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     cl->peer_state = PEER_STOPPED;
     if (cl->osd_num)
     {
-        // ...and forget OSD peer
-        osd_peer_fds.erase(cl->osd_num);
+        auto osd_it = osd_peer_fds.find(cl->osd_num);
+        if (osd_it != osd_peer_fds.end() && osd_it->second == cl->peer_fd)
+        {
+            // ...and forget OSD peer
+            osd_peer_fds.erase(osd_it);
+        }
     }
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
@@ -101,30 +108,17 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         }
     }
 #endif
+    if (cl->in_osd_num && break_pg_locks)
+    {
+        // Break PG locks
+        break_pg_locks(cl->in_osd_num);
+    }
     if (cl->osd_num)
     {
         // Then repeer PGs because cancel_op() callbacks can try to perform
         // some actions and we need correct PG states to not do something silly
         repeer_pgs(cl->osd_num);
     }
-    // Then cancel all operations
-    if (cl->read_op)
-    {
-        if (!cl->read_op->callback)
-        {
-            delete cl->read_op;
-        }
-        else
-        {
-            cancel_op(cl->read_op);
-        }
-        cl->read_op = NULL;
-    }
-    if (cl->osd_num)
-    {
-        // Cancel outbound operations
-        cancel_osd_ops(cl);
-    }
     // Find the item again because it can be invalidated at this point
     it = clients.find(peer_fd);
     if (it != clients.end())
@@ -149,6 +143,17 @@ osd_client_t::~osd_client_t()
         close(peer_fd);
         peer_fd = -1;
     }
+    // Then cancel all operations
+    // Operations have to be canceled only after clearing all references to osd_client_t
+    // because otherwise their buffers may be still present in io_uring asynchronous requests
+    if (read_op)
+    {
+        // read_op may be an incoming op or a continued response for an outbound op
+        read_op->cancel();
+        read_op = NULL;
+    }
+    // Cancel outbound ops
+    cancel_ops();
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.2.1
+Version: 2.2.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/osd/osd.cpp

@@ -85,6 +85,7 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
     msgr.ringloop = this->ringloop;
     msgr.exec_op = [this](osd_op_t *op) { exec_op(op); };
     msgr.repeer_pgs = [this](osd_num_t peer_osd) { repeer_pgs(peer_osd); };
+    msgr.break_pg_locks = [this](osd_num_t peer_osd) { break_pg_locks(peer_osd); };
     msgr.check_config_hook = [this](osd_client_t *cl, json11::Json conf) { return check_peer_config(cl, conf); };
     msgr.init();
 

src/osd/osd.h

@@ -279,6 +279,7 @@ class osd_t
     bool check_peer_config(osd_client_t *cl, json11::Json conf);
     void repeer_pgs(osd_num_t osd_num);
     void repeer_pg(pg_t & pg);
+    void break_pg_locks(osd_num_t osd_num);
     void start_pg_peering(pg_t & pg);
     void drop_dirty_pg_connections(pool_pg_num_t pg);
     void record_pg_lock(pg_t & pg, osd_num_t peer_osd, uint64_t pg_state);

src/osd/osd_cluster.cpp

@@ -434,8 +434,8 @@ void osd_t::apply_pg_locks_localize_only()
         auto & pg = pp.second;
         auto old_disable_pg_locks = pg.disable_pg_locks;
         pg.disable_pg_locks = pg_locks_localize_only &&
-            pool_cfg.scheme == POOL_SCHEME_REPLICATED &&
-            pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY;
+            (pool_cfg.scheme != POOL_SCHEME_REPLICATED ||
+            pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY);
         if (!pg.disable_pg_locks && old_disable_pg_locks)
         {
             // Relock PG
@@ -884,8 +884,8 @@ void osd_t::apply_pg_config()
                 pg.next_scrub = pg_cfg.next_scrub;
                 pg.target_set = pg_cfg.target_set;
                 pg.disable_pg_locks = pg_locks_localize_only &&
-                    pool_item.second.scheme == POOL_SCHEME_REPLICATED &&
-                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY;
+                    (pool_item.second.scheme != POOL_SCHEME_REPLICATED ||
+                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY);
                 if (pg.scheme == POOL_SCHEME_EC)
                 {
                     use_ec(pg.pg_size, pg.pg_data_size, true);

src/osd/osd_peering.cpp

@@ -73,18 +73,25 @@ void osd_t::handle_peers()
     }
 }
 
+void osd_t::break_pg_locks(osd_num_t peer_osd)
+{
+    for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
+    {
+        if (lock_it->second.primary_osd == peer_osd)
+        {
+            if (log_level > 3)
+            {
+                printf("Break PG %u/%u lock on disconnection of OSD %ju\n", lock_it->first.pool_id, lock_it->first.pg_num, peer_osd);
+            }
+            pg_locks.erase(lock_it++);
+        }
+        else
+            lock_it++;
+    }
+}
+
 void osd_t::repeer_pgs(osd_num_t peer_osd)
 {
-    if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
-    {
-        for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
-        {
-            if (lock_it->second.primary_osd == peer_osd)
-                pg_locks.erase(lock_it++);
-            else
-                lock_it++;
-        }
-    }
     // Re-peer affected PGs
     for (auto & p: pgs)
     {
@@ -471,6 +478,7 @@ void osd_t::relock_pg(pg_t & pg)
             auto pg_it = pgs.find(pg_id);
             if (pg_it == pgs.end())
             {
+                printf("Warning: PG %u/%u is gone during lock attempt\n", pg_id.pool_id, pg_id.pg_num);
                 return;
             }
             auto & pg = pg_it->second;

src/osd/osd_primary_subops.cpp

@@ -46,6 +46,11 @@ void osd_t::autosync()
 void osd_t::finish_op(osd_op_t *cur_op, int retval)
 {
     inflight_ops--;
+    if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
+    {
+        printf("%jx %jx+%x p%jx v%jx r=%x\n", cur_op->req.rw.inode, cur_op->req.rw.offset, cur_op->req.rw.len,
+            cur_op->op_data ? cur_op->op_data->orig_ver : 0, cur_op->reply.rw.version, retval);
+    }
     if (cur_op->req.hdr.opcode == OSD_OP_READ ||
         cur_op->req.hdr.opcode == OSD_OP_WRITE ||
         cur_op->req.hdr.opcode == OSD_OP_DELETE)
@@ -417,15 +422,17 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
     if (retval != expected)
     {
         int64_t peer_osd = (msgr.clients.find(subop->peer_fd) != msgr.clients.end()
-            ? msgr.clients[subop->peer_fd]->osd_num : -subop->peer_fd);
+            ? msgr.clients[subop->peer_fd]->osd_num : 0);
         if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
         {
             printf(
                 subop->peer_fd >= 0
-                    ? "%1$s subop to %2$jx:%3$jx v%4$ju failed on osd %7$jd: retval = %5$d (expected %6$d)\n"
+                    ? (peer_osd > 0
+                        ? "%1$s subop to %2$jx:%3$jx v%4$ju failed on osd %7$ju: retval = %5$d (expected %6$d)\n"
+                        : "%1$s subop to %2$jx:%3$jx v%4$ju failed on peer %8$d: retval = %5$d (expected %6$d)\n")
                     : "%1$s subop to %2$jx:%3$jx v%4$ju failed locally: retval = %5$d (expected %6$d)\n",
                 osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version,
-                retval, expected, peer_osd
+                retval, expected, peer_osd, subop->peer_fd
             );
         }
         else if (opcode == OSD_OP_SEC_DELETE)

src/osd/osd_scrub.cpp

@@ -505,7 +505,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 if (votes[role] > 0 && votes[role] < votes[best])
                 {
                     printf(
-                        "[PG %u/%u] Object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
+                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[role].osd_num, votes[best],
@@ -528,7 +528,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 best = -1;
                 inconsistent = true;
                 printf(
-                    "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
+                    "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
                     INODE_POOL(op_data->oid.inode), op_data->pg_num,
                     op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
                 );
@@ -547,7 +547,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
         {
             inconsistent = true;
             printf(
-                "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
+                "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
                 INODE_POOL(op_data->oid.inode), op_data->pg_num,
                 op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
             );
@@ -574,7 +574,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 {
                     op_data->stripes[i].read_error = true;
                     printf(
-                        "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
+                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[i].role, op_data->stripes[i].osd_num,

src/osd/osd_secondary.cpp

@@ -91,16 +91,17 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
     {
         return false;
     }
+    auto & pool_cfg = pool_cfg_it->second;
+    if (pg_locks_localize_only && (pool_cfg.scheme != POOL_SCHEME_REPLICATED || pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY))
+    {
+        return true;
+    }
     auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) };
     auto pg_it = pgs.find(ppg);
     if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE)
     {
         return false;
     }
-    if (pg_it->second.disable_pg_locks)
-    {
-        return true;
-    }
     auto lock_it = pg_locks.find(ppg);
     return lock_it != pg_locks.end() && lock_it->second.primary_osd == primary_osd;
 }
@@ -140,7 +141,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {
         if (!(cur_op->req.sec_rw.flags & OSD_OP_IGNORE_PG_LOCK) &&
-            !sec_check_pg_lock(cl->osd_num, cur_op->req.sec_rw.oid))
+            !sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_rw.oid))
         {
             cur_op->bs_op->retval = -EPIPE;
             secondary_op_callback(cur_op);
@@ -169,7 +170,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
     else if (cur_op->req.hdr.opcode == OSD_OP_SEC_DELETE)
     {
         if (!(cur_op->req.sec_del.flags & OSD_OP_IGNORE_PG_LOCK) &&
-            !sec_check_pg_lock(cl->osd_num, cur_op->req.sec_del.oid))
+            !sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_del.oid))
         {
             cur_op->bs_op->retval = -EPIPE;
             secondary_op_callback(cur_op);
@@ -193,7 +194,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         {
             for (int i = 0; i < cur_op->bs_op->len; i++)
             {
-                if (!sec_check_pg_lock(cl->osd_num, ((obj_ver_id*)cur_op->buf)[i].oid))
+                if (!sec_check_pg_lock(cl->in_osd_num, ((obj_ver_id*)cur_op->buf)[i].oid))
                 {
                     cur_op->bs_op->retval = -EPIPE;
                     secondary_op_callback(cur_op);
@@ -247,7 +248,7 @@ void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
         void *cur_buf = reply_buf;
         for (int i = 0; i < n; i++)
         {
-            if (!sec_check_pg_lock(cl->osd_num, ov[i].oid) &&
+            if (!sec_check_pg_lock(cl->in_osd_num, ov[i].oid) &&
                 !(cur_op->req.sec_read_bmp.flags & OSD_OP_IGNORE_PG_LOCK))
             {
                 free(reply_buf);
@@ -269,7 +270,7 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
 {
     cur_op->reply.sec_lock.cur_primary = 0;
     auto cl = msgr.clients.at(cur_op->peer_fd);
-    if (!cl->osd_num ||
+    if (!cl->in_osd_num ||
         cur_op->req.sec_lock.flags != OSD_SEC_LOCK_PG &&
         cur_op->req.sec_lock.flags != OSD_SEC_UNLOCK_PG ||
         cur_op->req.sec_lock.pool_id > ((uint64_t)1<<POOL_ID_BITS) ||
@@ -290,7 +291,7 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
     auto lock_it = pg_locks.find(ppg);
     if (cur_op->req.sec_lock.flags == OSD_SEC_LOCK_PG)
     {
-        if (lock_it != pg_locks.end() && lock_it->second.primary_osd != cl->osd_num)
+        if (lock_it != pg_locks.end() && lock_it->second.primary_osd != cl->in_osd_num)
         {
             cur_op->reply.sec_lock.cur_primary = lock_it->second.primary_osd;
             finish_op(cur_op, -EBUSY);
@@ -303,13 +304,21 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
             finish_op(cur_op, -EBUSY);
             return;
         }
+        if (log_level > 3)
+        {
+            printf("Lock PG %u/%u for OSD %ju\n", ppg.pool_id, ppg.pg_num, cl->in_osd_num);
+        }
         pg_locks[ppg] = (osd_pg_lock_t){
-            .primary_osd = cl->osd_num,
+            .primary_osd = cl->in_osd_num,
             .state = cur_op->req.sec_lock.pg_state,
         };
     }
-    else if (lock_it != pg_locks.end() && lock_it->second.primary_osd == cl->osd_num)
+    else if (lock_it != pg_locks.end() && lock_it->second.primary_osd == cl->in_osd_num)
     {
+        if (log_level > 3)
+        {
+            printf("Unlock PG %u/%u by OSD %ju\n", ppg.pool_id, ppg.pg_num, cl->in_osd_num);
+        }
         pg_locks.erase(lock_it);
     }
     finish_op(cur_op, 0);
@@ -323,7 +332,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
         : json11::Json();
     auto peer_osd_num = req_json["osd_num"].uint64_value();
     auto cl = msgr.clients.at(cur_op->peer_fd);
-    cl->osd_num = peer_osd_num;
+    cl->in_osd_num = peer_osd_num;
     if (req_json["features"]["check_sequencing"].bool_value())
     {
         cl->check_sequencing = true;

src/test/test_cluster_client.cpp

@@ -121,6 +121,7 @@ void pretend_connected(cluster_client_t *cli, osd_num_t osd_num)
     cli->msgr.osd_peer_fds[osd_num] = peer_fd;
     cli->msgr.clients[peer_fd] = new osd_client_t();
     cli->msgr.clients[peer_fd]->osd_num = osd_num;
+    cli->msgr.clients[peer_fd]->peer_fd = peer_fd;
     cli->msgr.clients[peer_fd]->peer_state = PEER_CONNECTED;
     cli->msgr.wanted_peers.erase(osd_num);
     cli->msgr.repeer_pgs(osd_num);

tests/test_heal.sh

@@ -55,6 +55,20 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
         -serialize_overlap=1 -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/bin/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
 
+# Wait for active/clean
+wait_up 120
+
+# Trigger scrub
+for i in $(seq 1 $PG_COUNT); do
+    $ETCDCTL put /vitastor/pg/history/1/$i `$ETCDCTL get --print-value-only /vitastor/pg/history/1/$i | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
+done
+
+# Wait for scrub to finish
+wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
+
+# Check that everything is ok
+wait_up 1
+
 qemu-img convert -S 4096 -p \
     -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
     -O raw ./testdata/bin/read.bin
@@ -64,7 +78,7 @@ if ! diff -q ./testdata/bin/read.bin ./testdata/bin/mirror.bin; then
     format_error Data lost during self-heal
 fi
 
-if grep -qP 'Checksum mismatch|BUG' ./testdata/osd*.log; then
+if grep -qP 'Checksum mismatch|BUG|Scrub error' ./testdata/osd*.log; then
     format_error Checksum mismatches or BUGs detected during test
 fi