Release 3.2.2

- New store (thanks Changwei Ye)
  - Fix rejected compaction when an object is already overwritten again
  - Reduce superblock I/O amplification introduced after previous correctness fixes
- Fix possible `vitastor-cli rm` crash with `assert(rwo->todo == 1)` introduced in 3.2.1
- Fix replicated objects possibly incorrectly transitioning to the INCOMPLETE state
  after multiple failed partial writes
- Fix possible "operation sequencing broken ... expected num 2, got 3, stopping client"
  possible when a cluster is overloaded by peering/OSD flaps

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.2.1")
+set(VITASTOR_VERSION "3.2.2")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.2.1
+VITASTOR_VERSION ?= v3.2.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.2.1
+          image: vitalif/vitastor-csi:v3.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -87,7 +87,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-attacher
-          image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.2
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.2.1
+          image: vitalif/vitastor-csi:v3.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.2.1"
+    vitastorCSIDriverVersion = "3.2.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.2.1-1) unstable; urgency=medium
+vitastor (3.2.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.2.1
+VITASTOR_VERSION ?= v3.2.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.2.1
+VITASTOR_VERSION=v3.2.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.2.1`
+   `docker pull vitalif/vitastor:v3.2.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.1 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 4. Enable the vitastor-host service: \

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.2.1`
+   `docker pull vitalif/vitastor:v3.2.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.1 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 4. Включите сервис vitastor-host: \

docs/usage/cli.en.md

@@ -196,6 +196,7 @@ Options can be specified in classic dd style (`key=value`) or like usual (`--key
 | `iimg=<image>`  | Copy from Vitastor image `<image>`                                      |
 | `if=<file>`     | Copy from file `<file>`                                                 |
 | `oimg=<image>`  | Copy to Vitastor image `<image>`                                        |
+| `opool=<pool>`  | Pool to create output image if it doesn't exist                         |
 | `of=<file>`     | Copy to file `<file>`                                                   |
 | `bs=1M`         | Set copy block size                                                     |
 | `count=N`       | Copy only N input blocks. If N ends in B it counts bytes, not blocks    |

docs/usage/cli.ru.md

@@ -203,6 +203,7 @@ vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
 | `iimg=<image>`  | Копировать из образа Vitastor `<image>`                                 |
 | `if=<file>`     | Копировать из файла `<file>`                                            |
 | `oimg=<image>`  | Копировать в образ Vitastor `<image>`                                   |
+| `opool=<pool>`  | Пул для создания выходного образа, если он не существует                |
 | `of=<file>`     | Копировать в файл `<file>`                                              |
 | `bs=1M`         | Задать размер блока копирования                                         |
 | `count=N`       | Копировать не более N блоков. Если N заканчивается на B - то N байт.    |

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.2.1'
+VITASTOR_VERSION = '3.2.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el10.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el10.tar.gz
+Source0:        vitastor-3.2.2.el10.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el7.tar.gz
+Source0:        vitastor-3.2.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el8.tar.gz
+Source0:        vitastor-3.2.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el9.tar.gz
+Source0:        vitastor-3.2.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.2.1")
+add_definitions(-DVITASTOR_VERSION="3.2.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_flush.cpp

@@ -213,18 +213,25 @@ resume_0:
     res = bs->heap->get_next_compact(cur_oid);
     if ((bs->intent_write_counter >= bs->journal_trim_interval || flusher->force_fsync > 0) && co_id == 0)
     {
-        // Advance fsynced_lsn every <journal_trim_interval> intent writes
-        bs->intent_write_counter = 0;
 resume_17:
 resume_18:
 resume_19:
         if (!fsync_buffer(17))
+        {
             return false;
+        }
+        if (bs->intent_write_counter >= bs->journal_trim_interval)
+        {
+            // Advance fsynced_lsn every <journal_trim_interval> intent writes
+            bs->intent_write_counter = 0;
 resume_29:
 resume_30:
 resume_31:
-        if (!trim_lsn(29))
-            return false;
+            if (!trim_lsn(29))
+            {
+                return false;
+            }
+        }
     }
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
         (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))

src/blockstore/blockstore_heap.cpp

@@ -422,7 +422,7 @@ corrupted_object:
             if ((wr->type() == BS_HEAP_BIG_INTENT || wr->type() == BS_HEAP_BIG_WRITE) &&
                 wr->big().block_num >= dsk->block_count)
             {
-                fprintf(stderr, "Error: big_write or big_intent entry %jx:%jx v%ju block_num is too large: %u > %lu. Metadata is incompatible with current parameters. ",
+                fprintf(stderr, "Error: big_write or big_intent entry %jx:%jx v%ju block_num is too large: %u > %ju. Metadata is incompatible with current parameters. ",
                     wr->inode, wr->stripe, wr->version, wr->big_intent().block_num, dsk->block_count);
                 goto corrupted_object;
             }
@@ -2436,9 +2436,11 @@ heap_compact_t blockstore_heap_t::iterate_compaction(heap_entry_t *obj, uint64_t
             res.do_delete = false;
             res.compact_lsn = 0;
             res.compact_version = 0;
-            if (!under_pressure && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_DELETE))
+            if ((!under_pressure || stable) && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT || wr->type() == BS_HEAP_DELETE))
             {
                 // We may postpone compaction if we have an unstable overwrite when not under pressure
+                // We MUST postpone if the overwrite is STABLE (even if under pressure), 
+                // because add_compact will reject STABLE overwrites with EBUSY.
                 return res;
             }
             continue;

src/client/cluster_client.cpp

@@ -48,7 +48,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     msgr.exec_op = [this](osd_op_t *op)
     {
         // Garbage in
-        fprintf(stderr, "Can't handle incoming operation from client %lu\n", op->client_id);
+        fprintf(stderr, "Can't handle incoming operation from client %ju\n", op->client_id);
         msgr.stop_client(op->client_id);
         delete op;
     };
@@ -591,7 +591,7 @@ void cluster_client_t::on_change_pool_config_hook()
         {
             if (log_level > 2 && pg_counts[pool_item.first])
             {
-                fprintf(stderr, "Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                fprintf(stderr, "Pool %u (%s) PG count changed from %ju to %ju\n", pool_item.first, pool_item.second.name.c_str(),
                     pg_counts[pool_item.first], pool_item.second.real_pg_count);
             }
             // At this point, all pool operations should have been suspended

src/client/msgr_receive.cpp

@@ -700,15 +700,13 @@ bool osd_messenger_t::handle_hdr(osd_client_t *cl)
     }
     else if (cl->read_op->req.hdr.magic == SECONDARY_OSD_OP_MAGIC)
     {
-        if (cl->check_sequencing)
+        if (cl->check_sequencing && cl->read_op->req.hdr.id != cl->read_op_id)
         {
-            if (cl->read_op->req.hdr.id != cl->read_op_id)
-            {
-                fprintf(stderr, "Warning: operation sequencing is broken on client %d: expected num %ju, got %ju, stopping client\n", cl->peer_fd, cl->read_op_id, cl->read_op->req.hdr.id);
-                return false;
-            }
-            cl->read_op_id++;
+            fprintf(stderr, "Warning: operation sequencing is broken on client %ju: expected num %ju, got %ju, stopping client\n", cl->client_id, cl->read_op_id, cl->read_op->req.hdr.id);
+            return false;
         }
+        // Track expected op_ids before check_sequencing to not break if a PING sneaks before SHOW_CONFIG reply
+        cl->read_op_id = cl->read_op->req.hdr.id + 1;
         if (!allocate_op_buffers(cl))
         {
             return false;
@@ -775,7 +773,7 @@ bool osd_messenger_t::allocate_op_buffers(osd_client_t *cl)
             {
                 if (log_level > 1)
                 {
-                    fprintf(stderr, "Error: peer %ju stabilize request size too large (%lu > %u bytes), stopping\n", cl->client_id,
+                    fprintf(stderr, "Error: peer %ju stabilize request size too large (%ju > %u bytes), stopping\n", cl->client_id,
                         cur_op->req.sec_stab.len, MAX_SIMPLE_PAYLOAD_SIZE);
                 }
                 return false;
@@ -792,7 +790,7 @@ bool osd_messenger_t::allocate_op_buffers(osd_client_t *cl)
             {
                 if (log_level > 1)
                 {
-                    fprintf(stderr, "Error: peer %ju sec_read_bmp request size too large (%lu > %u bytes), stopping\n", cl->client_id,
+                    fprintf(stderr, "Error: peer %ju sec_read_bmp request size too large (%ju > %u bytes), stopping\n", cl->client_id,
                         cur_op->req.sec_read_bmp.len, MAX_SIMPLE_PAYLOAD_SIZE);
                 }
                 return false;
@@ -826,7 +824,7 @@ bool osd_messenger_t::allocate_op_buffers(osd_client_t *cl)
             {
                 if (log_level > 1)
                 {
-                    fprintf(stderr, "Error: peer %ju show_config request length too large (%lu > %u bytes), stopping\n", cl->client_id,
+                    fprintf(stderr, "Error: peer %ju show_config request length too large (%ju > %u bytes), stopping\n", cl->client_id,
                         cur_op->req.show_conf.json_len, MAX_SIMPLE_PAYLOAD_SIZE);
                 }
                 return false;

src/client/qemu_driver.c

@@ -329,7 +329,9 @@ static void coroutine_fn vitastor_co_resize(void *opaque)
     }
 #else
     BdrvChild *c;
+#if QEMU_VERSION_MAJOR >= 8
     WITH_GRAPH_RDLOCK_GUARD() {
+#endif
 #if QEMU_VERSION_MAJOR >= 9
         assert_bdrv_graph_readable();
 #endif
@@ -344,7 +346,9 @@ static void coroutine_fn vitastor_co_resize(void *opaque)
             }
 #endif
         }
+#if QEMU_VERSION_MAJOR >= 8
     }
+#endif
 #endif
     bdrv_dec_in_flight(bs);
 }

src/client/ublk_server.cpp

@@ -706,7 +706,7 @@ protected:
         {
             if (data->res != 0)
             {
-                fprintf(stderr, "Failed to update size of /dev/ublkb%d to %lu bytes: %s (code %d)\n", ublk_dev.dev_id, size, strerror(-data->res), data->res);
+                fprintf(stderr, "Failed to update size of /dev/ublkb%d to %ju bytes: %s (code %d)\n", ublk_dev.dev_id, size, strerror(-data->res), data->res);
             }
         };
     }

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.2.1
+Version: 3.2.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_merge.cpp

@@ -583,7 +583,7 @@ struct snap_merger_t
                 if (use_cas && subop->retval == -EINTR)
                 {
                     // CAS failure - reread and repeat optimistically
-                    assert(rwo->todo == 1); // initial refcount from read_and_write
+                    assert(!rwo->todo);
                     rwo->error_code = -EINTR;
                     rwo->start = rwo->end = 0;
                     rwo->op.version = 0;

src/disk_tool/disk_tool_meta.cpp

@@ -184,7 +184,7 @@ csum_unknown:
                             uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + dsk.clean_entry_size - 4);
                             if (*entry_csum != crc32c(0, entry, dsk.clean_entry_size - 4))
                             {
-                                fprintf(stderr, "Metadata entry %lu is corrupt (checksum mismatch), skipping\n", block_num);
+                                fprintf(stderr, "Metadata entry %ju is corrupt (checksum mismatch), skipping\n", block_num);
                                 continue;
                             }
                         }

src/osd/osd_primary.cpp

@@ -485,24 +485,26 @@ pg_osd_set_state_t *osd_t::mark_partial_write(pg_t & pg, osd_op_t *cur_op)
         int changes = 0;
         for (auto & chunk: new_set)
         {
-            if (chunk.loc_bad != LOC_OUTDATED)
+            bool success = false;
+            for (int i = 0; i < op_data->n_subops; i++)
             {
-                bool success = false;
-                for (int i = 0; i < op_data->n_subops; i++)
+                if (op_data->subops[i].osd_num == chunk.osd_num &&
+                    op_data->subops[i].reply.hdr.retval == op_data->subops[i].req.sec_rw.len)
                 {
-                    if (op_data->subops[i].osd_num == chunk.osd_num &&
-                        op_data->subops[i].reply.hdr.retval == op_data->subops[i].req.sec_rw.len)
-                    {
-                        success = true;
-                        break;
-                    }
-                }
-                if (!success)
-                {
-                    changes++;
-                    chunk.loc_bad = LOC_OUTDATED;
+                    success = true;
+                    break;
                 }
             }
+            if (!success && !(chunk.loc_bad & LOC_OUTDATED))
+            {
+                changes++;
+                chunk.loc_bad = LOC_OUTDATED;
+            }
+            else if (success && (chunk.loc_bad & LOC_OUTDATED))
+            {
+                changes++;
+                chunk.loc_bad &= ~LOC_OUTDATED;
+            }
         }
         return changes;
     });

src/osd/osd_secondary.cpp

@@ -370,7 +370,6 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
     if (req_json["features"]["check_sequencing"].bool_value())
     {
         cl->check_sequencing = true;
-        cl->read_op_id = cur_op->req.hdr.id + 1;
     }
     auto features = json11::Json::object{ { "pg_locks", true } };
     if (msgr.use_proto_checksums)

src/test/osd_test.cpp

@@ -144,6 +144,20 @@ void test_replicated_write()
     printf("test_replicated_write passed\n");
 }
 
+// Find the object's current pg_osd_set_state_t in whatever "not clean" index
+// the PG keeps it in. NULL means the object is considered clean.
+static pg_osd_set_state_t *find_object_state(pg_t & pg, object_id oid)
+{
+    for (auto *idx: { &pg.inconsistent_objects, &pg.incomplete_objects,
+        &pg.degraded_objects, &pg.misplaced_objects })
+    {
+        auto it = idx->find(oid);
+        if (it != idx->end())
+            return it->second;
+    }
+    return NULL;
+}
+
 // Regression test for a bug where a second scrub on a still-failing replica
 // (already marked LOC_CORRUPTED) would clear the corruption flag.
 void test_scrub_corruption_persists()
@@ -2035,6 +2049,113 @@ void test_ec42_inconsistent_read()
     printf("test_ec42_inconsistent_read passed\n");
 }
 
+// Regression test for a bug where a repeated partially failed write to the same
+// object didn't clear LOC_OUTDATED from the replicas that succeeded this time —
+// so a copy stayed marked outdated even though it had just received the newest
+// version, and every failing write only ever added outdated copies.
+//
+// Layout: pool 1, replicated x3, primary = OSD 1 (us), secondaries = OSD 2 and 3.
+//
+// Write 1 fails with ENOSPC on OSD 3     -> OSD 3 is marked outdated.
+// Write 2 fails with ENOSPC on OSD 1 (us) but succeeds on OSD 2 and OSD 3
+//   -> OSD 1 must become outdated AND OSD 3 must stop being outdated.
+void test_repeated_partial_write_unmarks_outdated()
+{
+    printf("test_repeated_partial_write_unmarks_outdated\n");
+
+    osd_test_fixture_t f;
+    f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 3, /*pg_minsize*/ 1, /*pg_count*/ 1,
+        { { 1, 2, 3 } });
+    f.start(json11::Json::object {
+        { "osd_num", 1 },
+        { "etcd_address", "127.0.0.1:2379" },
+        { "immediate_commit", "all" },
+        { "block_size", 131072 },
+        { "bitmap_granularity", 4096 },
+    });
+    f.connect_peer(2);
+    f.connect_peer(3);
+    f.complete_peering_empty();
+    assert(f.pg(1, 1).state & PG_ACTIVE);
+
+    inode_t inode = INODE_WITH_POOL(1, 1);
+    object_id oid = { .inode = inode, .stripe = 0 };
+
+    // Is the copy on <osd_num> marked outdated? A copy that vanished from the
+    // set entirely counts as outdated, too.
+    auto outdated_on = [&](osd_num_t osd_num)
+    {
+        auto *st = find_object_state(f.pg(1, 1), oid);
+        assert(st);
+        for (auto & chunk: st->osd_set)
+            if (chunk.osd_num == osd_num)
+                return (chunk.loc_bad & LOC_OUTDATED) != 0;
+        return true;
+    };
+
+    // ============ WRITE 1 — ENOSPC on OSD 3 ============
+
+    // Full-block writes on purpose: a sub-block write to a degraded object
+    // would turn into a read-modify-write and only obscure what's tested here.
+    auto *write1 = make_write_op(inode, 0, 131072, 0xab);
+    int retval1 = -1;
+    write1->callback = [&retval1](osd_op_t *op) { retval1 = op->reply.hdr.retval; };
+    f.exec(write1);
+
+    // Zero-length read resolving the (not yet existing) object version
+    f.bs_zero_read_ok(0);
+
+    // One local write + one write per peer
+    assert(f.bs->queued.size() == 1);
+    f.bs_write_ok(BS_OP_WRITE_STABLE, 1);
+    f.peer_write_ok(2, OSD_OP_SEC_WRITE_STABLE, 1);
+    auto *peer3_w1 = f.peer_take(3, OSD_OP_SEC_WRITE_STABLE);
+    assert(retval1 == -1); // still waiting for OSD 3
+    f.peer_complete(peer3_w1, -ENOSPC);
+
+    assert(retval1 == -ENOSPC);
+    assert(f.pg(1, 1).state & PG_HAS_DEGRADED);
+    assert(!outdated_on(1));
+    assert(!outdated_on(2));
+    assert(outdated_on(3));
+    printf("test_repeated_partial_write_unmarks_outdated: write 1 -> OSD 3 outdated\n");
+    delete write1;
+
+    // ============ WRITE 2 — ENOSPC locally, OK on both peers ============
+
+    auto *write2 = make_write_op(inode, 0, 131072, 0xcd);
+    int retval2 = -1;
+    write2->callback = [&retval2](osd_op_t *op) { retval2 = op->reply.hdr.retval; };
+    f.exec(write2);
+
+    // The object exists at version 1 now
+    f.bs_zero_read_ok(1);
+
+    assert(f.bs->queued.size() == 1);
+    auto *local_w2 = f.bs->take(BS_OP_WRITE_STABLE);
+    assert(local_w2->version == 2);
+    local_w2->retval = -ENOSPC;
+    local_w2->callback(local_w2);
+    f.peer_write_ok(2, OSD_OP_SEC_WRITE_STABLE, 2);
+    f.peer_write_ok(3, OSD_OP_SEC_WRITE_STABLE, 2);
+
+    assert(retval2 == -ENOSPC);
+
+    // THE KEY ASSERTION — OSD 3 took the new version, it isn't outdated anymore
+    if (outdated_on(3))
+        printf("BUG: OSD 3 is still marked outdated after successfully taking the new version\n");
+    assert(!outdated_on(3));
+    // ... and the OSD that missed this write is
+    assert(outdated_on(1));
+    assert(!outdated_on(2));
+
+    assert(f.pg(1, 1).inflight == 0);
+    assert(f.pg(1, 1).write_queue.empty());
+
+    delete write2;
+    printf("test_repeated_partial_write_unmarks_outdated passed\n");
+}
+
 int main(int narg, char *args[])
 {
     test_load_global_config();
@@ -2058,5 +2179,6 @@ int main(int narg, char *args[])
     test_no_new_write_on_non_active_pg();
     test_pg_epoch_bump_blocks_write_until_reported();
     test_ec42_inconsistent_read();
+    test_repeated_partial_write_unmarks_outdated();
     return 0;
 }

src/test/test_heap.cpp

@@ -1010,6 +1010,211 @@ void test_iterate_compaction()
         assert(small_writes == 1);
     }
 
+    {
+        blockstore_heap_t heap(&dsk, buffer_area.data());
+        heap.finish_recheck();
+
+        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_STABLE(v3 l3) un-fsynced
+        // -> skip compaction of l2 into l1 EVEN WHEN under pressure,
+        // because add_compact rejects stable overwrites with EBUSY
+        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
+        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
+
+        heap.use_data(INODE_WITH_POOL(1, 1), 128*1024);
+        uint32_t mblock = 999999;
+        res = _test_do_big_write(heap, dsk, 1, 0, 3, 128*1024, true, 0, 0, buffer_area.data(), &mblock);
+        assert(res == 0);
+        heap.start_block_write(mblock);
+
+        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
+        auto obj = heap.read_entry(oid);
+        assert(obj->lsn == 3);
+        assert(obj->is_overwrite());
+        assert(heap.get_fsynced_lsn() == 2);
+
+        // When not under pressure: postpone compaction
+        int small_writes = 0;
+        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // When under pressure: MUST ALSO postpone compaction because obj is STABLE!
+        // Before the fix, this would incorrectly compact l2 (compact_lsn == 2)
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // Verify that add_compact() rejects stable overwrites with EBUSY
+        uint32_t compact_mblock = 999999;
+        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
+        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
+        assert(res == EBUSY);
+
+        heap.complete_block_write(mblock);
+        assert(heap.get_fsynced_lsn() == 3);
+
+        // Once fsynced, l3 is durable and becomes the clean_wr base
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.clean_wr && compact_info.clean_wr->lsn == 3);
+        assert(small_writes == 0);
+    }
+
+    {
+        blockstore_heap_t heap(&dsk, buffer_area.data());
+        heap.finish_recheck();
+
+        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_INTENT(v3 l3) un-fsynced
+        // -> skip compaction of l2 into l1 both when not under pressure and under pressure
+        // Before the fix, BIG_INTENT was missing from the check and was never skipped.
+        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
+        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
+
+        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
+        auto obj = heap.read_entry(oid);
+        uint8_t ext_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ext_bitmap, 0x8e, dsk.clean_entry_bitmap_size);
+        heap.use_data(INODE_WITH_POOL(1, 1), 128*1024);
+        uint32_t mblock = 999999;
+        res = heap.add_redirect_intent(oid, &obj, 3, 0, dsk.data_block_size, 128*1024, ext_bitmap, buffer_area.data(), &mblock);
+        assert(res == 0);
+        heap.start_block_write(mblock);
+
+        obj = heap.read_entry(oid);
+        assert(obj->lsn == 3);
+        assert(obj->type() == BS_HEAP_BIG_INTENT);
+        assert(obj->is_overwrite());
+        assert(heap.get_fsynced_lsn() == 2);
+
+        // When not under pressure: MUST postpone compaction!
+        // Before the fix, BIG_INTENT was not checked, so it incorrectly compacted l2 (compact_lsn == 2)
+        int small_writes = 0;
+        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // When under pressure: MUST ALSO postpone compaction!
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // Verify add_compact() rejects stable overwrite with EBUSY
+        uint32_t compact_mblock = 999999;
+        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
+        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
+        assert(res == EBUSY);
+
+        // Complete block write
+        heap.complete_block_write(mblock);
+        assert(heap.get_fsynced_lsn() == 3);
+
+        // Once fsynced, BIG_INTENT is clean_wr base
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.clean_wr && compact_info.clean_wr->lsn == 3);
+        assert(small_writes == 0);
+    }
+
+    {
+        blockstore_heap_t heap(&dsk, buffer_area.data());
+        heap.finish_recheck();
+
+        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) DELETE_STABLE(l3) un-fsynced
+        // -> skip compaction of l2 into l1 EVEN WHEN under pressure,
+        // because add_compact rejects stable overwrites with EBUSY
+        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
+        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
+
+        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
+        auto obj = heap.read_entry(oid);
+        uint32_t mblock = 999999;
+        res = heap.add_delete(obj, &mblock);
+        assert(res == 0);
+        heap.start_block_write(mblock);
+
+        obj = heap.read_entry(oid);
+        assert(obj->lsn == 3);
+        assert(obj->type() == BS_HEAP_DELETE);
+        assert(obj->is_overwrite());
+        assert(heap.get_fsynced_lsn() == 2);
+
+        // When not under pressure: postpone compaction
+        int small_writes = 0;
+        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // When under pressure: MUST ALSO postpone compaction because DELETE is STABLE!
+        // Before the fix, this would incorrectly compact l2 (compact_lsn == 2)
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // Verify add_compact() rejects stable overwrite with EBUSY
+        uint32_t compact_mblock = 999999;
+        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
+        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
+        assert(res == EBUSY);
+
+        // Complete block write
+        heap.complete_block_write(mblock);
+        assert(heap.get_fsynced_lsn() == 3);
+
+        // Once fsynced, DELETE is processed
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.do_delete);
+        assert(!compact_info.clean_wr);
+        assert(small_writes == 0);
+    }
+
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
         heap.finish_recheck();

tests/test_snapshot.sh

@@ -72,4 +72,19 @@ qemu-img convert -S 4096 -f qcow2 ./testdata/layer1.qcow2 -O raw ./testdata/bin/
 
 cmp ./testdata/bin/layer1.bin ./testdata/bin/rebased.bin
 
+# Test CAS merge
+
+$VITASTOR_CLI create -s 1M microimg
+
+$VITASTOR_FIO -bs=1M -direct=1 -iodepth=1 -rw=write -image=microimg
+
+$VITASTOR_CLI snap-create microimg@0
+
+$VITASTOR_FIO -bs=16k -direct=1 -iodepth=1 -rw=randwrite -image=microimg -size=16k -loops=1000000 -runtime=10 &
+FIO_PID=$!
+
+sleep 5
+$VITASTOR_CLI rm microimg@0
+wait $FIO_PID
+
 format_green OK