Release 3.2.2

- New store (thanks Changwei Ye)
  - Fix rejected compaction when an object is already overwritten again
  - Reduce superblock I/O amplification introduced after previous correctness fixes
- Fix possible `vitastor-cli rm` crash with `assert(rwo->todo == 1)` introduced in 3.2.1
- Fix replicated objects possibly incorrectly transitioning to the INCOMPLETE state
  after multiple failed partial writes
- Fix possible "operation sequencing broken ... expected num 2, got 3, stopping client"
  possible when a cluster is overloaded by peering/OSD flaps

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.2.1")
+set(VITASTOR_VERSION "3.2.2")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.2.1
+VITASTOR_VERSION ?= v3.2.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.2.1
+          image: vitalif/vitastor-csi:v3.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -87,7 +87,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-attacher
-          image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.2
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.2.1
+          image: vitalif/vitastor-csi:v3.2.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.2.1"
+    vitastorCSIDriverVersion = "3.2.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.2.1-1) unstable; urgency=medium
+vitastor (3.2.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.2.1
+VITASTOR_VERSION ?= v3.2.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.2.1
+VITASTOR_VERSION=v3.2.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.2.1`
+   `docker pull vitalif/vitastor:v3.2.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.1 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 4. Enable the vitastor-host service: \

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.2.1`
+   `docker pull vitalif/vitastor:v3.2.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.1 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.2.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 4. Включите сервис vitastor-host: \

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.2.1'
+VITASTOR_VERSION = '3.2.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el10.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el10.tar.gz
+Source0:        vitastor-3.2.2.el10.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el7.tar.gz
+Source0:        vitastor-3.2.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el8.tar.gz
+Source0:        vitastor-3.2.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.2.1
+Version:        3.2.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.2.1.el9.tar.gz
+Source0:        vitastor-3.2.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.2.1")
+add_definitions(-DVITASTOR_VERSION="3.2.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_flush.cpp

@@ -213,18 +213,25 @@ resume_0:
     res = bs->heap->get_next_compact(cur_oid);
     if ((bs->intent_write_counter >= bs->journal_trim_interval || flusher->force_fsync > 0) && co_id == 0)
     {
-        // Advance fsynced_lsn every <journal_trim_interval> intent writes
-        bs->intent_write_counter = 0;
 resume_17:
 resume_18:
 resume_19:
         if (!fsync_buffer(17))
+        {
             return false;
+        }
+        if (bs->intent_write_counter >= bs->journal_trim_interval)
+        {
+            // Advance fsynced_lsn every <journal_trim_interval> intent writes
+            bs->intent_write_counter = 0;
 resume_29:
 resume_30:
 resume_31:
-        if (!trim_lsn(29))
-            return false;
+            if (!trim_lsn(29))
+            {
+                return false;
+            }
+        }
     }
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
         (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))

src/blockstore/blockstore_heap.cpp

@@ -2436,9 +2436,11 @@ heap_compact_t blockstore_heap_t::iterate_compaction(heap_entry_t *obj, uint64_t
             res.do_delete = false;
             res.compact_lsn = 0;
             res.compact_version = 0;
-            if (!under_pressure && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_DELETE))
+            if ((!under_pressure || stable) && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT || wr->type() == BS_HEAP_DELETE))
             {
                 // We may postpone compaction if we have an unstable overwrite when not under pressure
+                // We MUST postpone if the overwrite is STABLE (even if under pressure), 
+                // because add_compact will reject STABLE overwrites with EBUSY.
                 return res;
             }
             continue;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.2.1
+Version: 3.2.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/test/test_heap.cpp

@@ -1010,6 +1010,211 @@ void test_iterate_compaction()
         assert(small_writes == 1);
     }
 
+    {
+        blockstore_heap_t heap(&dsk, buffer_area.data());
+        heap.finish_recheck();
+
+        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_STABLE(v3 l3) un-fsynced
+        // -> skip compaction of l2 into l1 EVEN WHEN under pressure,
+        // because add_compact rejects stable overwrites with EBUSY
+        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
+        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
+
+        heap.use_data(INODE_WITH_POOL(1, 1), 128*1024);
+        uint32_t mblock = 999999;
+        res = _test_do_big_write(heap, dsk, 1, 0, 3, 128*1024, true, 0, 0, buffer_area.data(), &mblock);
+        assert(res == 0);
+        heap.start_block_write(mblock);
+
+        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
+        auto obj = heap.read_entry(oid);
+        assert(obj->lsn == 3);
+        assert(obj->is_overwrite());
+        assert(heap.get_fsynced_lsn() == 2);
+
+        // When not under pressure: postpone compaction
+        int small_writes = 0;
+        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // When under pressure: MUST ALSO postpone compaction because obj is STABLE!
+        // Before the fix, this would incorrectly compact l2 (compact_lsn == 2)
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // Verify that add_compact() rejects stable overwrites with EBUSY
+        uint32_t compact_mblock = 999999;
+        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
+        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
+        assert(res == EBUSY);
+
+        heap.complete_block_write(mblock);
+        assert(heap.get_fsynced_lsn() == 3);
+
+        // Once fsynced, l3 is durable and becomes the clean_wr base
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.clean_wr && compact_info.clean_wr->lsn == 3);
+        assert(small_writes == 0);
+    }
+
+    {
+        blockstore_heap_t heap(&dsk, buffer_area.data());
+        heap.finish_recheck();
+
+        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_INTENT(v3 l3) un-fsynced
+        // -> skip compaction of l2 into l1 both when not under pressure and under pressure
+        // Before the fix, BIG_INTENT was missing from the check and was never skipped.
+        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
+        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
+
+        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
+        auto obj = heap.read_entry(oid);
+        uint8_t ext_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ext_bitmap, 0x8e, dsk.clean_entry_bitmap_size);
+        heap.use_data(INODE_WITH_POOL(1, 1), 128*1024);
+        uint32_t mblock = 999999;
+        res = heap.add_redirect_intent(oid, &obj, 3, 0, dsk.data_block_size, 128*1024, ext_bitmap, buffer_area.data(), &mblock);
+        assert(res == 0);
+        heap.start_block_write(mblock);
+
+        obj = heap.read_entry(oid);
+        assert(obj->lsn == 3);
+        assert(obj->type() == BS_HEAP_BIG_INTENT);
+        assert(obj->is_overwrite());
+        assert(heap.get_fsynced_lsn() == 2);
+
+        // When not under pressure: MUST postpone compaction!
+        // Before the fix, BIG_INTENT was not checked, so it incorrectly compacted l2 (compact_lsn == 2)
+        int small_writes = 0;
+        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // When under pressure: MUST ALSO postpone compaction!
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // Verify add_compact() rejects stable overwrite with EBUSY
+        uint32_t compact_mblock = 999999;
+        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
+        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
+        assert(res == EBUSY);
+
+        // Complete block write
+        heap.complete_block_write(mblock);
+        assert(heap.get_fsynced_lsn() == 3);
+
+        // Once fsynced, BIG_INTENT is clean_wr base
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.clean_wr && compact_info.clean_wr->lsn == 3);
+        assert(small_writes == 0);
+    }
+
+    {
+        blockstore_heap_t heap(&dsk, buffer_area.data());
+        heap.finish_recheck();
+
+        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) DELETE_STABLE(l3) un-fsynced
+        // -> skip compaction of l2 into l1 EVEN WHEN under pressure,
+        // because add_compact rejects stable overwrites with EBUSY
+        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
+        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
+
+        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
+        auto obj = heap.read_entry(oid);
+        uint32_t mblock = 999999;
+        res = heap.add_delete(obj, &mblock);
+        assert(res == 0);
+        heap.start_block_write(mblock);
+
+        obj = heap.read_entry(oid);
+        assert(obj->lsn == 3);
+        assert(obj->type() == BS_HEAP_DELETE);
+        assert(obj->is_overwrite());
+        assert(heap.get_fsynced_lsn() == 2);
+
+        // When not under pressure: postpone compaction
+        int small_writes = 0;
+        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // When under pressure: MUST ALSO postpone compaction because DELETE is STABLE!
+        // Before the fix, this would incorrectly compact l2 (compact_lsn == 2)
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.compact_lsn == 0);
+        assert(compact_info.compact_version == 0);
+        assert(!compact_info.clean_wr);
+        assert(!compact_info.do_delete);
+        assert(small_writes == 0);
+
+        // Verify add_compact() rejects stable overwrite with EBUSY
+        uint32_t compact_mblock = 999999;
+        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
+        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
+        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
+        assert(res == EBUSY);
+
+        // Complete block write
+        heap.complete_block_write(mblock);
+        assert(heap.get_fsynced_lsn() == 3);
+
+        // Once fsynced, DELETE is processed
+        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
+        {
+            small_writes++;
+        });
+        assert(compact_info.do_delete);
+        assert(!compact_info.clean_wr);
+        assert(small_writes == 0);
+    }
+
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
         heap.finish_recheck();