Release 1.4.5

- Fix a write stall caused by incorrect journal trimming introduced in 1.4.4 :)
- Fix PGs sometimes hanging in "starting" state on mass OSD restarts
- Fix a rare crash with "map::at" during OSD pings
- Use new defaults for non-capacitor (desktop) SSDs - improves T1Q256 random write from ~6k iops to ~45k iops
- Make journal_trim_interval configurable

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "1.4.4")
+set(VERSION "1.4.5")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v1.4.4
+VERSION ?= v1.4.5
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.4.4
+          image: vitalif/vitastor-csi:v1.4.5
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.4.4
+          image: vitalif/vitastor-csi:v1.4.5
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.4.4"
+    vitastorCSIDriverVersion = "1.4.5"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.4.4-1) unstable; urgency=medium
+vitastor (1.4.5-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-1.4.4; \
-    cd vitastor-1.4.4; \
+    cp -r /root/vitastor vitastor-1.4.5; \
+    cd vitastor-1.4.5; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.4.orig.tar.xz vitastor-1.4.4; \
-    cd vitastor-1.4.4; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.5.orig.tar.xz vitastor-1.4.5; \
+    cd vitastor-1.4.5; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config/pool.en.md

@@ -154,6 +154,9 @@ That is, if it becomes impossible to place PG data on at least (pg_minsize)
 OSDs, PG is deactivated for both read and write. So you know that a fresh
 write always goes to at least (pg_minsize) OSDs (disks).
 
+That is, pg_size minus pg_minsize sets the number of disk failures to tolerate
+without temporary downtime (for [osd_out_time](monitor.en.md#osd_out_time)).
+
 FIXME: pg_minsize behaviour may be changed in the future to only make PGs
 read-only instead of deactivating them.
 

docs/config/pool.ru.md

@@ -157,6 +157,10 @@
 OSD, PG деактивируется на чтение и запись. Иными словами, всегда известно,
 что новые блоки данных всегда записываются как минимум на pg_minsize дисков.
 
+По сути, разница pg_size и pg_minsize задаёт число отказов дисков, которые пул
+может пережить без временной (на [osd_out_time](monitor.ru.md#osd_out_time))
+остановки обслуживания.
+
 FIXME: Поведение pg_minsize может быть изменено в будущем с полной деактивации
 PG на перевод их в режим только для чтения.
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '1.4.4'
+VERSION = '1.4.5'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-1.4.4/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.4$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-1.4.5/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.5$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -36,7 +36,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.4.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.5.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.4
+Version:        1.4.5
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.4.el7.tar.gz
+Source0:        vitastor-1.4.5.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.4.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.5.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.4
+Version:        1.4.5
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.4.el8.tar.gz
+Source0:        vitastor-1.4.5.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.4.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.5.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.4
+Version:        1.4.5
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.4.el9.tar.gz
+Source0:        vitastor-1.4.5.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="1.4.4")
+add_definitions(-DVERSION="1.4.5")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore_flush.cpp

@@ -19,7 +19,6 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
     syncing_flushers = 0;
     // FIXME: allow to configure flusher_start_threshold and journal_trim_interval
     flusher_start_threshold = bs->dsk.journal_block_size / sizeof(journal_entry_stable);
-    journal_trim_interval = 512;
     journal_trim_counter = bs->journal.flush_journal ? 1 : 0;
     trim_wanted = bs->journal.flush_journal ? 1 : 0;
     journal_superblock = bs->journal.inmemory ? bs->journal.buffer : memalign_or_die(MEM_ALIGNMENT, bs->dsk.journal_block_size);
@@ -365,9 +364,10 @@ resume_0:
         !flusher->flush_queue.size() || !flusher->dequeuing)
     {
 stop_flusher:
-        if (flusher->trim_wanted > 0 && !flusher->journal_trim_counter)
+        if (flusher->trim_wanted > 0 && cur.oid.inode != 0)
         {
             // Attempt forced trim
+            cur.oid = {};
             flusher->active_flushers++;
             goto trim_journal;
         }
@@ -583,7 +583,8 @@ resume_2:
         flusher->sync_to_repeat.erase(repeat_it);
     trim_journal:
         // Clear unused part of the journal every <journal_trim_interval> flushes
-        if (!((++flusher->journal_trim_counter) % flusher->journal_trim_interval) || flusher->trim_wanted > 0)
+        if (bs->journal_trim_interval && !((++flusher->journal_trim_counter) % bs->journal_trim_interval) ||
+            flusher->trim_wanted > 0)
         {
     resume_26:
     resume_27:

src/blockstore_flush.h

@@ -107,7 +107,7 @@ class journal_flusher_t
     blockstore_impl_t *bs;
     friend class journal_flusher_co;
 
-    int journal_trim_counter, journal_trim_interval;
+    int journal_trim_counter;
     bool trimming;
     void* journal_superblock;
 

src/blockstore_impl.h

@@ -253,6 +253,7 @@ class blockstore_impl_t
     bool inmemory_meta = false;
     // Maximum and minimum flusher count
     unsigned max_flusher_count, min_flusher_count;
+    unsigned journal_trim_interval;
     // Maximum queue depth
     unsigned max_write_iodepth = 128;
     // Enable small (journaled) write throttling, useful for the SSD+HDD case

src/blockstore_open.cpp

@@ -13,6 +13,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
         max_flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
     }
     min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10);
+    journal_trim_interval = strtoull(config["journal_trim_interval"].c_str(), NULL, 10);
     max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
     throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes";
     throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10);
@@ -31,6 +32,10 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     {
         min_flusher_count = 1;
     }
+    if (!journal_trim_interval)
+    {
+        journal_trim_interval = 512;
+    }
     if (!max_write_iodepth)
     {
         max_write_iodepth = 128;

src/disk_tool_prepare.cpp

@@ -8,6 +8,7 @@
 int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd)
 {
     static const char *allow_additional_params[] = {
+        "autosync_writes",
         "data_io",
         "meta_io",
         "journal_io",
@@ -99,12 +100,9 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         options["disable_journal_fsync"] = options["disable_data_fsync"];
     }
     // Calculate offsets if the same device is used for two or more of data, meta, and journal
-    if (options["journal_size"] == "")
+    if (options["journal_size"] == "" && (options["journal_device"] == "" || options["journal_device"] == options["data_device"]))
     {
-        if (options["journal_device"] == "")
-            options["journal_size"] = is_hdd ? "128M" : "32M";
-        else if (is_hdd)
-            options["journal_size"] = DEFAULT_HYBRID_JOURNAL;
+        options["journal_size"] = is_hdd || !json_is_true(options["disable_data_fsync"]) ? "128M" : "32M";
     }
     bool is_hybrid = is_hdd && options["journal_device"] != "" && options["journal_device"] != options["data_device"];
     if (is_hdd)
@@ -114,6 +112,15 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         if (is_hybrid && options["throttle_small_writes"] == "")
             options["throttle_small_writes"] = "1";
     }
+    else if (!json_is_true(options["disable_data_fsync"]))
+    {
+        if (options.find("min_flusher_count") == options.end())
+            options["min_flusher_count"] = "32";
+        if (options.find("max_flusher_count") == options.end())
+            options["max_flusher_count"] = "256";
+        if (options.find("autosync_writes") == options.end())
+            options["autosync_writes"] = "512";
+    }
     json11::Json::object sb;
     blockstore_disk_t dsk;
     try
@@ -616,6 +623,7 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
         options.erase("disable_meta_fsync");
         options.erase("disable_journal_fsync");
     }
+    auto journal_size = options["journal_size"];
     for (auto & dev: devinfo)
     {
         if (!hybrid || dev.is_hdd)
@@ -633,11 +641,13 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
                     {
                         return 1;
                     }
+                    options.erase("journal_size");
                 }
                 // Treat all disks as SSDs if not in the hybrid mode
                 prepare_one(options, dev.is_hdd ? 1 : 0);
                 if (hybrid)
                 {
+                    options["journal_size"] = journal_size;
                     options.erase("journal_device");
                     options.erase("meta_device");
                 }

src/messenger.cpp

@@ -45,11 +45,12 @@ void osd_messenger_t::init()
 #endif
     keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
     {
-        std::vector<int> to_stop;
-        std::vector<osd_op_t*> to_ping;
-        for (auto cl_it = clients.begin(); cl_it != clients.end(); cl_it++)
+        auto cl_it = clients.begin();
+        while (cl_it != clients.end())
         {
             auto cl = cl_it->second;
+            cl_it++;
+            auto peer_fd = cl->peer_fd;
             if (!cl->osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
             {
                 // Do not run keepalive on regular clients
@@ -62,7 +63,9 @@ void osd_messenger_t::init()
                 {
                     // Ping timed out, stop the client
                     fprintf(stderr, "Ping timed out for OSD %lu (client %d), disconnecting peer\n", cl->osd_num, cl->peer_fd);
-                    to_stop.push_back(cl->peer_fd);
+                    stop_client(peer_fd, true);
+                    // Restart iterator because it may be invalidated
+                    cl_it = clients.upper_bound(peer_fd);
                 }
             }
             else if (cl->idle_time_remaining > 0)
@@ -100,9 +103,11 @@ void osd_messenger_t::init()
                             stop_client(fail_fd, true);
                         }
                     };
-                    to_ping.push_back(op);
                     cl->ping_time_remaining = osd_ping_timeout;
                     cl->idle_time_remaining = osd_idle_timeout;
+                    outbox_push(op);
+                    // Restart iterator because it may be invalidated
+                    cl_it = clients.upper_bound(peer_fd);
                 }
             }
             else
@@ -110,15 +115,6 @@ void osd_messenger_t::init()
                 cl->idle_time_remaining = osd_idle_timeout;
             }
         }
-        // Don't stop clients while a 'clients' iterator is still active
-        for (int peer_fd: to_stop)
-        {
-            stop_client(peer_fd, true);
-        }
-        for (auto op: to_ping)
-        {
-            outbox_push(op);
-        }
     });
 }
 

src/osd_cluster.cpp

@@ -843,7 +843,13 @@ void osd_t::report_pg_states()
                     pg_state_exists = true;
                     if (pg.state == PG_OFFLINE && pg_it->second.cur_primary != this->osd_num)
                     {
-                        // Nothing to check or report, PG is already taken over by another OSD
+                        // Nothing to report, PG is already taken over by another OSD
+                        checks.push_back(json11::Json::object {
+                            { "target", "MOD" },
+                            { "key", state_key_base64 },
+                            { "result", "LESS" },
+                            { "mod_revision", st_cli.etcd_watch_revision+1 },
+                        });
                         continue;
                     }
                 }
@@ -851,11 +857,6 @@ void osd_t::report_pg_states()
         }
         if (!pg_state_exists)
         {
-            if (pg.state == PG_OFFLINE)
-            {
-                // Nothing to check or report, PG is already stopped
-                continue;
-            }
             // Check that the PG key does not exist
             // Failed check indicates an unsuccessful PG lock attempt in this case
             checks.push_back(json11::Json::object {

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.4.4
+Version: 1.4.5
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}