Release 0.8.3

- Implement a new "vitastor-disk purge" command to remove OSDs with safety checks
- Implement a new "vitastor-cli rm-osd" command to only remove OSD metadata from etcd
- Fix a bug where the monitor could ignore OSD removal and other /osd/stats key changes
- Fix a bug where garbage could be returned when reading objects being written at the same time
- Fix a rare write stall where journal space could be not reclaimed where there
  were no new operations in the flush queue
- Fix a rare peering stall caused by a previous long listing operations queues limiting attempt
- Fix total object count statistic in OSD on object creation
- Add missing offset&len into vitastor-disk dump-journal for big_writes, fix JSON format
- Make vitastor-cli print help on missing command
- Make vitastor-cli translate all '-' to '_' in CLI options

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.8.2")
+set(VERSION "0.8.3")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.8.2
+VERSION ?= v0.8.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.8.2
+          image: vitalif/vitastor-csi:v0.8.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.8.2
+          image: vitalif/vitastor-csi:v0.8.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.8.2"
+    vitastorCSIDriverVersion = "0.8.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.8.2-1) unstable; urgency=medium
+vitastor (0.8.3-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.8.2-1) unstable; urgency=medium
+vitastor (0.8.3-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/vitastor.Dockerfile

@@ -34,8 +34,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.8.2; \
-    cd vitastor-0.8.2; \
+    cp -r /root/vitastor vitastor-0.8.3; \
+    cd vitastor-0.8.3; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -48,8 +48,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.8.2.orig.tar.xz vitastor-0.8.2; \
-    cd vitastor-0.8.2; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.8.3.orig.tar.xz vitastor-0.8.3; \
+    cd vitastor-0.8.3; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/usage/cli.en.md

@@ -20,6 +20,7 @@ It supports the following commands:
 - [rm-data](#rm-data)
 - [merge-data](#merge-data)
 - [alloc-osd](#alloc-osd)
+- [rm-osd](#rm-osd)
 
 Global options:
 
@@ -175,3 +176,14 @@ Merge layer data without changing metadata. Merge `<from>`..`<to>` to `<target>`
 `vitastor-cli alloc-osd`
 
 Allocate a new OSD number and reserve it by creating empty `/osd/stats/<n>` key.
+
+## rm-osd
+
+`vitastor-cli rm-osd [--force] [--allow-data-loss] [--dry-run] <osd_id> [osd_id...]`
+
+Remove metadata and configuration for specified OSD(s) from etcd.
+
+Refuses to remove OSDs with data without `--force` and `--allow-data-loss`.
+
+With `--dry-run` only checks if deletion is possible without data loss and
+redundancy degradation.

docs/usage/cli.ru.md

@@ -21,6 +21,7 @@ vitastor-cli - интерфейс командной строки для адм
 - [rm-data](#rm-data)
 - [merge-data](#merge-data)
 - [alloc-osd](#alloc-osd)
+- [rm-osd](#rm-osd)
 
 Глобальные опции:
 
@@ -186,3 +187,14 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 
 Атомарно выделить новый номер OSD и зарезервировать его, создав в etcd пустой
 ключ `/osd/stats/<n>`.
+
+## rm-osd
+
+`vitastor-cli rm-osd [--force] [--allow-data-loss] [--dry-run] <osd_id> [osd_id...]`
+
+Удалить метаданные и конфигурацию для заданных OSD из etcd.
+
+Отказывается удалять OSD с данными без опций `--force` и `--allow-data-loss`.
+
+С опцией `--dry-run` только проверяет, возможно ли удаление без потери данных и деградации
+избыточности.

docs/usage/disk.en.md

@@ -14,6 +14,7 @@ It supports the following commands:
 - [upgrade-simple](#upgrade-simple)
 - [resize](#resize)
 - [start/stop/restart/enable/disable](#start/stop/restart/enable/disable)
+- [purge](#purge)
 - [read-sb](#read-sb)
 - [write-sb](#write-sb)
 - [udev](#udev)
@@ -155,11 +156,22 @@ Commands are passed to `systemctl` with `vitastor-osd@<num>` units as arguments.
 
 When `--now` is added to enable/disable, OSDs are also immediately started/stopped.
 
+## purge
+
+`vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]`
+
+Purge Vitastor OSD(s) on specified device(s). Uses `vitastor-cli rm-osd` to check
+if deletion is possible without data loss and to actually remove metadata from etcd.
+`--force` and `--allow-data-loss` options may be used to ignore safety check results.
+
+Requires `vitastor-cli`, `sfdisk` and `partprobe` (from parted) utilities.
+
 ## read-sb
 
-`vitastor-disk read-sb <device>`
+`vitastor-disk read-sb [--force] <device>`
 
 Try to read Vitastor OSD superblock from `<device>` and print it in JSON format.
+`--force` allows to ignore validation errors.
 
 ## write-sb
 

docs/usage/disk.ru.md

@@ -14,6 +14,7 @@ vitastor-disk - инструмент командной строки для уп
 - [upgrade-simple](#upgrade-simple)
 - [resize](#resize)
 - [start/stop/restart/enable/disable](#start/stop/restart/enable/disable)
+- [purge](#purge)
 - [read-sb](#read-sb)
 - [write-sb](#write-sb)
 - [udev](#udev)
@@ -158,12 +159,25 @@ throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.
 Когда к командам включения/выключения добавляется параметр `--now`, OSD также сразу
 запускаются/останавливаются.
 
+## purge
+
+`vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]`
+
+Удалить OSD на заданном диске/дисках. Использует `vitastor-cli rm-osd` для проверки
+возможности удаления без потери данных и для удаления OSD из etcd. Опции `--force`
+и `--allow-data-loss` служат для обхода данной защиты в случае необходимости.
+
+Команде требуются утилиты `vitastor-cli`, `sfdisk` и `partprobe` (из состава parted).
+
 ## read-sb
 
-`vitastor-disk read-sb <device>`
+`vitastor-disk read-sb [--force] <device>`
 
 Прочитать суперблок OSD с диска `<device>` и вывести его в формате JSON.
 
+Опция `--force` позволяет читать суперблок, даже если он считается некорректным
+из-за ошибок валидации.
+
 ## write-sb
 
 `vitastor-disk write-sb <device>`

mon/mon.js

@@ -1693,6 +1693,7 @@ class Mon
             // Do not clear these to null
             kv.value = kv.value || {};
         }
+        const old = cur[key_parts[key_parts.length-1]];
         cur[key_parts[key_parts.length-1]] = kv.value;
         if (key === 'config/global')
         {
@@ -1717,6 +1718,11 @@ class Mon
         }
         else if (key_parts[0] === 'osd' && key_parts[1] === 'stats')
         {
+            // Recheck OSD tree on OSD addition/deletion
+            if ((!old) != (!kv.value) || old && kv.value && (old.size != kv.value.size || old.time != kv.value.time))
+            {
+                this.schedule_recheck();
+            }
             // Recheck PGs <osd_out_time> later
             this.schedule_next_recheck_at(
                 !this.state.osd.stats[key[2]] ? 0 : this.state.osd.stats[key[2]].time+this.config.osd_out_time

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.8.2'
+VERSION = '0.8.3'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.8.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.8.2$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.8.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.8.3$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.8.2.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.8.3.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.8.2
+Version:        0.8.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.8.2.el7.tar.gz
+Source0:        vitastor-0.8.3.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.8.2.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.8.3.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.8.2
+Version:        0.8.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.8.2.el8.tar.gz
+Source0:        vitastor-0.8.3.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.8.2")
+add_definitions(-DVERSION="0.8.3")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -140,6 +140,7 @@ add_library(vitastor_client SHARED
 	cli_merge.cpp
 	cli_rm_data.cpp
 	cli_rm.cpp
+	cli_rm_osd.cpp
 )
 set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h")
 target_link_libraries(vitastor_client

src/blockstore_flush.cpp

@@ -77,7 +77,7 @@ void journal_flusher_t::loop()
             cur_flusher_count--;
         }
     }
-    for (int i = 0; (active_flushers > 0 || dequeuing) && i < cur_flusher_count; i++)
+    for (int i = 0; (active_flushers > 0 || dequeuing || trim_wanted > 0) && i < cur_flusher_count; i++)
         co[i].loop();
 }
 

src/blockstore_impl.cpp

@@ -107,7 +107,7 @@ void blockstore_impl_t::loop()
         // has_writes == 0 - no writes before the current queue item
         // has_writes == 1 - some writes in progress
         // has_writes == 2 - tried to submit some writes, but failed
-        int has_writes = 0, op_idx = 0, new_idx = 0, done_lists = 0;
+        int has_writes = 0, op_idx = 0, new_idx = 0;
         for (; op_idx < submit_queue.size(); op_idx++, new_idx++)
         {
             auto op = submit_queue[op_idx];
@@ -188,13 +188,8 @@ void blockstore_impl_t::loop()
             else if (op->opcode == BS_OP_LIST)
             {
                 // LIST doesn't have to be blocked by previous modifications
-                // But don't do a lot of LISTs at once, because they're blocking and potentially slow
-                if (single_tick_list_limit <= 0 || done_lists < single_tick_list_limit)
-                {
-                    process_list(op);
-                    done_lists++;
-                    wr_st = 2;
-                }
+                process_list(op);
+                wr_st = 2;
             }
             if (wr_st == 2)
             {

src/blockstore_impl.h

@@ -240,8 +240,6 @@ class blockstore_impl_t
     int throttle_target_parallelism = 1;
     // Minimum difference in microseconds between target and real execution times to throttle the response
     int throttle_threshold_us = 50;
-    // Maximum number of LIST operations to be processed between
-    int single_tick_list_limit = 1;
     /******* END OF OPTIONS *******/
 
     struct ring_consumer_t ring_consumer;

src/blockstore_read.cpp

@@ -139,7 +139,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         while (dirty_it->first.oid == read_op->oid)
         {
             dirty_entry& dirty = dirty_it->second;
-            bool version_ok = read_op->version >= dirty_it->first.version;
+            bool version_ok = !IS_IN_FLIGHT(dirty.state) && read_op->version >= dirty_it->first.version;
             if (IS_SYNCED(dirty.state))
             {
                 if (!version_ok && read_op->version != 0)
@@ -174,7 +174,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
             dirty_it--;
         }
     }
-    if (clean_it != clean_db.end())
+    if (clean_found)
     {
         if (!result_version)
         {

src/blockstore_write.cpp

@@ -139,7 +139,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             uint8_t *bmp_ptr = (uint8_t*)(dsk.clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp);
             uint32_t bit = op->offset/dsk.bitmap_granularity;
             uint32_t bits_left = op->len/dsk.bitmap_granularity;
-            while (!(bit % 8) && bits_left > 8)
+            while (!(bit % 8) && bits_left >= 8)
             {
                 // Copy bytes
                 bmp_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];

src/cli.cpp

@@ -76,6 +76,12 @@ static const char* help_text =
     "vitastor-cli alloc-osd\n"
     "  Allocate a new OSD number and reserve it by creating empty /osd/stats/<n> key.\n"
     "\n"
+    "vitastor-cli rm-osd [--force] [--allow-data-loss] [--dry-run] <osd_id> [osd_id...]\n"
+    "  Remove metadata and configuration for specified OSD(s) from etcd.\n"
+    "  Refuses to remove OSDs with data without --force and --allow-data-loss.\n"
+    "  With --dry-run only checks if deletion is possible without data loss and\n"
+    "  redundancy degradation.\n"
+    "\n"
     "Use vitastor-cli --help <command> for command details or vitastor-cli --help --all for all details.\n"
     "\n"
     "GLOBAL OPTIONS:\n"
@@ -95,43 +101,47 @@ static json11::Json::object parse_args(int narg, const char *args[])
     cfg["progress"] = "1";
     for (int i = 1; i < narg; i++)
     {
-        if (args[i][0] == '-' && args[i][1] == 'h')
+        if (args[i][0] == '-' && args[i][1] == 'h' && args[i][2] == 0)
         {
             cfg["help"] = "1";
         }
-        else if (args[i][0] == '-' && args[i][1] == 'l')
+        else if (args[i][0] == '-' && args[i][1] == 'l' && args[i][2] == 0)
         {
             cfg["long"] = "1";
         }
-        else if (args[i][0] == '-' && args[i][1] == 'n')
+        else if (args[i][0] == '-' && args[i][1] == 'n' && args[i][2] == 0)
         {
             cfg["count"] = args[++i];
         }
-        else if (args[i][0] == '-' && args[i][1] == 'p')
+        else if (args[i][0] == '-' && args[i][1] == 'p' && args[i][2] == 0)
         {
             cfg["pool"] = args[++i];
         }
-        else if (args[i][0] == '-' && args[i][1] == 's')
+        else if (args[i][0] == '-' && args[i][1] == 's' && args[i][2] == 0)
         {
             cfg["size"] = args[++i];
         }
-        else if (args[i][0] == '-' && args[i][1] == 'r')
+        else if (args[i][0] == '-' && args[i][1] == 'r' && args[i][2] == 0)
         {
             cfg["reverse"] = "1";
         }
-        else if (args[i][0] == '-' && args[i][1] == 'f')
+        else if (args[i][0] == '-' && args[i][1] == 'f' && args[i][2] == 0)
         {
             cfg["force"] = "1";
         }
         else if (args[i][0] == '-' && args[i][1] == '-')
         {
             const char *opt = args[i]+2;
-            cfg[opt] = i == narg-1 || !strcmp(opt, "json") || !strcmp(opt, "wait-list") ||
-                !strcmp(opt, "long") || !strcmp(opt, "del") || !strcmp(opt, "no-color") ||
+            cfg[opt] = i == narg-1 || !strcmp(opt, "json") ||
+                !strcmp(opt, "wait-list") || !strcmp(opt, "wait_list") ||
+                !strcmp(opt, "long") || !strcmp(opt, "del") ||
+                !strcmp(opt, "no-color") || !strcmp(opt, "no_color") ||
                 !strcmp(opt, "readonly") || !strcmp(opt, "readwrite") ||
                 !strcmp(opt, "force") || !strcmp(opt, "reverse") ||
+                !strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") ||
+                !strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
                 !strcmp(opt, "help") || !strcmp(opt, "all") ||
-                !strcmp(opt, "writers-stopped") && strcmp("1", args[i+1]) != 0
+                (!strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped")) && strcmp("1", args[i+1]) != 0
                 ? "1" : args[++i];
         }
         else
@@ -139,10 +149,6 @@ static json11::Json::object parse_args(int narg, const char *args[])
             cmd.push_back(std::string(args[i]));
         }
     }
-    if (cfg["help"].bool_value())
-    {
-        print_help(help_text, "vitastor-cli", cmd.size() ? cmd[0].string_value() : "", cfg["all"].bool_value());
-    }
     if (!cmd.size())
     {
         std::string exe(exe_name);
@@ -151,6 +157,10 @@ static json11::Json::object parse_args(int narg, const char *args[])
             cmd.push_back("rm-data");
         }
     }
+    if (!cmd.size() || cfg["help"].bool_value())
+    {
+        print_help(help_text, "vitastor-cli", cmd.size() ? cmd[0].string_value() : "", cfg["all"].bool_value());
+    }
     cfg["command"] = cmd;
     return cfg;
 }
@@ -225,6 +235,16 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
         // Delete inode data
         action_cb = p->start_rm_data(cfg);
     }
+    else if (cmd[0] == "rm-osd")
+    {
+        // Delete OSD metadata from etcd
+        if (cmd.size() > 1)
+        {
+            cmd.erase(cmd.begin(), cmd.begin()+1);
+            cfg["osd_id"] = cmd;
+        }
+        action_cb = p->start_rm_osd(cfg);
+    }
     else if (cmd[0] == "merge-data")
     {
         // Merge layer data without affecting metadata

src/cli.h

@@ -45,7 +45,7 @@ public:
     cli_result_t etcd_err;
     json11::Json etcd_result;
 
-    void parse_config(json11::Json cfg);
+    void parse_config(json11::Json::object & cfg);
 
     void change_parent(inode_t cur, inode_t new_parent, cli_result_t *result);
     inode_config_t* get_inode_cfg(const std::string & name);
@@ -64,6 +64,7 @@ public:
     std::function<bool(cli_result_t &)> start_merge(json11::Json);
     std::function<bool(cli_result_t &)> start_flatten(json11::Json);
     std::function<bool(cli_result_t &)> start_rm(json11::Json);
+    std::function<bool(cli_result_t &)> start_rm_osd(json11::Json cfg);
     std::function<bool(cli_result_t &)> start_alloc_osd(json11::Json cfg);
 
     // Should be called like loop_and_wait(start_status(), <completion callback>)

src/cli_common.cpp

@@ -100,9 +100,20 @@ inode_config_t* cli_tool_t::get_inode_cfg(const std::string & name)
     return NULL;
 }
 
-void cli_tool_t::parse_config(json11::Json cfg)
+void cli_tool_t::parse_config(json11::Json::object & cfg)
 {
-    color = !cfg["no-color"].bool_value();
+    for (auto kv_it = cfg.begin(); kv_it != cfg.end();)
+    {
+        // Translate all options with - to _
+        if (kv_it->first.find("-") != std::string::npos)
+        {
+            cfg[str_replace(kv_it->first, "-", "_")] = kv_it->second;
+            cfg.erase(kv_it++);
+        }
+        else
+            kv_it++;
+    }
+    color = !cfg["no_color"].bool_value();
     json_output = cfg["json"].bool_value();
     iodepth = cfg["iodepth"].uint64_value();
     if (!iodepth)
@@ -112,7 +123,7 @@ void cli_tool_t::parse_config(json11::Json cfg)
         parallel_osds = 4;
     log_level = cfg["log_level"].int64_value();
     progress = cfg["progress"].uint64_value() ? true : false;
-    list_first = cfg["wait-list"].uint64_value() ? true : false;
+    list_first = cfg["wait_list"].uint64_value() ? true : false;
 }
 
 struct cli_result_looper_t

src/cli_create.cpp

@@ -517,7 +517,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
     image_creator->force_size = cfg["force_size"].bool_value();
     if (cfg["image_meta"].is_object())
     {
-        image_creator->new_meta = cfg["image-meta"];
+        image_creator->new_meta = cfg["image_meta"];
     }
     if (cfg["snapshot"].string_value() != "")
     {

src/cli_flatten.cpp

@@ -133,7 +133,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_flatten(json11::Json cfg)
     auto flattener = new snap_flattener_t();
     flattener->parent = this;
     flattener->target_name = cfg["image"].string_value();
-    flattener->fsync_interval = cfg["fsync-interval"].uint64_value();
+    flattener->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!flattener->fsync_interval)
         flattener->fsync_interval = 128;
     if (!cfg["cas"].is_null())

src/cli_merge.cpp

@@ -631,8 +631,8 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_merge(json11::Json cfg)
     merger->from_name = cfg["from"].string_value();
     merger->to_name = cfg["to"].string_value();
     merger->target_name = cfg["target"].string_value();
-    merger->delete_source = cfg["delete-source"].string_value() != "";
-    merger->fsync_interval = cfg["fsync-interval"].uint64_value();
+    merger->delete_source = cfg["delete_source"].string_value() != "";
+    merger->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!merger->fsync_interval)
         merger->fsync_interval = 128;
     if (!cfg["cas"].is_null())

src/cli_modify.cpp

@@ -236,7 +236,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_modify(json11::Json cfg)
     changer->force = cfg["force"].bool_value();
     changer->set_readonly = cfg["readonly"].bool_value();
     changer->set_readwrite = cfg["readwrite"].bool_value();
-    changer->fsync_interval = cfg["fsync-interval"].uint64_value();
+    changer->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!changer->fsync_interval)
         changer->fsync_interval = 128;
     // FIXME Check that the image doesn't have children when shrinking

src/cli_rm.cpp

@@ -639,7 +639,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm(json11::Json cfg)
     snap_remover->parent = this;
     snap_remover->from_name = cfg["from"].string_value();
     snap_remover->to_name = cfg["to"].string_value();
-    snap_remover->fsync_interval = cfg["fsync-interval"].uint64_value();
+    snap_remover->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!snap_remover->fsync_interval)
         snap_remover->fsync_interval = 128;
     if (!cfg["cas"].is_null())

src/cli_rm_data.cpp

@@ -218,7 +218,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm_data(json11::Json cfg)
         remover->inode = (remover->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (((uint64_t)remover->pool_id) << (64-POOL_ID_BITS));
     }
     remover->pool_id = INODE_POOL(remover->inode);
-    remover->min_offset = cfg["min-offset"].uint64_value();
+    remover->min_offset = cfg["min_offset"].uint64_value();
     return [remover](cli_result_t & result)
     {
         remover->loop();

src/cli_rm_osd.cpp

@@ -0,0 +1,230 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include <ctype.h>
+#include "cli.h"
+#include "cluster_client.h"
+#include "str_util.h"
+
+#include <algorithm>
+
+// Delete OSD metadata from etcd
+struct rm_osd_t
+{
+    cli_tool_t *parent;
+
+    bool dry_run, force_warning, force_dataloss;
+    std::vector<uint64_t> osd_ids;
+
+    int state = 0;
+    cli_result_t result;
+
+    std::set<uint64_t> to_remove;
+    json11::Json::array pool_effects;
+    bool is_warning, is_dataloss;
+
+    bool is_done()
+    {
+        return state == 100;
+    }
+
+    void loop()
+    {
+        if (state == 1)
+            goto resume_1;
+        if (!osd_ids.size())
+        {
+            result = (cli_result_t){ .err = EINVAL, .text = "OSD numbers are not specified" };
+            state = 100;
+            return;
+        }
+        for (auto osd_id: osd_ids)
+        {
+            if (!osd_id)
+            {
+                result = (cli_result_t){ .err = EINVAL, .text = "OSD number can't be zero" };
+                state = 100;
+                return;
+            }
+            to_remove.insert(osd_id);
+        }
+        // Check if OSDs are still used in data distribution
+        is_warning = is_dataloss = false;
+        for (auto & pp: parent->cli->st_cli.pool_config)
+        {
+            // Will OSD deletion make pool incomplete / down / degraded?
+            bool pool_incomplete = false, pool_down = false, pool_degraded = false;
+            bool hist_incomplete = false, hist_degraded = false;
+            auto & pool_cfg = pp.second;
+            uint64_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
+            for (auto & pgp: pool_cfg.pg_config)
+            {
+                auto & pg_cfg = pgp.second;
+                int pg_cursize = 0, pg_rm = 0;
+                for (auto pg_osd: pg_cfg.target_set)
+                {
+                    if (pg_osd != 0)
+                    {
+                        pg_cursize++;
+                        if (to_remove.find(pg_osd) != to_remove.end())
+                            pg_rm++;
+                    }
+                }
+                for (auto & hist_item: pg_cfg.target_history)
+                {
+                    int hist_size = 0, hist_rm = 0;
+                    for (auto & old_osd: hist_item)
+                    {
+                        if (old_osd != 0)
+                        {
+                            hist_size++;
+                            if (to_remove.find(old_osd) != to_remove.end())
+                                hist_rm++;
+                        }
+                    }
+                    if (hist_rm > 0)
+                    {
+                        hist_degraded = true;
+                        if (hist_size-hist_rm == 0)
+                            pool_incomplete = true;
+                        else if (hist_size-hist_rm < pg_data_size)
+                            hist_incomplete = true;
+                    }
+                }
+                if (pg_rm > 0)
+                {
+                    pool_degraded = true;
+                    if (pg_cursize-pg_rm < pg_data_size)
+                        pool_incomplete = true;
+                    else if (pg_cursize-pg_rm < pool_cfg.pg_minsize)
+                        pool_down = true;
+                }
+            }
+            if (pool_incomplete || pool_down || pool_degraded || hist_incomplete || hist_degraded)
+            {
+                pool_effects.push_back(json11::Json::object {
+                    { "pool_id", (uint64_t)pool_cfg.id },
+                    { "pool_name", pool_cfg.name },
+                    { "effect", (pool_incomplete
+                        ? "incomplete"
+                        : (hist_incomplete
+                            ? "has_incomplete"
+                            : (pool_down
+                                ? "offline"
+                                : (pool_degraded
+                                    ? "degraded"
+                                    : (hist_degraded ? "has_degraded" : "?")
+                                )
+                            )
+                        )
+                    ) },
+                });
+                is_warning = true;
+                if (pool_incomplete || hist_incomplete)
+                    is_dataloss = true;
+            }
+        }
+        result.data = json11::Json::object {
+            { "osd_ids", osd_ids },
+            { "pool_errors", pool_effects },
+        };
+        if (is_dataloss || is_warning || dry_run)
+        {
+            std::string error;
+            for (auto & e: pool_effects)
+            {
+                error += "Pool "+e["pool_name"].string_value()+" (ID "+e["pool_id"].as_string()+") will have "+(
+                    e["effect"] == "has_incomplete"
+                        ? std::string("INCOMPLETE objects (DATA LOSS)")
+                        : (e["effect"] == "incomplete"
+                            ? std::string("INCOMPLETE PGs (DATA LOSS)")
+                            : (e["effect"] == "has_degraded"
+                                ? std::string("DEGRADED objects")
+                                : strtoupper(e["effect"].string_value())+" PGs"))
+                )+" after deleting OSD(s).\n";
+            }
+            if (is_dataloss && !force_dataloss && !dry_run)
+                error += "OSDs not deleted. Please move data to other OSDs or bypass this check with --allow-data-loss if you know what you are doing.\n";
+            else if (is_warning && !force_warning && !dry_run)
+                error += "OSDs not deleted. Please move data to other OSDs or bypass this check with --force if you know what you are doing.\n";
+            else if (!is_dataloss && !is_warning && dry_run)
+                error += "OSDs can be deleted without data loss.\n";
+            result.text = error;
+            if (dry_run || is_dataloss && !force_dataloss || is_warning && !force_warning)
+            {
+                result.err = is_dataloss || is_warning ? EBUSY : 0;
+                state = 100;
+                return;
+            }
+        }
+        // Remove keys from etcd
+        {
+            json11::Json::array rm_items;
+            for (auto osd_id: osd_ids)
+            {
+                rm_items.push_back("/config/osd/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/stats/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/state/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/inodestats/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/space/"+std::to_string(osd_id));
+            }
+            for (int i = 0; i < rm_items.size(); i++)
+            {
+                rm_items[i] = json11::Json::object {
+                    { "request_delete_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+rm_items[i].string_value()
+                        ) },
+                    } },
+                };
+            }
+            parent->etcd_txn(json11::Json::object { { "success", rm_items } });
+        }
+    resume_1:
+        state = 1;
+        if (parent->waiting > 0)
+            return;
+        if (parent->etcd_err.err)
+        {
+            result = parent->etcd_err;
+            state = 100;
+            return;
+        }
+        std::string ids = "";
+        for (auto osd_id: osd_ids)
+        {
+            ids += (ids.size() ? ", " : "")+std::to_string(osd_id);
+        }
+        ids = (osd_ids.size() > 1 ? "OSDs " : "OSD ")+ids+(osd_ids.size() > 1 ? " are" : " is")+" removed from etcd";
+        state = 100;
+        result.text = (result.text != "" ? ids+"\n"+result.text : ids);
+        result.err = 0;
+    }
+};
+
+std::function<bool(cli_result_t &)> cli_tool_t::start_rm_osd(json11::Json cfg)
+{
+    auto rm_osd = new rm_osd_t();
+    rm_osd->parent = this;
+    rm_osd->dry_run = cfg["dry_run"].bool_value();
+    rm_osd->force_dataloss = cfg["allow_data_loss"].bool_value();
+    rm_osd->force_warning = rm_osd->force_dataloss || cfg["force"].bool_value();
+    if (cfg["osd_id"].is_number() || cfg["osd_id"].is_string())
+        rm_osd->osd_ids.push_back(cfg["osd_id"].uint64_value());
+    else
+    {
+        for (auto & id: cfg["osd_id"].array_items())
+            rm_osd->osd_ids.push_back(id.uint64_value());
+    }
+    return [rm_osd](cli_result_t & result)
+    {
+        rm_osd->loop();
+        if (rm_osd->is_done())
+        {
+            result = rm_osd->result;
+            delete rm_osd;
+            return true;
+        }
+        return false;
+    };
+}

src/disk_tool.cpp

@@ -110,9 +110,16 @@ static const char *help_text =
     "  Commands are passed to systemctl with vitastor-osd@<num> units as arguments.\n"
     "  When --now is added to enable/disable, OSDs are also immediately started/stopped.\n"
     "\n"
+    "vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]\n"
+    "  Purge Vitastor OSD(s) on specified device(s). Uses vitastor-cli rm-osd to check\n"
+    "  if deletion is possible without data loss and to actually remove metadata from etcd.\n"
+    "  --force and --allow-data-loss options may be used to ignore safety check results.\n"
+    "  \n"
+    "  Requires `vitastor-cli`, `sfdisk` and `partprobe` (from parted) utilities.\n"
+    "\n"
     "vitastor-disk read-sb [--force] <device>\n"
     "  Try to read Vitastor OSD superblock from <device> and print it in JSON format.\n"
-    "  --force allows to bypass \"does not refer to the device itself\" errors.\n"
+    "  --force allows to ignore validation errors.\n"
     "\n"
     "vitastor-disk write-sb <device>\n"
     "  Read JSON from STDIN and write it into Vitastor OSD superblock on <device>.\n"
@@ -213,6 +220,10 @@ int main(int argc, char *argv[])
         {
             self.options["force"] = "1";
         }
+        else if (!strcmp(argv[i], "--allow-data-loss"))
+        {
+            self.options["allow_data_loss"] = "1";
+        }
         else if (argv[i][0] == '-' && argv[i][1] == '-')
         {
             char *key = argv[i]+2;
@@ -345,6 +356,10 @@ int main(int argc, char *argv[])
         }
         return self.systemd_start_stop_osds(systemd_cmd, std::vector<std::string>(cmd.begin()+1, cmd.end()));
     }
+    else if (!strcmp(cmd[0], "purge"))
+    {
+        return self.purge_devices(std::vector<std::string>(cmd.begin()+1, cmd.end()));
+    }
     else if (!strcmp(cmd[0], "exec-osd"))
     {
         if (cmd.size() != 2)

src/disk_tool.h

@@ -56,7 +56,7 @@ struct disk_tool_t
     uint64_t meta_pos;
     uint64_t journal_pos, journal_calc_data_pos;
 
-    bool first, first2;
+    bool first_block, first_entry;
 
     allocator *data_alloc;
     std::map<uint64_t, uint64_t> data_remap;
@@ -108,8 +108,9 @@ struct disk_tool_t
     int read_sb(std::string device);
     int write_sb(std::string device);
     int exec_osd(std::string device);
-    int systemd_start_stop_osds(std::vector<std::string> cmd, std::vector<std::string> devices);
+    int systemd_start_stop_osds(const std::vector<std::string> & cmd, const std::vector<std::string> & devices);
     int pre_exec_osd(std::string device);
+    int purge_devices(const std::vector<std::string> & devices);
 
     json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
     uint32_t write_osd_superblock(std::string device, json11::Json params);
@@ -139,3 +140,4 @@ int write_zero(int fd, uint64_t offset, uint64_t size);
 json11::Json read_parttable(std::string dev);
 uint64_t dev_size_from_parttable(json11::Json pt);
 uint64_t free_from_parttable(json11::Json pt);
+int fix_partition_type(std::string dev_by_uuid);

src/disk_tool_journal.cpp

@@ -13,7 +13,7 @@ int disk_tool_t::dump_journal()
         fprintf(stderr, "Invalid journal block size\n");
         return 1;
     }
-    first = true;
+    first_block = true;
     if (json)
         printf("[\n");
     if (all)
@@ -38,8 +38,8 @@ int disk_tool_t::dump_journal()
             }
             if (json)
             {
-                printf("%s{\"offset\":\"0x%lx\"", first ? "" : ",\n", journal_pos);
-                first = false;
+                printf("%s{\"offset\":\"0x%lx\"", first_block ? "" : ",\n", journal_pos);
+                first_block = false;
             }
             if (s == dsk.journal_block_size)
             {
@@ -55,10 +55,10 @@ int disk_tool_t::dump_journal()
                     printf("offset %08lx:\n", journal_pos);
                 else
                     printf(",\"entries\":[\n");
-                first2 = true;
+                first_entry = true;
                 process_journal_block(journal_buf, [this](int num, journal_entry *je) { dump_journal_entry(num, je, json); });
                 if (json)
-                    printf(first2 ? "]}" : "\n]}");
+                    printf(first_entry ? "]}" : "\n]}");
             }
             else
             {
@@ -75,39 +75,30 @@ int disk_tool_t::dump_journal()
     }
     else
     {
+        first_entry = true;
         process_journal([this](void *data)
         {
-            first2 = true;
+            if (json && dump_with_blocks)
+                first_entry = true;
             if (!json)
                 printf("offset %08lx:\n", journal_pos);
             auto pos = journal_pos;
             int r = process_journal_block(data, [this, pos](int num, journal_entry *je)
             {
-                if (json)
-                {
-                    if (dump_with_blocks)
-                    {
-                        if (first2)
-                            printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first ? "" : ",\n", pos);
-                    }
-                    else if (!first)
-                        printf("%s", ",\n");
-                    first = false;
-                }
+                if (json && dump_with_blocks && first_entry)
+                    printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first_block ? "" : ",\n", pos);
                 dump_journal_entry(num, je, json);
+                first_block = false;
             });
-            if (json)
-            {
-                if (dump_with_blocks && !first2)
-                    printf("\n]}");
-            }
-            else if (r <= 0)
+            if (json && dump_with_blocks && !first_entry)
+                printf("\n]}");
+            else if (!json && r <= 0)
                 printf("end of the journal\n");
             return r;
         });
     }
     if (json)
-        printf(first ? "]\n" : "\n]\n");
+        printf(first_block ? "]\n" : "\n]\n");
     return 0;
 }
 
@@ -214,9 +205,9 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
 {
     if (json)
     {
-        if (!first2)
+        if (!first_entry)
             printf(",\n");
-        first2 = false;
+        first_entry = false;
         printf(
             "{\"crc32\":\"%08x\",\"valid\":%s,\"crc32_prev\":\"%08x\"",
             je->crc32, (je_crc32(je) == je->crc32 ? "true" : "false"), je->crc32_prev
@@ -280,10 +271,12 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
     else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
     {
         printf(
-            json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"loc\":\"0x%lx\""
-                : "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx",
+            json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"offset\":%u,\"len\":%u,\"loc\":\"0x%lx\""
+                : "je_big_write%s oid=%lx:%lx ver=%lu offset=%u len=%u loc=%08lx",
             je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
-            je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location
+            je->big_write.oid.inode, je->big_write.oid.stripe,
+            je->big_write.version, je->big_write.offset, je->big_write.len,
+            je->big_write.location
         );
         if (je->big_write.size > sizeof(journal_entry_big_write))
         {
@@ -429,6 +422,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                     .stripe = sscanf_json(NULL, rec["stripe"]),
                 },
                 .version = rec["ver"].uint64_value(),
+                .offset = (uint32_t)rec["offset"].uint64_value(),
                 .len = (uint32_t)rec["len"].uint64_value(),
                 .location = sscanf_json(NULL, rec["loc"]),
             };

src/disk_tool_meta.cpp

@@ -124,14 +124,14 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v1_t *hdr)
     {
         printf("{\"version\":\"0.5\",\"meta_block_size\":%lu,\"entries\":[\n", dsk.meta_block_size);
     }
-    first = true;
+    first_entry = true;
 }
 
 void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
 {
     printf(
 #define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"version\":%lu"
-        (first ? ENTRY_FMT : (",\n" ENTRY_FMT)),
+        (first_entry ? ENTRY_FMT : (",\n" ENTRY_FMT)),
 #undef ENTRY_FMT
         block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
         entry->oid.stripe, entry->version
@@ -154,7 +154,7 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
     {
         printf("}");
     }
-    first = false;
+    first_entry = false;
 }
 
 int disk_tool_t::write_json_meta(json11::Json meta)

src/disk_tool_prepare.cpp

@@ -61,6 +61,11 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
                 fprintf(stderr, "%s already contains Vitastor OSD superblock, not creating OSD without --force\n", dev.c_str());
                 return 1;
             }
+            if (fix_partition_type(dev) != 0)
+            {
+                fprintf(stderr, "%s has incorrect type and we failed to change it to Vitastor type\n", dev.c_str());
+                return 1;
+            }
         }
     }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
@@ -317,7 +322,8 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
     {
         script += "+ "+size+" "+std::string(VITASTOR_PART_TYPE)+"\n";
     }
-    if (shell_exec({ "sfdisk", "--force", devinfo.path }, script, NULL, NULL) != 0)
+    std::string out;
+    if (shell_exec({ "sfdisk", "--no-reread", "--force", devinfo.path }, script, &out, NULL) != 0)
     {
         fprintf(stderr, "Failed to add %lu partition(s) with sfdisk\n", sizes.size());
         return {};
@@ -351,7 +357,8 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
                 {
                     iter++;
                     // Run partprobe
-                    if (iter > 1 || (r = shell_exec({ "partprobe", devinfo.path }, "", NULL, NULL)) != 0)
+                    std::string out;
+                    if (iter > 1 || (r = shell_exec({ "partprobe", devinfo.path }, "", &out, NULL)) != 0)
                     {
                         fprintf(
                             stderr, iter == 1 && r == 255

src/disk_tool_udev.cpp

@@ -5,6 +5,7 @@
 
 #include "disk_tool.h"
 #include "rw_blocking.h"
+#include "str_util.h"
 
 struct __attribute__((__packed__)) vitastor_disk_superblock_t
 {
@@ -123,7 +124,7 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
     return sb_size;
 }
 
-json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_nonref)
+json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors)
 {
     vitastor_disk_superblock_t *sb = NULL;
     uint8_t *buf = NULL;
@@ -144,7 +145,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         goto ex;
     }
     sb = (vitastor_disk_superblock_t*)buf;
-    if (sb->magic != VITASTOR_DISK_MAGIC)
+    if (sb->magic != VITASTOR_DISK_MAGIC && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "Invalid OSD superblock on %s: magic number mismatch\n", device.c_str());
@@ -172,7 +173,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         }
         sb = (vitastor_disk_superblock_t*)buf;
     }
-    if (sb->crc32c != crc32c(0, &sb->size, sb->size - ((uint8_t*)&sb->size - buf)))
+    if (sb->crc32c != crc32c(0, &sb->size, sb->size - ((uint8_t*)&sb->size - buf)) && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "Invalid OSD superblock on %s: crc32 mismatch\n", device.c_str());
@@ -186,14 +187,14 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         goto ex;
     }
     // Validate superblock
-    if (!osd_params["osd_num"].uint64_value())
+    if (!osd_params["osd_num"].uint64_value() && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "OSD superblock on %s lacks osd_num\n", device.c_str());
         osd_params = json11::Json();
         goto ex;
     }
-    if (osd_params["data_device"].string_value() == "")
+    if (osd_params["data_device"].string_value() == "" && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "OSD superblock on %s lacks data_device\n", device.c_str());
@@ -226,7 +227,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
     {
         device_type = "journal";
     }
-    else if (!ignore_nonref)
+    else if (!ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "Invalid OSD superblock on %s: does not refer to the device itself\n", device.c_str());
@@ -246,7 +247,7 @@ ex:
     return osd_params;
 }
 
-int disk_tool_t::systemd_start_stop_osds(std::vector<std::string> cmd, std::vector<std::string> devices)
+int disk_tool_t::systemd_start_stop_osds(const std::vector<std::string> & cmd, const std::vector<std::string> & devices)
 {
     if (!devices.size())
     {
@@ -306,8 +307,7 @@ int disk_tool_t::exec_osd(std::string device)
         argv[i] = (char*)argstr[i].c_str();
     }
     argv[argstr.size()] = NULL;
-    execvpe(osd_binary.c_str(), argv, environ);
-    return 0;
+    return execvpe(osd_binary.c_str(), argv, environ);
 }
 
 static int check_disabled_cache(std::string dev)
@@ -362,3 +362,140 @@ int disk_tool_t::pre_exec_osd(std::string device)
     }
     return 0;
 }
+
+int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
+{
+    std::vector<uint64_t> osd_numbers;
+    json11::Json::array superblocks;
+    for (auto & device: devices)
+    {
+        json11::Json sb = read_osd_superblock(device);
+        if (!sb.is_null())
+        {
+            uint64_t osd_num = sb["params"]["osd_num"].uint64_value();
+            osd_numbers.push_back(osd_num);
+            superblocks.push_back(sb);
+        }
+    }
+    if (!osd_numbers.size())
+    {
+        return 0;
+    }
+    std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd" };
+    for (auto osd_num: osd_numbers)
+    {
+        rm_osd_cli.push_back(std::to_string(osd_num));
+    }
+    // Check for data loss
+    rm_osd_cli.push_back("--dry-run");
+    std::string dry_run_ignore_stdout;
+    if (shell_exec(rm_osd_cli, "", &dry_run_ignore_stdout, NULL) != 0)
+    {
+        return 1;
+    }
+    // Disable & stop OSDs
+    std::vector<std::string> systemctl_cli = { "systemctl", "disable", "--now" };
+    for (auto osd_num: osd_numbers)
+    {
+        systemctl_cli.push_back("vitastor-osd@"+std::to_string(osd_num));
+    }
+    if (shell_exec(systemctl_cli, "", NULL, NULL) != 0)
+    {
+        return 1;
+    }
+    // Remove OSD metadata
+    rm_osd_cli.pop_back();
+    if (options["force"] != "")
+    {
+        rm_osd_cli.push_back("--force");
+    }
+    else if (options["allow_data_loss"] != "")
+    {
+        rm_osd_cli.push_back("--allow-data-loss");
+    }
+    if (shell_exec(rm_osd_cli, "", NULL, NULL) != 0)
+    {
+        return 1;
+    }
+    // Destroy OSD superblocks
+    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096);
+    for (auto & sb: superblocks)
+    {
+        for (auto dev_type: std::vector<std::string>{ "data", "meta", "journal" })
+        {
+            auto dev = sb["real_"+dev_type+"_device"].string_value();
+            if (dev != "")
+            {
+                int fd = -1, r = open(dev.c_str(), O_DIRECT|O_RDWR);
+                if (r >= 0)
+                {
+                    fd = r;
+                    r = read_blocking(fd, buf, 4096);
+                    if (r == 4096)
+                    {
+                        // Clear magic and CRC
+                        memset(buf, 0, 12);
+                        r = lseek64(fd, 0, 0);
+                        if (r == 0)
+                        {
+                            r = write_blocking(fd, buf, 4096);
+                            if (r == 4096)
+                                r = 0;
+                        }
+                    }
+                }
+                if (fd >= 0)
+                    close(fd);
+                if (r != 0)
+                {
+                    fprintf(stderr, "Failed to clear OSD %lu %s device %s superblock: %s\n",
+                        sb["params"]["osd_num"].uint64_value(), dev_type.c_str(), dev.c_str(), strerror(errno));
+                }
+                else
+                {
+                    fprintf(stderr, "OSD %lu %s device %s superblock cleared\n",
+                        sb["params"]["osd_num"].uint64_value(), dev_type.c_str(), dev.c_str());
+                }
+                if (sb["params"][dev_type+"_device"].string_value().substr(0, 22) == "/dev/disk/by-partuuid/")
+                {
+                    // Delete the partition itself
+                    auto uuid_to_del = strtolower(sb["params"][dev_type+"_device"].string_value().substr(22));
+                    auto parent_dev = get_parent_device(dev);
+                    if (parent_dev == "" || parent_dev == dev)
+                    {
+                        fprintf(stderr, "Failed to delete partition %s: failed to find parent device\n", dev.c_str());
+                        continue;
+                    }
+                    auto pt = read_parttable("/dev/"+parent_dev);
+                    if (!pt.is_object())
+                        continue;
+                    json11::Json::array newpt = pt["partitions"].array_items();
+                    for (int i = 0; i < newpt.size(); i++)
+                    {
+                        if (strtolower(newpt[i]["uuid"].string_value()) == uuid_to_del)
+                        {
+                            auto old_part = newpt[i];
+                            newpt.erase(newpt.begin()+i, newpt.begin()+i+1);
+                            vitastor_dev_info_t devinfo = {
+                                .path = "/dev/"+parent_dev,
+                                .pt = json11::Json::object{ { "partitions", newpt } },
+                            };
+                            add_partitions(devinfo, {});
+                            struct stat st;
+                            if (stat(old_part["node"].string_value().c_str(), &st) == 0 ||
+                                errno != ENOENT)
+                            {
+                                std::string out;
+                                shell_exec({ "partprobe", "/dev/"+parent_dev }, "", &out, NULL);
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    free(buf);
+    buf = NULL;
+    return 0;
+}

src/disk_tool_upgrade.cpp

@@ -38,42 +38,6 @@ static std::map<std::string, std::string> read_vitastor_unit(std::string unit)
     return r;
 }
 
-static int fix_partition_type(std::string dev_by_uuid)
-{
-    auto uuid = strtolower(dev_by_uuid.substr(dev_by_uuid.rfind('/')+1));
-    std::string parent_dev = get_parent_device(realpath_str(dev_by_uuid, false));
-    if (parent_dev == "")
-        return 1;
-    auto pt = read_parttable("/dev/"+parent_dev);
-    if (pt.is_null())
-        return 1;
-    std::string script = "label: gpt\n\n";
-    for (const auto & part: pt["partitions"].array_items())
-    {
-        bool this_part = (strtolower(part["uuid"].string_value()) == uuid);
-        if (this_part && strtolower(part["type"].string_value()) == "e7009fac-a5a1-4d72-af72-53de13059903")
-        {
-            // Already correct type
-            return 0;
-        }
-        script += part["node"].string_value()+": ";
-        bool first = true;
-        for (const auto & kv: part.object_items())
-        {
-            if (kv.first != "node")
-            {
-                script += (first ? "" : ", ")+kv.first+"="+
-                    (kv.first == "type" && this_part
-                        ? "e7009fac-a5a1-4d72-af72-53de13059903"
-                        : (kv.second.is_string() ? kv.second.string_value() : kv.second.dump()));
-                first = false;
-            }
-        }
-        script += "\n";
-    }
-    return shell_exec({ "sfdisk", "--no-reread", "--force", "/dev/"+parent_dev }, script, NULL, NULL);
-}
-
 int disk_tool_t::upgrade_simple_unit(std::string unit)
 {
     if (stoull_full(unit) != 0)

src/disk_tool_utils.cpp

@@ -239,7 +239,8 @@ int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std
     {
         // Child
         dup2(child_stdin[0], 0);
-        dup2(child_stdout[1], 1);
+        if (out)
+            dup2(child_stdout[1], 1);
         if (err)
             dup2(child_stderr[1], 2);
         close(child_stdin[0]);
@@ -250,9 +251,7 @@ int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std
         close(child_stderr[1]);
         char *argv[cmd.size()+1];
         for (int i = 0; i < cmd.size(); i++)
-        {
             argv[i] = (char*)cmd[i].c_str();
-        }
         argv[cmd.size()] = NULL;
         execvp(argv[0], argv);
         std::string full_cmd;
@@ -354,3 +353,40 @@ uint64_t free_from_parttable(json11::Json pt)
         free *= pt["sectorsize"].uint64_value();
     return free;
 }
+
+int fix_partition_type(std::string dev_by_uuid)
+{
+    auto uuid = strtolower(dev_by_uuid.substr(dev_by_uuid.rfind('/')+1));
+    std::string parent_dev = get_parent_device(realpath_str(dev_by_uuid, false));
+    if (parent_dev == "")
+        return 1;
+    auto pt = read_parttable("/dev/"+parent_dev);
+    if (pt.is_null() || pt.is_bool())
+        return 1;
+    std::string script = "label: gpt\n\n";
+    for (const auto & part: pt["partitions"].array_items())
+    {
+        bool this_part = (strtolower(part["uuid"].string_value()) == uuid);
+        if (this_part && strtolower(part["type"].string_value()) == "e7009fac-a5a1-4d72-af72-53de13059903")
+        {
+            // Already correct type
+            return 0;
+        }
+        script += part["node"].string_value()+": ";
+        bool first = true;
+        for (const auto & kv: part.object_items())
+        {
+            if (kv.first != "node")
+            {
+                script += (first ? "" : ", ")+kv.first+"="+
+                    (kv.first == "type" && this_part
+                        ? "e7009fac-a5a1-4d72-af72-53de13059903"
+                        : (kv.second.is_string() ? kv.second.string_value() : kv.second.dump()));
+                first = false;
+            }
+        }
+        script += "\n";
+    }
+    std::string out;
+    return shell_exec({ "sfdisk", "--no-reread", "--force", "/dev/"+parent_dev }, script, &out, NULL);
+}

src/osd_primary.h

@@ -22,7 +22,7 @@ struct osd_primary_op_data_t
     pg_num_t pg_num;
     object_id oid;
     uint64_t target_ver;
-    uint64_t fact_ver = 0;
+    uint64_t orig_ver = 0, fact_ver = 0;
     uint64_t scheme = 0;
     int n_subops = 0, done = 0, errors = 0, epipe = 0;
     int degraded = 0, pg_size, pg_data_size;

src/osd_primary_write.cpp

@@ -138,6 +138,7 @@ resume_3:
         }
     }
     // Send writes
+    op_data->orig_ver = op_data->fact_ver;
     if ((op_data->fact_ver >> (64-PG_EPOCH_BITS)) < pg.epoch)
     {
         op_data->target_ver = ((uint64_t)pg.epoch << (64-PG_EPOCH_BITS)) | 1;
@@ -194,7 +195,7 @@ resume_7:
     {
         return;
     }
-    if (op_data->fact_ver == 1)
+    if (op_data->orig_ver == 0)
     {
         // Object is created
         pg.clean_count++;

src/str_util.cpp

@@ -56,6 +56,16 @@ std::string base64_decode(const std::string &in)
     return out;
 }
 
+std::string strtoupper(const std::string & in)
+{
+    std::string s = in;
+    for (int i = 0; i < s.length(); i++)
+    {
+        s[i] = toupper(s[i]);
+    }
+    return s;
+}
+
 std::string strtolower(const std::string & in)
 {
     std::string s = in;

src/str_util.h

@@ -8,6 +8,7 @@
 std::string base64_encode(const std::string &in);
 std::string base64_decode(const std::string &in);
 uint64_t parse_size(std::string size_str, bool *ok = NULL);
+std::string strtoupper(const std::string & in);
 std::string strtolower(const std::string & in);
 std::string trim(const std::string & in, const char *rm_chars = " \n\r\t");
 std::string str_replace(const std::string & in, const std::string & needle, const std::string & replacement);

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.8.2
+Version: 0.8.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/run_3osds.sh

@@ -61,15 +61,31 @@ fi
 POOLCFG='"name":"testpool","failure_domain":"osd",'$POOLCFG
 $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}'
 
-sleep 2
+wait_up()
+{
+    local sec=$1
+    local i=0
+    local configured=0
+    while [[ $i -lt $sec ]]; do
+        if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] |
+            select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT; then
+            configured=1
+            if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then
+                break
+            fi
+        fi
+        sleep 1
+        i=$((i+1))
+        if [ $i -eq $sec ]; then
+            if [[ $configured -ne 0 ]]; then
+                format_error "FAILED: $PG_COUNT PG(s) NOT CONFIGURED"
+            fi
+            format_error "FAILED: $PG_COUNT PG(s) NOT UP"
+        fi
+    done
+}
 
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == '$PG_SIZE') | length) == '$PG_COUNT); then
-    format_error "FAILED: $PG_COUNT PGS NOT CONFIGURED"
-fi
-
-if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT); then
-    format_error "FAILED: $PG_COUNT PGS NOT UP"
-fi
+wait_up 60
 
 try_reweight()
 {

tests/test_add_osd.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -ex
 
-PG_COUNT=16
+PG_COUNT=2048
 
 . `dirname $0`/run_3osds.sh
 
@@ -10,8 +10,7 @@ LD_PRELOAD="build/src/libfio_vitastor.so" \
 
 for i in 4; do
     dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-disk simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
-    eval OSD${i}_PID=$!
+    start_osd $i
 done
 
 sleep 2
@@ -33,4 +32,28 @@ if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '(
     format_error "FAILED: $PG_COUNT PGS NOT ACTIVE"
 fi
 
+sleep 1
+kill -9 $OSD4_PID
+sleep 1
+build/src/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4
+
+sleep 2
+
+for i in {1..10}; do
+    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+        jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \
+        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \
+        break
+    sleep 1
+done
+
+if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then
+    format_error "FAILED: OSD NOT REMOVED FROM DISTRIBUTION"
+fi
+
+if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT''); then
+    format_error "FAILED: $PG_COUNT PGS NOT ACTIVE"
+fi
+
 format_green OK