WIP/experimental LRC matrix generator

This reduces the probability of hitting OSD stalls with EC due to "deadlocks"
where two parallel write operations wait for each other to complete

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.8.0")
+set(VERSION "0.8.3")
 
 add_subdirectory(src)

csi/Dockerfile

@@ -18,15 +18,19 @@ ENV CSI_ENDPOINT=""
 
 RUN apt-get update && \
     apt-get install -y wget && \
-    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
-    (echo deb http://vitastor.io/debian buster main > /etc/apt/sources.list.d/vitastor.list) && \
     (echo deb http://deb.debian.org/debian buster-backports main > /etc/apt/sources.list.d/backports.list) && \
     (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
     apt-get update && \
-    apt-get install -y e2fsprogs xfsprogs vitastor kmod && \
+    apt-get install -y e2fsprogs xfsprogs kmod && \
     apt-get clean && \
     (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
 
 COPY --from=build /app/vitastor-csi /bin/
 
+RUN (echo deb http://vitastor.io/debian buster main > /etc/apt/sources.list.d/vitastor.list) && \
+    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
+    apt-get update && \
+    apt-get install -y vitastor-client && \
+    apt-get clean
+
 ENTRYPOINT ["/bin/vitastor-csi"]

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.8.0
+VERSION ?= v0.8.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.8.0
+          image: vitalif/vitastor-csi:v0.8.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"
@@ -102,7 +102,7 @@ spec:
             - "--health-port=9898"
           env:
             - name: CSI_ENDPOINT
-              value: unix://csi/csi.sock
+              value: unix:///csi/csi.sock
           volumeMounts:
           - mountPath: /csi
             name: socket-dir

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.8.0
+          image: vitalif/vitastor-csi:v0.8.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.8.0"
+    vitastorCSIDriverVersion = "0.8.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.8.0-1) unstable; urgency=medium
+vitastor (0.8.3-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.8.0-1) unstable; urgency=medium
+vitastor (0.8.3-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/pve-storage-vitastor.install

@@ -1 +1 @@
-patches/PVE_VitastorPlugin.pm usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm
+patches/VitastorPlugin.pm usr/share/perl5/PVE/Storage/Custom/

debian/vitastor.Dockerfile

@@ -34,8 +34,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.8.0; \
-    cd vitastor-0.8.0; \
+    cp -r /root/vitastor vitastor-0.8.3; \
+    cd vitastor-0.8.3; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -48,8 +48,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.8.0.orig.tar.xz vitastor-0.8.0; \
-    cd vitastor-0.8.0; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.8.3.orig.tar.xz vitastor-0.8.3; \
+    cd vitastor-0.8.3; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config/osd.en.md

@@ -17,6 +17,7 @@ initialization and can be changed with an OSD restart.
 - [autosync_interval](#autosync_interval)
 - [autosync_writes](#autosync_writes)
 - [recovery_queue_depth](#recovery_queue_depth)
+- [recovery_pg_switch](#recovery_pg_switch)
 - [recovery_sync_batch](#recovery_sync_batch)
 - [readonly](#readonly)
 - [no_recovery](#no_recovery)
@@ -115,6 +116,16 @@ Maximum recovery operations per one primary OSD at any given moment of time.
 Currently it's the only parameter available to tune the speed or recovery
 and rebalancing, but it's planned to implement more.
 
+## recovery_pg_switch
+
+- Type: integer
+- Default: 128
+
+Number of recovery operations before switching to recovery of the next PG.
+The idea is to mix all PGs during recovery for more even space and load
+distribution but still benefit from recovery queue depth greater than 1.
+Degraded PGs are anyway scanned first.
+
 ## recovery_sync_batch
 
 - Type: integer

docs/config/osd.ru.md

@@ -18,6 +18,7 @@
 - [autosync_interval](#autosync_interval)
 - [autosync_writes](#autosync_writes)
 - [recovery_queue_depth](#recovery_queue_depth)
+- [recovery_pg_switch](#recovery_pg_switch)
 - [recovery_sync_batch](#recovery_sync_batch)
 - [readonly](#readonly)
 - [no_recovery](#no_recovery)
@@ -119,6 +120,17 @@ OSD, чтобы успевать очищать журнал - без них OSD
 для ускорения или замедления восстановления и перебалансировки данных, но
 в планах реализация других параметров.
 
+## recovery_pg_switch
+
+- Тип: целое число
+- Значение по умолчанию: 128
+
+Число операций восстановления перед переключением на восстановление другой PG.
+Идея заключается в том, чтобы восстанавливать все PG одновременно для более
+равномерного распределения места и нагрузки, но при этом всё равно выигрывать
+от глубины очереди восстановления, большей, чем 1. Деградированные PG в любом
+случае сканируются первыми.
+
 ## recovery_sync_batch
 
 - Тип: целое число

docs/config/pool.en.md

@@ -82,7 +82,7 @@ Parent node reference is required for intermediate tree nodes.
 Separate OSD settings are set in etc keys `/vitastor/config/osd/<number>`
 in JSON format `{"<key>":<value>}`.
 
-As of now, there is only one setting:
+As of now, two settings are supported:
 
 ## reweight
 
@@ -96,6 +96,15 @@ This means an OSD configured with reweight lower than 1 receives less PGs than
 it normally would. An OSD with reweight = 0 won't store any data. You can set
 reweight to 0 to trigger rebalance and remove all data from an OSD.
 
+## tags
+
+- Type: string or array of strings
+
+Sets tag or multiple tags for this OSD. Tags can be used to group OSDs into
+subsets and then use a specific subset for pool instead of all OSDs.
+For example you can mark SSD OSDs with tag "ssd" and HDD OSDs with "hdd" and
+such tags will work as device classes.
+
 # Pool parameters
 
 ## name

docs/config/pool.ru.md

@@ -81,7 +81,10 @@
 Настройки отдельных OSD задаются в ключах etcd `/vitastor/config/osd/<number>`
 в JSON-формате `{"<key>":<value>}`.
 
-На данный момент поддерживается одна настройка:
+На данный момент поддерживаются две настройки:
+
+- [reweight](#reweight)
+- [tags](#tags)
 
 ## reweight
 
@@ -96,6 +99,15 @@
 хранении данных вообще. Вы можете установить reweight в 0, чтобы убрать
 все данные с OSD.
 
+## tags
+
+- Тип: строка или массив строк
+
+Задаёт тег или набор тегов для данного OSD. Теги можно использовать, чтобы
+делить OSD на множества и потом размещать пул только на части OSD, а не на
+всех. Можно, например, пометить SSD OSD тегом "ssd", а HDD тегом "hdd", в
+этом смысле теги работают аналогично классам устройств.
+
 # Параметры
 
 ## name

docs/config/src/osd.yml

@@ -102,6 +102,20 @@
     момент времени. На данный момент единственный параметр, который можно менять
     для ускорения или замедления восстановления и перебалансировки данных, но
     в планах реализация других параметров.
+- name: recovery_pg_switch
+  type: int
+  default: 128
+  info: |
+    Number of recovery operations before switching to recovery of the next PG.
+    The idea is to mix all PGs during recovery for more even space and load
+    distribution but still benefit from recovery queue depth greater than 1.
+    Degraded PGs are anyway scanned first.
+  info_ru: |
+    Число операций восстановления перед переключением на восстановление другой PG.
+    Идея заключается в том, чтобы восстанавливать все PG одновременно для более
+    равномерного распределения места и нагрузки, но при этом всё равно выигрывать
+    от глубины очереди восстановления, большей, чем 1. Деградированные PG в любом
+    случае сканируются первыми.
 - name: recovery_sync_batch
   type: int
   default: 16

docs/installation/proxmox.en.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4 and 7.1 are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-7.3 are supported):
 
-- Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts
-  (buster for 6.4, bullseye for 7.1)
+- Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
+  buster for 6.4, bullseye for 7.3, pve7.1 for 7.1, pve7.2 for 7.2
 - Install vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* or see note) packages from Vitastor repository
 - Define storage in `/etc/pve/storage.cfg` (see below)
 - Block network access from VMs to Vitastor network (to OSDs and etcd),
@@ -35,5 +35,5 @@ vitastor: vitastor
     vitastor_nbd 0
 ```
 
-\* Note: you can also manually copy [patches/PVE_VitastorPlugin.pm](patches/PVE_VitastorPlugin.pm) to Proxmox hosts
+\* Note: you can also manually copy [patches/VitastorPlugin.pm](patches/VitastorPlugin.pm) to Proxmox hosts
 as `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm` instead of installing pve-storage-vitastor.

docs/installation/proxmox.ru.md

@@ -6,10 +6,10 @@
 
 # Proxmox
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4 и 7.1):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-7.3):
 
-- Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox
-  (buster для 6.4, bullseye для 7.1)
+- Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
+  buster для 6.4, bullseye для 7.3, pve7.1 для 7.1, pve7.2 для 7.2
 - Установите пакеты vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* или см. сноску) из репозитория Vitastor
 - Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже)
 - Обязательно заблокируйте доступ от виртуальных машин к сети Vitastor (OSD и etcd), т.к. Vitastor (пока) не поддерживает аутентификацию
@@ -35,5 +35,5 @@ vitastor: vitastor
 ```
 
 \* Примечание: вместо установки пакета pve-storage-vitastor вы можете вручную скопировать файл
-[patches/PVE_VitastorPlugin.pm](patches/PVE_VitastorPlugin.pm) на хосты Proxmox как
+[patches/VitastorPlugin.pm](patches/VitastorPlugin.pm) на хосты Proxmox как
 `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm`.

docs/intro/quickstart.en.md

@@ -70,7 +70,7 @@ For EC pools the configuration should look like the following:
 
 ```
 etcdctl --endpoints=... put /vitastor/config/pools '{"2":{"name":"ecpool",
-  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`
+  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}'
 ```
 
 After you do this, one of the monitors will configure PGs and OSDs will start them.

docs/intro/quickstart.ru.md

@@ -71,7 +71,7 @@ etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool",
 
 ```
 etcdctl --endpoints=... put /vitastor/config/pools '{"2":{"name":"ecpool",
-  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`
+  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}'
 ```
 
 После этого один из мониторов должен сконфигурировать PG, а OSD должны запустить их.

docs/usage/cli.en.md

@@ -20,6 +20,7 @@ It supports the following commands:
 - [rm-data](#rm-data)
 - [merge-data](#merge-data)
 - [alloc-osd](#alloc-osd)
+- [rm-osd](#rm-osd)
 
 Global options:
 
@@ -175,3 +176,14 @@ Merge layer data without changing metadata. Merge `<from>`..`<to>` to `<target>`
 `vitastor-cli alloc-osd`
 
 Allocate a new OSD number and reserve it by creating empty `/osd/stats/<n>` key.
+
+## rm-osd
+
+`vitastor-cli rm-osd [--force] [--allow-data-loss] [--dry-run] <osd_id> [osd_id...]`
+
+Remove metadata and configuration for specified OSD(s) from etcd.
+
+Refuses to remove OSDs with data without `--force` and `--allow-data-loss`.
+
+With `--dry-run` only checks if deletion is possible without data loss and
+redundancy degradation.

docs/usage/cli.ru.md

@@ -21,6 +21,7 @@ vitastor-cli - интерфейс командной строки для адм
 - [rm-data](#rm-data)
 - [merge-data](#merge-data)
 - [alloc-osd](#alloc-osd)
+- [rm-osd](#rm-osd)
 
 Глобальные опции:
 
@@ -186,3 +187,14 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 
 Атомарно выделить новый номер OSD и зарезервировать его, создав в etcd пустой
 ключ `/osd/stats/<n>`.
+
+## rm-osd
+
+`vitastor-cli rm-osd [--force] [--allow-data-loss] [--dry-run] <osd_id> [osd_id...]`
+
+Удалить метаданные и конфигурацию для заданных OSD из etcd.
+
+Отказывается удалять OSD с данными без опций `--force` и `--allow-data-loss`.
+
+С опцией `--dry-run` только проверяет, возможно ли удаление без потери данных и деградации
+избыточности.

docs/usage/disk.en.md

@@ -14,6 +14,7 @@ It supports the following commands:
 - [upgrade-simple](#upgrade-simple)
 - [resize](#resize)
 - [start/stop/restart/enable/disable](#start/stop/restart/enable/disable)
+- [purge](#purge)
 - [read-sb](#read-sb)
 - [write-sb](#write-sb)
 - [udev](#udev)
@@ -155,11 +156,22 @@ Commands are passed to `systemctl` with `vitastor-osd@<num>` units as arguments.
 
 When `--now` is added to enable/disable, OSDs are also immediately started/stopped.
 
+## purge
+
+`vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]`
+
+Purge Vitastor OSD(s) on specified device(s). Uses `vitastor-cli rm-osd` to check
+if deletion is possible without data loss and to actually remove metadata from etcd.
+`--force` and `--allow-data-loss` options may be used to ignore safety check results.
+
+Requires `vitastor-cli`, `sfdisk` and `partprobe` (from parted) utilities.
+
 ## read-sb
 
-`vitastor-disk read-sb <device>`
+`vitastor-disk read-sb [--force] <device>`
 
 Try to read Vitastor OSD superblock from `<device>` and print it in JSON format.
+`--force` allows to ignore validation errors.
 
 ## write-sb
 

docs/usage/disk.ru.md

@@ -14,6 +14,7 @@ vitastor-disk - инструмент командной строки для уп
 - [upgrade-simple](#upgrade-simple)
 - [resize](#resize)
 - [start/stop/restart/enable/disable](#start/stop/restart/enable/disable)
+- [purge](#purge)
 - [read-sb](#read-sb)
 - [write-sb](#write-sb)
 - [udev](#udev)
@@ -158,12 +159,25 @@ throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.
 Когда к командам включения/выключения добавляется параметр `--now`, OSD также сразу
 запускаются/останавливаются.
 
+## purge
+
+`vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]`
+
+Удалить OSD на заданном диске/дисках. Использует `vitastor-cli rm-osd` для проверки
+возможности удаления без потери данных и для удаления OSD из etcd. Опции `--force`
+и `--allow-data-loss` служат для обхода данной защиты в случае необходимости.
+
+Команде требуются утилиты `vitastor-cli`, `sfdisk` и `partprobe` (из состава parted).
+
 ## read-sb
 
-`vitastor-disk read-sb <device>`
+`vitastor-disk read-sb [--force] <device>`
 
 Прочитать суперблок OSD с диска `<device>` и вывести его в формате JSON.
 
+Опция `--force` позволяет читать суперблок, даже если он считается некорректным
+из-за ошибок валидации.
+
 ## write-sb
 
 `vitastor-disk write-sb <device>`

mon/PGUtil.js

@@ -21,7 +21,7 @@ function add_pg_history(new_pg_history, new_pg, prev_pgs, prev_pg_history, old_p
     {
         for (const pg of oh.osd_sets)
         {
-            nh.osd_sets[pg.join(' ')] = pg;
+            nh.osd_sets[pg.join(' ')] = pg.map(osd_num => Number(osd_num));
         }
     }
     if (oh && oh.all_peers && oh.all_peers.length)

mon/mon.js

@@ -663,12 +663,15 @@ class Mon
     async save_last_clean()
     {
         // last_clean_pgs is used to avoid extra data move when observing a series of changes in the cluster
+        const new_clean_pgs = { items: {} };
+    next_pool:
         for (const pool_id in this.state.config.pools)
         {
+            new_clean_pgs.items[pool_id] = (this.state.history.last_clean_pgs.items||{})[pool_id];
             const pool_cfg = this.state.config.pools[pool_id];
             if (!this.validate_pool_cfg(pool_id, pool_cfg, false))
             {
-                continue;
+                continue next_pool;
             }
             for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
             {
@@ -677,17 +680,18 @@ class Mon
                     !(this.state.pg.state[pool_id][pg_num].state instanceof Array))
                 {
                     // Unclean
-                    return;
+                    continue next_pool;
                 }
                 let st = this.state.pg.state[pool_id][pg_num].state.join(',');
                 if (st != 'active' && st != 'active,left_on_dead' && st != 'left_on_dead,active')
                 {
                     // Unclean
-                    return;
+                    continue next_pool;
                 }
             }
+            new_clean_pgs.items[pool_id] = this.state.config.pgs.items[pool_id];
         }
-        this.state.history.last_clean_pgs = JSON.parse(JSON.stringify(this.state.config.pgs));
+        this.state.history.last_clean_pgs = new_clean_pgs;
         await this.etcd_call('/kv/txn', {
             success: [ { requestPut: {
                 key: b64(this.etcd_prefix+'/history/last_clean_pgs'),
@@ -1374,16 +1378,14 @@ class Mon
     // This is required for multiple change events to trigger at most 1 recheck in 1s
     schedule_recheck()
     {
-        if (this.recheck_timer)
+        if (!this.recheck_timer)
         {
-            clearTimeout(this.recheck_timer);
-            this.recheck_timer = null;
+            this.recheck_timer = setTimeout(() =>
+            {
+                this.recheck_timer = null;
+                this.recheck_pgs().catch(this.die);
+            }, this.config.mon_change_timeout || 1000);
         }
-        this.recheck_timer = setTimeout(() =>
-        {
-            this.recheck_timer = null;
-            this.recheck_pgs().catch(this.die);
-        }, this.config.mon_change_timeout || 1000);
     }
 
     sum_op_stats(timestamp, prev_stats)
@@ -1693,6 +1695,7 @@ class Mon
             // Do not clear these to null
             kv.value = kv.value || {};
         }
+        const old = cur[key_parts[key_parts.length-1]];
         cur[key_parts[key_parts.length-1]] = kv.value;
         if (key === 'config/global')
         {
@@ -1717,7 +1720,12 @@ class Mon
         }
         else if (key_parts[0] === 'osd' && key_parts[1] === 'stats')
         {
-            // Recheck PGs <osd_out_time> later
+            // Recheck OSD tree on OSD addition/deletion
+            if ((!old) != (!kv.value) || old && kv.value && old.size != kv.value.size)
+            {
+                this.schedule_recheck();
+            }
+            // Recheck PGs <osd_out_time> after last OSD statistics report
             this.schedule_next_recheck_at(
                 !this.state.osd.stats[key[2]] ? 0 : this.state.osd.stats[key[2]].time+this.config.osd_out_time
             );

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.8.0'
+VERSION = '0.8.3'
 
 LOG = logging.getLogger(__name__)
 

patches/pve-qemu-6.2-vitastor.patch

@@ -0,0 +1,169 @@
+Index: qemu/block/meson.build
+===================================================================
+--- qemu.orig/block/meson.build
++++ qemu/block/meson.build
+@@ -91,6 +91,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: qemu/meson.build
+===================================================================
+--- qemu.orig/meson.build
++++ qemu/meson.build
+@@ -838,6 +838,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'), kwargs: static_kwargs)
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -1459,6 +1479,7 @@ config_host_data.set('CONFIG_LINUX_AIO',
+ config_host_data.set('CONFIG_LINUX_IO_URING', linux_io_uring.found())
+ config_host_data.set('CONFIG_LIBPMEM', libpmem.found())
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_SDL', sdl.found())
+ config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
+ config_host_data.set('CONFIG_SECCOMP', seccomp.found())
+@@ -3424,6 +3445,7 @@ if spice_protocol.found()
+   summary_info += {'  spice server support': spice}
+ endif
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'xfsctl support':    config_host.has_key('CONFIG_XFS')}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+Index: qemu/meson_options.txt
+===================================================================
+--- qemu.orig/meson_options.txt
++++ qemu/meson_options.txt
+@@ -121,6 +121,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('gtk', type : 'feature', value : 'auto',
+        description: 'GTK+ user interface')
+ option('sdl', type : 'feature', value : 'auto',
+Index: qemu/qapi/block-core.json
+===================================================================
+--- qemu.orig/qapi/block-core.json
++++ qemu/qapi/block-core.json
+@@ -3179,7 +3179,7 @@
+             'preallocate', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor', 'vmdk', 'vpc', 'vvfat' ] }
+ 
+ ##
+ # @BlockdevOptionsFile:
+@@ -4125,6 +4125,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -4520,6 +4542,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'vmdk':       'BlockdevOptionsGenericCOWFormat',
+       'vpc':        'BlockdevOptionsGenericFormat',
+       'vvfat':      'BlockdevOptionsVVFAT'
+@@ -4910,6 +4933,17 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5108,6 +5142,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: qemu/scripts/ci/org.centos/stream/8/x86_64/configure
+===================================================================
+--- qemu.orig/scripts/ci/org.centos/stream/8/x86_64/configure
++++ qemu/scripts/ci/org.centos/stream/8/x86_64/configure
+@@ -31,7 +31,7 @@
+ --with-git=meson \
+ --with-git-submodules=update \
+ --target-list="x86_64-softmmu" \
+---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
++--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
+ --audio-drv-list="" \
+ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
+ --with-coroutine=ucontext \
+@@ -183,6 +183,7 @@
+ --enable-opengl \
+ --enable-pie \
+ --enable-rbd \
++--enable-vitastor \
+ --enable-rdma \
+ --enable-seccomp \
+ --enable-snappy \

patches/pve-qemu-7.1-vitastor.patch

@@ -0,0 +1,169 @@
+Index: qemu/block/meson.build
+===================================================================
+--- qemu.orig/block/meson.build
++++ qemu/block/meson.build
+@@ -111,6 +111,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: qemu/meson.build
+===================================================================
+--- qemu.orig/meson.build
++++ qemu/meson.build
+@@ -967,6 +967,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'), kwargs: static_kwargs)
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -1802,6 +1822,7 @@ config_host_data.set('CONFIG_NUMA', numa
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_SDL', sdl.found())
+ config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
+@@ -3965,6 +3986,7 @@ if spice_protocol.found()
+   summary_info += {'  spice server support': spice}
+ endif
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: qemu/meson_options.txt
+===================================================================
+--- qemu.orig/meson_options.txt
++++ qemu/meson_options.txt
+@@ -167,6 +167,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: qemu/qapi/block-core.json
+===================================================================
+--- qemu.orig/qapi/block-core.json
++++ qemu/qapi/block-core.json
+@@ -3209,7 +3209,7 @@
+             'preallocate', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor', 'vmdk', 'vpc', 'vvfat' ] }
+ 
+ ##
+ # @BlockdevOptionsFile:
+@@ -4149,6 +4149,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -4593,6 +4615,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'vmdk':       'BlockdevOptionsGenericCOWFormat',
+       'vpc':        'BlockdevOptionsGenericFormat',
+       'vvfat':      'BlockdevOptionsVVFAT'
+@@ -4985,6 +5008,17 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5182,6 +5216,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: qemu/scripts/ci/org.centos/stream/8/x86_64/configure
+===================================================================
+--- qemu.orig/scripts/ci/org.centos/stream/8/x86_64/configure
++++ qemu/scripts/ci/org.centos/stream/8/x86_64/configure
+@@ -31,7 +31,7 @@
+ --with-git=meson \
+ --with-git-submodules=update \
+ --target-list="x86_64-softmmu" \
+---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
++--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
+ --audio-drv-list="" \
+ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
+ --with-coroutine=ucontext \
+@@ -179,6 +179,7 @@
+ --enable-opengl \
+ --enable-pie \
+ --enable-rbd \
++--enable-vitastor \
+ --enable-rdma \
+ --enable-seccomp \
+ --enable-snappy \

patches/qemu-make-patches.sh

@@ -9,7 +9,7 @@ for i in "$DIR"/qemu-*-vitastor.patch "$DIR"/pve-qemu-*-vitastor.patch; do
         echo '===================================================================' >> $i
         echo '--- /dev/null' >> $i
         echo '+++ a/block/vitastor.c' >> $i
-        echo '@@ -0,0 +1,'$(wc -l "$DIR"/../src/qemu_driver.c)' @@' >> $i
+        echo '@@ -0,0 +1,'$(wc -l "$DIR"/../src/qemu_driver.c | cut -d ' ' -f 1)' @@' >> $i
         cat "$DIR"/../src/qemu_driver.c | sed 's/^/+/' >> $i
     fi
 done

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.8.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.8.0$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.8.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.8.3$(rpm --eval '%dist').tar.gz *

rpm/qemu-kvm-4.2-el7.spec.patch

@@ -58,7 +58,7 @@
 +BuildRequires: gperftools-devel
 +BuildRequires: libusbx-devel >= 1.0.21
  %if %{have_usbredir}
- BuildRequires: usbredir-devel >= 0.8.0
+ BuildRequires: usbredir-devel >= 0.8.2
  %endif
 @@ -856,12 +861,13 @@ BuildRequires: virglrenderer-devel
  # For smartcard NSS support

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.8.0.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.8.3.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.8.0
+Version:        0.8.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.8.0.el7.tar.gz
+Source0:        vitastor-0.8.3.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.8.0.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.8.3.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.8.0
+Version:        0.8.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.8.0.el8.tar.gz
+Source0:        vitastor-0.8.3.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.8.0")
+add_definitions(-DVERSION="0.8.3")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -140,6 +140,7 @@ add_library(vitastor_client SHARED
 	cli_merge.cpp
 	cli_rm_data.cpp
 	cli_rm.cpp
+	cli_rm_osd.cpp
 )
 set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h")
 target_link_libraries(vitastor_client
@@ -234,9 +235,16 @@ add_executable(osd_test osd_test.cpp rw_blocking.cpp addr_util.cpp)
 target_link_libraries(osd_test tcmalloc_minimal)
 
 # osd_rmw_test
+# FIXME: Move to tests
 add_executable(osd_rmw_test osd_rmw_test.cpp allocator.cpp)
 target_link_libraries(osd_rmw_test Jerasure ${ISAL_LIBRARIES} tcmalloc_minimal)
 
+if (ISAL_LIBRARIES)
+	add_executable(osd_rmw_test_je osd_rmw_test.cpp allocator.cpp)
+	target_compile_definitions(osd_rmw_test_je PUBLIC -DNO_ISAL)
+	target_link_libraries(osd_rmw_test_je Jerasure tcmalloc_minimal)
+endif (ISAL_LIBRARIES)
+
 # stub_uring_osd
 add_executable(stub_uring_osd
 	stub_uring_osd.cpp
@@ -263,6 +271,14 @@ target_link_libraries(test_cas
 	vitastor_client
 )
 
+# test_crc32
+add_executable(test_crc32
+	test_crc32.cpp
+)
+target_link_libraries(test_crc32
+	vitastor_blk
+)
+
 # test_cluster_client
 add_executable(test_cluster_client
 	test_cluster_client.cpp

src/blockstore_flush.cpp

@@ -35,24 +35,14 @@ journal_flusher_co::journal_flusher_co()
     {
         bs->live = true;
         if (data->res != data->iov.iov_len)
-        {
-            throw std::runtime_error(
-                "data read operation failed during flush ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-                "). can't continue, sorry :-("
-            );
-        }
+            bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len);
         wait_count--;
     };
     simple_callback_w = [this](ring_data_t* data)
     {
         bs->live = true;
         if (data->res != data->iov.iov_len)
-        {
-            throw std::runtime_error(
-                "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-                "). state "+std::to_string(wait_state)+". in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-            );
-        }
+            bs->disk_error_abort("write operation during flush", data->res, data->iov.iov_len);
         wait_count--;
     };
 }
@@ -87,7 +77,7 @@ void journal_flusher_t::loop()
             cur_flusher_count--;
         }
     }
-    for (int i = 0; (active_flushers > 0 || dequeuing) && i < cur_flusher_count; i++)
+    for (int i = 0; (active_flushers > 0 || dequeuing || trim_wanted > 0) && i < cur_flusher_count; i++)
         co[i].loop();
 }
 
@@ -172,7 +162,8 @@ void journal_flusher_t::mark_trim_possible()
     if (trim_wanted > 0)
     {
         dequeuing = true;
-        journal_trim_counter++;
+        if (!journal_trim_counter)
+            journal_trim_counter = journal_trim_interval;
         bs->ringloop->wakeup();
     }
 }
@@ -306,6 +297,8 @@ bool journal_flusher_co::loop()
         goto resume_20;
     else if (wait_state == 21)
         goto resume_21;
+    else if (wait_state == 22)
+        goto resume_22;
 resume_0:
     if (flusher->flush_queue.size() < flusher->min_flusher_count && !flusher->trim_wanted ||
         !flusher->flush_queue.size() || !flusher->dequeuing)
@@ -511,6 +504,13 @@ resume_1:
             );
             wait_count++;
         }
+        // Wait for data writes before fsyncing it
+    resume_22:
+        if (wait_count > 0)
+        {
+            wait_state = 22;
+            return false;
+        }
         // Sync data before writing metadata
     resume_16:
     resume_17:
@@ -521,7 +521,7 @@ resume_1:
             return false;
         }
     resume_5:
-        // And metadata writes, but only after data writes complete
+        // Submit metadata writes, but only when data is written and fsynced
         if (!bs->inmemory_meta && meta_new.it->second.state == 0 || wait_count > 0)
         {
             // metadata sector is still being read or data is still being written, wait for it
@@ -615,7 +615,12 @@ resume_1:
         }
         for (it = v.begin(); it != v.end(); it++)
         {
-            free(it->buf);
+            // Free it if it's not taken from the journal
+            if (it->buf && (!bs->journal.inmemory || it->buf < bs->journal.buffer ||
+                it->buf >= (uint8_t*)bs->journal.buffer + bs->journal.len))
+            {
+                free(it->buf);
+            }
         }
         v.clear();
         // And sync metadata (in batches - not per each operation!)
@@ -760,16 +765,17 @@ bool journal_flusher_co::scan_dirty(int wait_base)
                     {
                         submit_offset = dirty_it->second.location + offset - dirty_it->second.offset;
                         submit_len = it == v.end() || it->offset >= end_offset ? end_offset-offset : it->offset-offset;
-                        it = v.insert(it, (copy_buffer_t){ .offset = offset, .len = submit_len, .buf = memalign_or_die(MEM_ALIGNMENT, submit_len) });
+                        it = v.insert(it, (copy_buffer_t){ .offset = offset, .len = submit_len });
                         copy_count++;
                         if (bs->journal.inmemory)
                         {
-                            // Take it from memory
-                            memcpy(it->buf, (uint8_t*)bs->journal.buffer + submit_offset, submit_len);
+                            // Take it from memory, don't copy it
+                            it->buf = (uint8_t*)bs->journal.buffer + submit_offset;
                         }
                         else
                         {
                             // Read it from disk
+                            it->buf = memalign_or_die(MEM_ALIGNMENT, submit_len);
                             await_sqe(0);
                             data->iov = (struct iovec){ it->buf, (size_t)submit_len };
                             data->callback = simple_callback_r;

src/blockstore_impl.cpp

@@ -107,7 +107,7 @@ void blockstore_impl_t::loop()
         // has_writes == 0 - no writes before the current queue item
         // has_writes == 1 - some writes in progress
         // has_writes == 2 - tried to submit some writes, but failed
-        int has_writes = 0, op_idx = 0, new_idx = 0, done_lists = 0;
+        int has_writes = 0, op_idx = 0, new_idx = 0;
         for (; op_idx < submit_queue.size(); op_idx++, new_idx++)
         {
             auto op = submit_queue[op_idx];
@@ -188,16 +188,12 @@ void blockstore_impl_t::loop()
             else if (op->opcode == BS_OP_LIST)
             {
                 // LIST doesn't have to be blocked by previous modifications
-                // But don't do a lot of LISTs at once, because they're blocking and potentially slow
-                if (single_tick_list_limit <= 0 || done_lists < single_tick_list_limit)
-                {
-                    process_list(op);
-                    done_lists++;
-                    wr_st = 2;
-                }
+                process_list(op);
+                wr_st = 2;
             }
             if (wr_st == 2)
             {
+                submit_queue[op_idx] = NULL;
                 new_idx--;
             }
             if (wr_st == 0)
@@ -306,17 +302,6 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
             // do not submit
 #ifdef BLOCKSTORE_DEBUG
             printf("Still waiting for a journal buffer\n");
-#endif
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_FREE)
-    {
-        if (!data_alloc->get_free_count() && flusher->is_active())
-        {
-#ifdef BLOCKSTORE_DEBUG
-            printf("Still waiting for free space on the data device\n");
 #endif
             return;
         }
@@ -598,7 +583,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                         replace_stable(dirty_it->first.oid, 0, clean_stable_count, stable_count, stable);
                     }
                 }
-                else if (IS_STABLE(dirty_it->second.state))
+                else if (IS_STABLE(dirty_it->second.state) || (dirty_it->second.state & BS_ST_INSTANT))
                 {
                     // First try to replace a clean stable version in the first part of the list
                     if (!replace_stable(dirty_it->first.oid, dirty_it->first.version, 0, clean_stable_count, stable))
@@ -687,3 +672,16 @@ void blockstore_impl_t::dump_diagnostics()
     journal.dump_diagnostics();
     flusher->dump_diagnostics();
 }
+
+void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expected)
+{
+    if (retval == -EAGAIN)
+    {
+        fprintf(stderr, "EAGAIN error received from a disk %s during flush."
+            " It must never happen with io_uring and indicates a kernel bug."
+            " Please upgrade your kernel. Aborting.\n", op);
+        exit(1);
+    }
+    fprintf(stderr, "Disk %s failed: result is %d, expected %d. Can't continue, sorry :-(\n", op, retval, expected);
+    exit(1);
+}

src/blockstore_impl.h

@@ -160,12 +160,11 @@ struct __attribute__((__packed__)) dirty_entry
 #define WAIT_JOURNAL 3
 // Suspend operation until the next journal sector buffer is free
 #define WAIT_JOURNAL_BUFFER 4
-// Suspend operation until there is some free space on the data device
-#define WAIT_FREE 5
 
 struct fulfill_read_t
 {
     uint64_t offset, len;
+    uint64_t journal_sector; // sector+1 if used and !journal.inmemory, otherwise 0
 };
 
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
@@ -241,8 +240,6 @@ class blockstore_impl_t
     int throttle_target_parallelism = 1;
     // Minimum difference in microseconds between target and real execution times to throttle the response
     int throttle_threshold_us = 50;
-    // Maximum number of LIST operations to be processed between
-    int single_tick_list_limit = 1;
     /******* END OF OPTIONS *******/
 
     struct ring_consumer_t ring_consumer;
@@ -293,6 +290,7 @@ class blockstore_impl_t
     // Journaling
     void prepare_journal_sector_write(int sector, blockstore_op_t *op);
     void handle_journal_write(ring_data_t *data, uint64_t flush_id);
+    void disk_error_abort(const char *op, int retval, int expected);
 
     // Asynchronous init
     int initialized;
@@ -305,7 +303,7 @@ class blockstore_impl_t
     // Read
     int dequeue_read(blockstore_op_t *read_op);
     int fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-        uint32_t item_state, uint64_t item_version, uint64_t item_location);
+        uint32_t item_state, uint64_t item_version, uint64_t item_location, uint64_t journal_sector);
     int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
         uint32_t item_state, uint64_t item_version);
     void handle_read_event(ring_data_t *data, blockstore_op_t *op);

src/blockstore_init.cpp

@@ -48,14 +48,12 @@ void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num)
 
 int blockstore_init_meta::loop()
 {
-    if (wait_state == 1)
-        goto resume_1;
-    else if (wait_state == 2)
-        goto resume_2;
-    else if (wait_state == 3)
-        goto resume_3;
-    else if (wait_state == 4)
-        goto resume_4;
+    if (wait_state == 1)      goto resume_1;
+    else if (wait_state == 2) goto resume_2;
+    else if (wait_state == 3) goto resume_3;
+    else if (wait_state == 4) goto resume_4;
+    else if (wait_state == 5) goto resume_5;
+    else if (wait_state == 6) goto resume_6;
     printf("Reading blockstore metadata\n");
     if (bs->inmemory_meta)
         metadata_buffer = bs->metadata_buffer;
@@ -140,6 +138,7 @@ resume_1:
     // Skip superblock
     md_offset = bs->dsk.meta_block_size;
     next_offset = md_offset;
+    entries_per_block = bs->dsk.meta_block_size / bs->dsk.clean_entry_size;
     // Read the rest of the metadata
 resume_2:
     if (next_offset < bs->dsk.meta_len && submitted == 0)
@@ -179,17 +178,15 @@ resume_2:
         if (bufs[i].state == INIT_META_READ_DONE)
         {
             // Handle result
-            unsigned entries_per_block = bs->dsk.meta_block_size / bs->dsk.clean_entry_size;
             bool changed = false;
             for (uint64_t sector = 0; sector < bufs[i].size; sector += bs->dsk.meta_block_size)
             {
                 // handle <count> entries
-                changed = changed || handle_entries(
-                    bufs[i].buf + sector, entries_per_block,
-                    ((bufs[i].offset + sector - md_offset) / bs->dsk.meta_block_size) * entries_per_block
-                );
+                if (handle_meta_block(bufs[i].buf + sector, entries_per_block,
+                    ((bufs[i].offset + sector - md_offset) / bs->dsk.meta_block_size) * entries_per_block))
+                    changed = true;
             }
-            if (changed && !bs->inmemory_meta)
+            if (changed && !bs->inmemory_meta && !bs->readonly)
             {
                 // write the modified buffer back
                 GET_SQE();
@@ -211,6 +208,43 @@ resume_2:
         wait_state = 2;
         return 1;
     }
+    if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly)
+    {
+        // we have to zero out additional entries
+        for (i = 0; i < entries_to_zero.size(); )
+        {
+            next_offset = entries_to_zero[i]/entries_per_block;
+            for (j = i; j < entries_to_zero.size() && entries_to_zero[j]/entries_per_block == next_offset; j++) {}
+            GET_SQE();
+            data->iov = { metadata_buffer, bs->dsk.meta_block_size };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+            my_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
+            submitted++;
+resume_5:
+            if (submitted > 0)
+            {
+                wait_state = 5;
+                return 1;
+            }
+            for (; i < j; i++)
+            {
+                uint64_t pos = (entries_to_zero[i] % entries_per_block);
+                memset((uint8_t*)metadata_buffer + pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size);
+            }
+            GET_SQE();
+            data->iov = { metadata_buffer, bs->dsk.meta_block_size };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+            my_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
+            submitted++;
+resume_6:
+            if (submitted > 0)
+            {
+                wait_state = 6;
+                return 1;
+            }
+        }
+        entries_to_zero.clear();
+    }
     // metadata read finished
     printf("Metadata entries loaded: %lu, free blocks: %lu / %lu\n", entries_loaded, bs->data_alloc->get_free_count(), bs->dsk.block_count);
     if (!bs->inmemory_meta)
@@ -236,10 +270,13 @@ resume_2:
     return 0;
 }
 
-bool blockstore_init_meta::handle_entries(uint8_t *buf, uint64_t count, uint64_t done_cnt)
+bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_block, uint64_t done_cnt)
 {
     bool updated = false;
-    for (uint64_t i = 0; i < count; i++)
+    uint64_t max_i = entries_per_block;
+    if (max_i > bs->dsk.block_count-done_cnt)
+        max_i = bs->dsk.block_count-done_cnt;
+    for (uint64_t i = 0; i < max_i; i++)
     {
         clean_disk_entry *entry = (clean_disk_entry*)(buf + i*bs->dsk.clean_entry_size);
         if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size)
@@ -255,17 +292,35 @@ bool blockstore_init_meta::handle_entries(uint8_t *buf, uint64_t count, uint64_t
                 if (clean_it != clean_db.end())
                 {
                     // free the previous block
-                    // here we have to zero out the entry because otherwise we'll hit
+                    // here we have to zero out the previous entry because otherwise we'll hit
                     // "tried to overwrite non-zero metadata entry" later
-                    updated = true;
-                    memset(entry, 0, bs->dsk.clean_entry_size);
+                    uint64_t old_clean_loc = clean_it->second.location >> bs->dsk.block_order;
+                    if (bs->inmemory_meta)
+                    {
+                        uint64_t sector = (old_clean_loc / entries_per_block) * bs->dsk.meta_block_size;
+                        uint64_t pos = (old_clean_loc % entries_per_block);
+                        clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)bs->metadata_buffer + sector + pos*bs->dsk.clean_entry_size);
+                        memset(old_entry, 0, bs->dsk.clean_entry_size);
+                    }
+                    else if (old_clean_loc >= done_cnt)
+                    {
+                        updated = true;
+                        uint64_t sector = ((old_clean_loc - done_cnt) / entries_per_block) * bs->dsk.meta_block_size;
+                        uint64_t pos = (old_clean_loc % entries_per_block);
+                        clean_disk_entry *old_entry = (clean_disk_entry*)(buf + sector + pos*bs->dsk.clean_entry_size);
+                        memset(old_entry, 0, bs->dsk.clean_entry_size);
+                    }
+                    else
+                    {
+                        entries_to_zero.push_back(clean_it->second.location >> bs->dsk.block_order);
+                    }
 #ifdef BLOCKSTORE_DEBUG
                     printf("Free block %lu from %lx:%lx v%lu (new location is %lu)\n",
-                        clean_it->second.location >> bs->dsk.block_order,
+                        old_clean_loc,
                         clean_it->first.inode, clean_it->first.stripe, clean_it->second.version,
                         done_cnt+i);
 #endif
-                    bs->data_alloc->set(clean_it->second.location >> bs->dsk.block_order, false);
+                    bs->data_alloc->set(old_clean_loc, false);
                 }
                 else
                 {

src/blockstore_init.h

@@ -24,7 +24,10 @@ class blockstore_init_meta
     uint64_t md_offset = 0;
     uint64_t next_offset = 0;
     uint64_t entries_loaded = 0;
-    bool handle_entries(uint8_t *buf, uint64_t count, uint64_t done_cnt);
+    unsigned entries_per_block = 0;
+    int i = 0, j = 0;
+    std::vector<uint64_t> entries_to_zero;
+    bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
     void handle_event(ring_data_t *data, int buf_num);
 public:
     blockstore_init_meta(blockstore_impl_t *bs);

src/blockstore_journal.cpp

@@ -198,10 +198,7 @@ void blockstore_impl_t::handle_journal_write(ring_data_t *data, uint64_t flush_i
     if (data->res != data->iov.iov_len)
     {
         // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
-        throw std::runtime_error(
-            "journal write failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-            "). in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-        );
+        disk_error_abort("journal write", data->res, data->iov.iov_len);
     }
     auto fl_it = journal.flushing_ops.upper_bound((pending_journaling_t){ .flush_id = flush_id });
     if (fl_it != journal.flushing_ops.end() && fl_it->flush_id == flush_id)

src/blockstore_journal.h

@@ -16,6 +16,7 @@
 // FIXME: This value should be dynamic i.e. Blockstore ideally shouldn't allow
 // writing more than can be stabilized afterwards
 #define JOURNAL_STABILIZE_RESERVATION 65536
+#define JOURNAL_INSTANT_RESERVATION 131072
 
 // Journal entries
 // Journal entries are linked to each other by their crc32 value

src/blockstore_read.cpp

@@ -42,7 +42,7 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
 
 // FIXME I've seen a bug here so I want some tests
 int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-    uint32_t item_state, uint64_t item_version, uint64_t item_location)
+    uint32_t item_state, uint64_t item_version, uint64_t item_location, uint64_t journal_sector)
 {
     uint32_t cur_start = item_start;
     if (cur_start < read_op->offset + read_op->len && item_end > read_op->offset)
@@ -72,6 +72,7 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, uint64_t &fulfille
                 fulfill_read_t el = {
                     .offset = cur_start,
                     .len = it == PRIV(read_op)->read_vec.end() || it->offset >= item_end ? item_end-cur_start : it->offset-cur_start,
+                    .journal_sector = journal_sector,
                 };
                 it = PRIV(read_op)->read_vec.insert(it, el);
                 if (!fulfill_read_push(read_op,
@@ -138,7 +139,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         while (dirty_it->first.oid == read_op->oid)
         {
             dirty_entry& dirty = dirty_it->second;
-            bool version_ok = read_op->version >= dirty_it->first.version;
+            bool version_ok = !IS_IN_FLIGHT(dirty.state) && read_op->version >= dirty_it->first.version;
             if (IS_SYNCED(dirty.state))
             {
                 if (!version_ok && read_op->version != 0)
@@ -156,8 +157,10 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
                         memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
                     }
                 }
+                // If inmemory_journal is false, journal trim will have to wait until the read is completed
                 if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
-                    dirty.state, dirty_it->first.version, dirty.location + (IS_JOURNAL(dirty.state) ? 0 : dirty.offset)))
+                    dirty.state, dirty_it->first.version, dirty.location + (IS_JOURNAL(dirty.state) ? 0 : dirty.offset),
+                    (IS_JOURNAL(dirty.state) ? dirty.journal_sector+1 : 0)))
                 {
                     // need to wait. undo added requests, don't dequeue op
                     PRIV(read_op)->read_vec.clear();
@@ -171,7 +174,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
             dirty_it--;
         }
     }
-    if (clean_it != clean_db.end())
+    if (clean_found)
     {
         if (!result_version)
         {
@@ -186,7 +189,8 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         {
             if (!dsk.clean_entry_bitmap_size)
             {
-                if (!fulfill_read(read_op, fulfilled, 0, dsk.data_block_size, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_it->second.location))
+                if (!fulfill_read(read_op, fulfilled, 0, dsk.data_block_size,
+                    (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_it->second.location, 0))
                 {
                     // need to wait. undo added requests, don't dequeue op
                     PRIV(read_op)->read_vec.clear();
@@ -207,7 +211,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
                     {
                         // fill with zeroes
                         assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
-                            bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0));
+                            bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0));
                     }
                     bmp_start = bmp_end;
                     while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
@@ -218,7 +222,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
                     {
                         if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
                             bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0,
-                            clean_it->second.location + bmp_start * dsk.bitmap_granularity))
+                            clean_it->second.location + bmp_start * dsk.bitmap_granularity, 0))
                         {
                             // need to wait. undo added requests, don't dequeue op
                             PRIV(read_op)->read_vec.clear();
@@ -233,7 +237,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     else if (fulfilled < read_op->len)
     {
         // fill remaining parts with zeroes
-        assert(fulfill_read(read_op, fulfilled, 0, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0));
+        assert(fulfill_read(read_op, fulfilled, 0, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0));
     }
     assert(fulfilled == read_op->len);
     read_op->version = result_version;
@@ -249,6 +253,15 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
         FINISH_OP(read_op);
         return 2;
     }
+    if (!journal.inmemory)
+    {
+        // Journal trim has to wait until the read is completed - record journal sector usage
+        for (auto & rv: PRIV(read_op)->read_vec)
+        {
+            if (rv.journal_sector)
+                journal.used_sectors[rv.journal_sector-1]++;
+        }
+    }
     read_op->retval = 0;
     return 2;
 }
@@ -264,6 +277,22 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
     }
     if (PRIV(op)->pending_ops == 0)
     {
+        if (!journal.inmemory)
+        {
+            // Release journal sector usage
+            for (auto & rv: PRIV(op)->read_vec)
+            {
+                if (rv.journal_sector)
+                {
+                    auto used = --journal.used_sectors[rv.journal_sector-1];
+                    if (used == 0)
+                    {
+                        journal.used_sectors.erase(rv.journal_sector-1);
+                        flusher->mark_trim_possible();
+                    }
+                }
+            }
+        }
         if (op->retval == 0)
             op->retval = op->len;
         FINISH_OP(op);

src/blockstore_rollback.cpp

@@ -127,7 +127,6 @@ resume_4:
     {
         mark_rolled_back(*v);
     }
-    flusher->mark_trim_possible();
     // Acknowledge op
     op->retval = 0;
     FINISH_OP(op);
@@ -222,7 +221,7 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
 #endif
             data_alloc->set(dirty_it->second.location >> dsk.block_order, false);
         }
-        int used = --journal.used_sectors[dirty_it->second.journal_sector];
+        auto used = --journal.used_sectors[dirty_it->second.journal_sector];
 #ifdef BLOCKSTORE_DEBUG
         printf(
             "remove usage of journal offset %08lx by %lx:%lx v%lu (%d refs)\n", dirty_it->second.journal_sector,
@@ -232,6 +231,7 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
         if (used == 0)
         {
             journal.used_sectors.erase(dirty_it->second.journal_sector);
+            flusher->mark_trim_possible();
         }
         if (dsk.clean_entry_bitmap_size > sizeof(void*))
         {

src/blockstore_write.cpp

@@ -89,6 +89,9 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         else
         {
             // Invalid version requested
+#ifdef BLOCKSTORE_DEBUG
+            printf("Write %lx:%lx v%lu requested, but we already have v%lu\n", op->oid.inode, op->oid.stripe, op->version, version);
+#endif
             op->retval = -EEXIST;
             if (!is_del && dsk.clean_entry_bitmap_size > sizeof(void*))
             {
@@ -115,8 +118,8 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
     else if (!wait_del)
         printf("Write %lx:%lx v%lu offset=%u len=%u\n", op->oid.inode, op->oid.stripe, op->version, op->offset, op->len);
 #endif
-    // FIXME No strict need to add it into dirty_db here, it's just left
-    // from the previous implementation where reads waited for writes
+    // No strict need to add it into dirty_db here except maybe for listings to return
+    // correct data when there are inflight operations in the queue
     uint32_t state;
     if (is_del)
         state = BS_ST_DELETE | BS_ST_IN_FLIGHT;
@@ -139,7 +142,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             uint8_t *bmp_ptr = (uint8_t*)(dsk.clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp);
             uint32_t bit = op->offset/dsk.bitmap_granularity;
             uint32_t bits_left = op->len/dsk.bitmap_granularity;
-            while (!(bit % 8) && bits_left > 8)
+            while (!(bit % 8) && bits_left >= 8)
             {
                 // Copy bytes
                 bmp_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];
@@ -182,8 +185,15 @@ void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_
     bool found = false;
     for (auto other_op: submit_queue)
     {
-        if (!found && other_op == op)
+        if (!other_op)
+        {
+            // freed operations during submitting are zeroed
+        }
+        else if (other_op == op)
+        {
+            // <op> may be present in queue multiple times due to moving operations in submit_queue
             found = true;
+        }
         else if (found && other_op->oid == op->oid &&
             (other_op->opcode == BS_OP_WRITE || other_op->opcode == BS_OP_WRITE_STABLE))
         {
@@ -251,7 +261,8 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         blockstore_journal_check_t space_check(this);
         if (!space_check.check_available(op, unsynced_big_write_count + 1,
-            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, JOURNAL_STABILIZE_RESERVATION))
+            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size,
+            (dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION))
         {
             return 0;
         }
@@ -260,12 +271,6 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         if (loc == UINT64_MAX)
         {
             // no space
-            if (flusher->is_active())
-            {
-                // hope that some space will be available after flush
-                PRIV(op)->wait_for = WAIT_FREE;
-                return 0;
-            }
             cancel_all_writes(op, dirty_it, -ENOSPC);
             return 2;
         }
@@ -337,7 +342,8 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             !space_check.check_available(op, unsynced_big_write_count,
                 sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0)
             || !space_check.check_available(op, 1,
-                sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size, op->len + JOURNAL_STABILIZE_RESERVATION))
+                sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size,
+                op->len + ((dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION)))
         {
             return 0;
         }
@@ -448,12 +454,19 @@ int blockstore_impl_t::continue_write(blockstore_op_t *op)
 resume_2:
     // Only for the immediate_commit mode: prepare and submit big_write journal entry
     {
-        BS_SUBMIT_CHECK_SQES(1);
         auto dirty_it = dirty_db.find((obj_ver_id){
             .oid = op->oid,
             .version = op->version,
         });
         assert(dirty_it != dirty_db.end());
+        blockstore_journal_check_t space_check(this);
+        if (!space_check.check_available(op, 1,
+            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size,
+            ((dirty_it->second.state & BS_ST_INSTANT) ? JOURNAL_INSTANT_RESERVATION : JOURNAL_STABILIZE_RESERVATION)))
+        {
+            return 0;
+        }
+        BS_SUBMIT_CHECK_SQES(1);
         journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
             journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
             sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size
@@ -585,10 +598,7 @@ void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *o
     if (data->res != data->iov.iov_len)
     {
         // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
-        throw std::runtime_error(
-            "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-            "). in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-        );
+        disk_error_abort("data write", data->res, data->iov.iov_len);
     }
     PRIV(op)->pending_ops--;
     assert(PRIV(op)->pending_ops >= 0);
@@ -643,7 +653,7 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
     });
     assert(dirty_it != dirty_db.end());
     blockstore_journal_check_t space_check(this);
-    if (!space_check.check_available(op, 1, sizeof(journal_entry_del), JOURNAL_STABILIZE_RESERVATION))
+    if (!space_check.check_available(op, 1, sizeof(journal_entry_del), JOURNAL_INSTANT_RESERVATION))
     {
         return 0;
     }

src/cli.cpp

@@ -76,6 +76,12 @@ static const char* help_text =
     "vitastor-cli alloc-osd\n"
     "  Allocate a new OSD number and reserve it by creating empty /osd/stats/<n> key.\n"
     "\n"
+    "vitastor-cli rm-osd [--force] [--allow-data-loss] [--dry-run] <osd_id> [osd_id...]\n"
+    "  Remove metadata and configuration for specified OSD(s) from etcd.\n"
+    "  Refuses to remove OSDs with data without --force and --allow-data-loss.\n"
+    "  With --dry-run only checks if deletion is possible without data loss and\n"
+    "  redundancy degradation.\n"
+    "\n"
     "Use vitastor-cli --help <command> for command details or vitastor-cli --help --all for all details.\n"
     "\n"
     "GLOBAL OPTIONS:\n"
@@ -95,43 +101,47 @@ static json11::Json::object parse_args(int narg, const char *args[])
     cfg["progress"] = "1";
     for (int i = 1; i < narg; i++)
     {
-        if (args[i][0] == '-' && args[i][1] == 'h')
+        if (args[i][0] == '-' && args[i][1] == 'h' && args[i][2] == 0)
         {
             cfg["help"] = "1";
         }
-        else if (args[i][0] == '-' && args[i][1] == 'l')
+        else if (args[i][0] == '-' && args[i][1] == 'l' && args[i][2] == 0)
         {
             cfg["long"] = "1";
         }
-        else if (args[i][0] == '-' && args[i][1] == 'n')
+        else if (args[i][0] == '-' && args[i][1] == 'n' && args[i][2] == 0)
         {
             cfg["count"] = args[++i];
         }
-        else if (args[i][0] == '-' && args[i][1] == 'p')
+        else if (args[i][0] == '-' && args[i][1] == 'p' && args[i][2] == 0)
         {
             cfg["pool"] = args[++i];
         }
-        else if (args[i][0] == '-' && args[i][1] == 's')
+        else if (args[i][0] == '-' && args[i][1] == 's' && args[i][2] == 0)
         {
             cfg["size"] = args[++i];
         }
-        else if (args[i][0] == '-' && args[i][1] == 'r')
+        else if (args[i][0] == '-' && args[i][1] == 'r' && args[i][2] == 0)
         {
             cfg["reverse"] = "1";
         }
-        else if (args[i][0] == '-' && args[i][1] == 'f')
+        else if (args[i][0] == '-' && args[i][1] == 'f' && args[i][2] == 0)
         {
             cfg["force"] = "1";
         }
         else if (args[i][0] == '-' && args[i][1] == '-')
         {
             const char *opt = args[i]+2;
-            cfg[opt] = i == narg-1 || !strcmp(opt, "json") || !strcmp(opt, "wait-list") ||
-                !strcmp(opt, "long") || !strcmp(opt, "del") || !strcmp(opt, "no-color") ||
+            cfg[opt] = i == narg-1 || !strcmp(opt, "json") ||
+                !strcmp(opt, "wait-list") || !strcmp(opt, "wait_list") ||
+                !strcmp(opt, "long") || !strcmp(opt, "del") ||
+                !strcmp(opt, "no-color") || !strcmp(opt, "no_color") ||
                 !strcmp(opt, "readonly") || !strcmp(opt, "readwrite") ||
                 !strcmp(opt, "force") || !strcmp(opt, "reverse") ||
+                !strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") ||
+                !strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
                 !strcmp(opt, "help") || !strcmp(opt, "all") ||
-                !strcmp(opt, "writers-stopped") && strcmp("1", args[i+1]) != 0
+                (!strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped")) && strcmp("1", args[i+1]) != 0
                 ? "1" : args[++i];
         }
         else
@@ -139,10 +149,6 @@ static json11::Json::object parse_args(int narg, const char *args[])
             cmd.push_back(std::string(args[i]));
         }
     }
-    if (cfg["help"].bool_value())
-    {
-        print_help(help_text, "vitastor-cli", cmd.size() ? cmd[0].string_value() : "", cfg["all"].bool_value());
-    }
     if (!cmd.size())
     {
         std::string exe(exe_name);
@@ -151,6 +157,10 @@ static json11::Json::object parse_args(int narg, const char *args[])
             cmd.push_back("rm-data");
         }
     }
+    if (!cmd.size() || cfg["help"].bool_value())
+    {
+        print_help(help_text, "vitastor-cli", cmd.size() ? cmd[0].string_value() : "", cfg["all"].bool_value());
+    }
     cfg["command"] = cmd;
     return cfg;
 }
@@ -225,6 +235,16 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
         // Delete inode data
         action_cb = p->start_rm_data(cfg);
     }
+    else if (cmd[0] == "rm-osd")
+    {
+        // Delete OSD metadata from etcd
+        if (cmd.size() > 1)
+        {
+            cmd.erase(cmd.begin(), cmd.begin()+1);
+            cfg["osd_id"] = cmd;
+        }
+        action_cb = p->start_rm_osd(cfg);
+    }
     else if (cmd[0] == "merge-data")
     {
         // Merge layer data without affecting metadata

src/cli.h

@@ -45,7 +45,7 @@ public:
     cli_result_t etcd_err;
     json11::Json etcd_result;
 
-    void parse_config(json11::Json cfg);
+    void parse_config(json11::Json::object & cfg);
 
     void change_parent(inode_t cur, inode_t new_parent, cli_result_t *result);
     inode_config_t* get_inode_cfg(const std::string & name);
@@ -64,6 +64,7 @@ public:
     std::function<bool(cli_result_t &)> start_merge(json11::Json);
     std::function<bool(cli_result_t &)> start_flatten(json11::Json);
     std::function<bool(cli_result_t &)> start_rm(json11::Json);
+    std::function<bool(cli_result_t &)> start_rm_osd(json11::Json cfg);
     std::function<bool(cli_result_t &)> start_alloc_osd(json11::Json cfg);
 
     // Should be called like loop_and_wait(start_status(), <completion callback>)

src/cli_common.cpp

@@ -100,9 +100,20 @@ inode_config_t* cli_tool_t::get_inode_cfg(const std::string & name)
     return NULL;
 }
 
-void cli_tool_t::parse_config(json11::Json cfg)
+void cli_tool_t::parse_config(json11::Json::object & cfg)
 {
-    color = !cfg["no-color"].bool_value();
+    for (auto kv_it = cfg.begin(); kv_it != cfg.end();)
+    {
+        // Translate all options with - to _
+        if (kv_it->first.find("-") != std::string::npos)
+        {
+            cfg[str_replace(kv_it->first, "-", "_")] = kv_it->second;
+            cfg.erase(kv_it++);
+        }
+        else
+            kv_it++;
+    }
+    color = !cfg["no_color"].bool_value();
     json_output = cfg["json"].bool_value();
     iodepth = cfg["iodepth"].uint64_value();
     if (!iodepth)
@@ -112,7 +123,7 @@ void cli_tool_t::parse_config(json11::Json cfg)
         parallel_osds = 4;
     log_level = cfg["log_level"].int64_value();
     progress = cfg["progress"].uint64_value() ? true : false;
-    list_first = cfg["wait-list"].uint64_value() ? true : false;
+    list_first = cfg["wait_list"].uint64_value() ? true : false;
 }
 
 struct cli_result_looper_t

src/cli_create.cpp

@@ -517,7 +517,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
     image_creator->force_size = cfg["force_size"].bool_value();
     if (cfg["image_meta"].is_object())
     {
-        image_creator->new_meta = cfg["image-meta"];
+        image_creator->new_meta = cfg["image_meta"];
     }
     if (cfg["snapshot"].string_value() != "")
     {

src/cli_flatten.cpp

@@ -133,7 +133,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_flatten(json11::Json cfg)
     auto flattener = new snap_flattener_t();
     flattener->parent = this;
     flattener->target_name = cfg["image"].string_value();
-    flattener->fsync_interval = cfg["fsync-interval"].uint64_value();
+    flattener->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!flattener->fsync_interval)
         flattener->fsync_interval = 128;
     if (!cfg["cas"].is_null())

src/cli_merge.cpp

@@ -631,8 +631,8 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_merge(json11::Json cfg)
     merger->from_name = cfg["from"].string_value();
     merger->to_name = cfg["to"].string_value();
     merger->target_name = cfg["target"].string_value();
-    merger->delete_source = cfg["delete-source"].string_value() != "";
-    merger->fsync_interval = cfg["fsync-interval"].uint64_value();
+    merger->delete_source = cfg["delete_source"].string_value() != "";
+    merger->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!merger->fsync_interval)
         merger->fsync_interval = 128;
     if (!cfg["cas"].is_null())

src/cli_modify.cpp

@@ -236,7 +236,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_modify(json11::Json cfg)
     changer->force = cfg["force"].bool_value();
     changer->set_readonly = cfg["readonly"].bool_value();
     changer->set_readwrite = cfg["readwrite"].bool_value();
-    changer->fsync_interval = cfg["fsync-interval"].uint64_value();
+    changer->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!changer->fsync_interval)
         changer->fsync_interval = 128;
     // FIXME Check that the image doesn't have children when shrinking

src/cli_rm.cpp

@@ -639,7 +639,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm(json11::Json cfg)
     snap_remover->parent = this;
     snap_remover->from_name = cfg["from"].string_value();
     snap_remover->to_name = cfg["to"].string_value();
-    snap_remover->fsync_interval = cfg["fsync-interval"].uint64_value();
+    snap_remover->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!snap_remover->fsync_interval)
         snap_remover->fsync_interval = 128;
     if (!cfg["cas"].is_null())

src/cli_rm_data.cpp

@@ -218,7 +218,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm_data(json11::Json cfg)
         remover->inode = (remover->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (((uint64_t)remover->pool_id) << (64-POOL_ID_BITS));
     }
     remover->pool_id = INODE_POOL(remover->inode);
-    remover->min_offset = cfg["min-offset"].uint64_value();
+    remover->min_offset = cfg["min_offset"].uint64_value();
     return [remover](cli_result_t & result)
     {
         remover->loop();

src/cli_rm_osd.cpp

@@ -0,0 +1,491 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include <ctype.h>
+#include "cli.h"
+#include "cluster_client.h"
+#include "str_util.h"
+#include "epoll_manager.h"
+
+#include <algorithm>
+
+// Delete OSD metadata from etcd
+struct rm_osd_t
+{
+    cli_tool_t *parent;
+
+    bool dry_run, force_warning, force_dataloss;
+    uint64_t etcd_tx_retry_ms = 500;
+    uint64_t etcd_tx_retries = 10000;
+    std::vector<uint64_t> osd_ids;
+
+    int state = 0;
+    cli_result_t result;
+
+    std::set<uint64_t> to_remove;
+    std::set<uint64_t> to_restart;
+    json11::Json::array pool_effects;
+    json11::Json::array history_updates, history_checks;
+    json11::Json new_pgs, new_clean_pgs;
+    uint64_t new_pgs_mod_rev, new_clean_pgs_mod_rev;
+    uint64_t cur_retry = 0;
+    uint64_t retry_wait = 0;
+    bool is_warning, is_dataloss;
+
+    bool is_done()
+    {
+        return state == 100;
+    }
+
+    void loop()
+    {
+        if (state == 1)
+            goto resume_1;
+        else if (state == 2)
+            goto resume_2;
+        else if (state == 3)
+            goto resume_3;
+        else if (state == 4)
+            goto resume_4;
+        if (!osd_ids.size())
+        {
+            result = (cli_result_t){ .err = EINVAL, .text = "OSD numbers are not specified" };
+            state = 100;
+            return;
+        }
+        for (auto osd_id: osd_ids)
+        {
+            if (!osd_id)
+            {
+                result = (cli_result_t){ .err = EINVAL, .text = "OSD number can't be zero" };
+                state = 100;
+                return;
+            }
+            to_remove.insert(osd_id);
+        }
+        // Check if OSDs are still used in data distribution
+        is_warning = is_dataloss = false;
+        for (auto & pp: parent->cli->st_cli.pool_config)
+        {
+            // Will OSD deletion make pool incomplete / down / degraded?
+            bool pool_incomplete = false, pool_down = false, pool_degraded = false;
+            bool hist_incomplete = false, hist_degraded = false;
+            auto & pool_cfg = pp.second;
+            uint64_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
+            for (auto & pgp: pool_cfg.pg_config)
+            {
+                auto & pg_cfg = pgp.second;
+                int pg_cursize = 0, pg_rm = 0;
+                for (auto pg_osd: pg_cfg.target_set)
+                {
+                    if (pg_osd != 0)
+                    {
+                        pg_cursize++;
+                        if (to_remove.find(pg_osd) != to_remove.end())
+                            pg_rm++;
+                    }
+                }
+                for (auto & hist_item: pg_cfg.target_history)
+                {
+                    int hist_size = 0, hist_rm = 0;
+                    for (auto & old_osd: hist_item)
+                    {
+                        if (old_osd != 0)
+                        {
+                            hist_size++;
+                            if (to_remove.find(old_osd) != to_remove.end())
+                                hist_rm++;
+                        }
+                    }
+                    if (hist_rm > 0)
+                    {
+                        hist_degraded = true;
+                        if (hist_size-hist_rm == 0)
+                            pool_incomplete = true;
+                        else if (hist_size-hist_rm < pg_data_size)
+                            hist_incomplete = true;
+                    }
+                }
+                if (pg_rm > 0)
+                {
+                    pool_degraded = true;
+                    if (pg_cursize-pg_rm < pg_data_size)
+                        pool_incomplete = true;
+                    else if (pg_cursize-pg_rm < pool_cfg.pg_minsize)
+                        pool_down = true;
+                }
+            }
+            if (pool_incomplete || pool_down || pool_degraded || hist_incomplete || hist_degraded)
+            {
+                pool_effects.push_back(json11::Json::object {
+                    { "pool_id", (uint64_t)pool_cfg.id },
+                    { "pool_name", pool_cfg.name },
+                    { "effect", (pool_incomplete
+                        ? "incomplete"
+                        : (hist_incomplete
+                            ? "has_incomplete"
+                            : (pool_down
+                                ? "offline"
+                                : (pool_degraded
+                                    ? "degraded"
+                                    : (hist_degraded ? "has_degraded" : "?")
+                                )
+                            )
+                        )
+                    ) },
+                });
+                is_warning = true;
+                if (pool_incomplete || hist_incomplete)
+                    is_dataloss = true;
+            }
+        }
+        result.data = json11::Json::object {
+            { "osd_ids", osd_ids },
+            { "pool_errors", pool_effects },
+        };
+        if (is_dataloss || is_warning || dry_run)
+        {
+            std::string error;
+            for (auto & e: pool_effects)
+            {
+                error += "Pool "+e["pool_name"].string_value()+" (ID "+e["pool_id"].as_string()+") will have "+(
+                    e["effect"] == "has_incomplete"
+                        ? std::string("INCOMPLETE objects (DATA LOSS)")
+                        : (e["effect"] == "incomplete"
+                            ? std::string("INCOMPLETE PGs (DATA LOSS)")
+                            : (e["effect"] == "has_degraded"
+                                ? std::string("DEGRADED objects")
+                                : strtoupper(e["effect"].string_value())+" PGs"))
+                )+" after deleting OSD(s).\n";
+            }
+            if (is_dataloss && !force_dataloss && !dry_run)
+                error += "OSDs not deleted. Please move data to other OSDs or bypass this check with --allow-data-loss if you know what you are doing.\n";
+            else if (is_warning && !force_warning && !dry_run)
+                error += "OSDs not deleted. Please move data to other OSDs or bypass this check with --force if you know what you are doing.\n";
+            else if (!is_dataloss && !is_warning && dry_run)
+                error += "OSDs can be deleted without data loss.\n";
+            result.text = error;
+            if (dry_run || is_dataloss && !force_dataloss || is_warning && !force_warning)
+            {
+                result.err = is_dataloss && !force_dataloss || is_warning && !force_warning ? EBUSY : 0;
+                state = 100;
+                return;
+            }
+        }
+        parent->etcd_txn(json11::Json::object { { "success", json11::Json::array {
+            json11::Json::object {
+                { "request_range", json11::Json::object {
+                    { "key", base64_encode(
+                        parent->cli->st_cli.etcd_prefix+"/config/pgs"
+                    ) },
+                } },
+            },
+            json11::Json::object {
+                { "request_range", json11::Json::object {
+                    { "key", base64_encode(
+                        parent->cli->st_cli.etcd_prefix+"/history/last_clean_pgs"
+                    ) },
+                } },
+            },
+        } } });
+    resume_4:
+        state = 4;
+        if (parent->waiting > 0)
+            return;
+        if (parent->etcd_err.err)
+        {
+            result = parent->etcd_err;
+            state = 100;
+            return;
+        }
+        {
+            auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+            new_pgs = remove_osds_from_pgs(kv);
+            new_pgs_mod_rev = kv.mod_revision;
+            kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][1]["response_range"]["kvs"][0]);
+            new_clean_pgs = remove_osds_from_pgs(kv);
+            new_clean_pgs_mod_rev = kv.mod_revision;
+        }
+        // Remove keys from etcd
+        {
+            json11::Json::array rm_items, rm_checks;
+            for (auto osd_id: osd_ids)
+            {
+                rm_items.push_back("/config/osd/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/stats/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/state/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/inodestats/"+std::to_string(osd_id));
+                rm_items.push_back("/osd/space/"+std::to_string(osd_id));
+            }
+            for (int i = 0; i < rm_items.size(); i++)
+            {
+                rm_items[i] = json11::Json::object {
+                    { "request_delete_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+rm_items[i].string_value()
+                        ) },
+                    } },
+                };
+            }
+            if (!new_pgs.is_null())
+            {
+                auto pgs_key = base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pgs");
+                rm_items.push_back(json11::Json::object {
+                    { "request_put", json11::Json::object {
+                        { "key", pgs_key },
+                        { "value", base64_encode(new_pgs.dump()) },
+                    } },
+                });
+                rm_checks.push_back(json11::Json::object {
+                    { "target", "MOD" },
+                    { "key", pgs_key },
+                    { "result", "LESS" },
+                    { "mod_revision", new_pgs_mod_rev+1 },
+                });
+            }
+            if (!new_clean_pgs.is_null())
+            {
+                auto pgs_key = base64_encode(parent->cli->st_cli.etcd_prefix+"/history/last_clean_pgs");
+                rm_items.push_back(json11::Json::object {
+                    { "request_put", json11::Json::object {
+                        { "key", pgs_key },
+                        { "value", base64_encode(new_clean_pgs.dump()) },
+                    } },
+                });
+                rm_checks.push_back(json11::Json::object {
+                    { "target", "MOD" },
+                    { "key", pgs_key },
+                    { "result", "LESS" },
+                    { "mod_revision", new_clean_pgs_mod_rev+1 },
+                });
+            }
+            parent->etcd_txn(json11::Json::object { { "success", rm_items }, { "checks", rm_checks } });
+        }
+    resume_1:
+        state = 1;
+        if (parent->waiting > 0)
+            return;
+        if (parent->etcd_err.err)
+        {
+            result = parent->etcd_err;
+            state = 100;
+            return;
+        }
+        // Remove old OSD from PG all_peers to prevent left_on_dead and from
+        // target_history to prevent INCOMPLETE if --allow-data-loss is specified
+        for (auto & rsp: parent->etcd_result["responses"].array_items())
+        {
+            if (rsp["response_delete_range"]["deleted"].uint64_value() > 0)
+            {
+                // Wait for mon_change_timeout before updating PG history, or the monitor's change will likely interfere with ours
+                retry_wait = parent->cli->merged_config["mon_change_timeout"].uint64_value();
+                if (!retry_wait)
+                    retry_wait = 1000;
+                retry_wait += etcd_tx_retry_ms;
+            }
+        }
+        while (1)
+        {
+    resume_2:
+            if (!remove_osds_from_history(2))
+                return;
+    resume_3:
+            state = 3;
+            if (parent->waiting > 0)
+                return;
+            if (parent->etcd_err.err)
+            {
+                result = parent->etcd_err;
+                state = 100;
+                return;
+            }
+            if (parent->etcd_result["succeeded"].bool_value())
+                break;
+            if ((++cur_retry) >= etcd_tx_retries)
+            {
+                result.err = EAGAIN;
+                result.text += "Failed to remove OSDs from PG history due to update conflicts."
+                    " Some PGs may remain left_on_dead or incomplete. Please retry later\n";
+                state = 100;
+                return;
+            }
+            retry_wait = etcd_tx_retry_ms;
+        }
+        std::string ids = "";
+        for (auto osd_id: osd_ids)
+        {
+            ids += (ids.size() ? ", " : "")+std::to_string(osd_id);
+        }
+        ids = (osd_ids.size() > 1 ? "OSDs " : "OSD ")+ids+(osd_ids.size() > 1 ? " are" : " is")+" removed from etcd";
+        state = 100;
+        result.text = (result.text != "" ? ids+"\n"+result.text : ids);
+        result.err = 0;
+    }
+
+    json11::Json remove_osds_from_pgs(const etcd_kv_t & kv)
+    {
+        if (kv.value.is_null())
+        {
+            return kv.value;
+        }
+        json11::Json::object new_pgs;
+        for (auto & pp: kv.value["items"].object_items())
+        {
+            if (pp.second.is_object())
+            {
+                json11::Json::object new_pool;
+                for (auto & pgp: pp.second.object_items())
+                {
+                    json11::Json::array osd_set;
+                    for (auto & osd_json: pgp.second["osd_set"].array_items())
+                    {
+                        uint64_t osd_num = osd_json.uint64_value();
+                        osd_set.push_back(osd_num == 0 || to_remove.find(osd_num) != to_remove.end() ? 0 : osd_num);
+                    }
+                    json11::Json::object new_pg = pgp.second.object_items();
+                    new_pg["osd_set"] = osd_set;
+                    new_pool[pgp.first] = new_pg;
+                }
+                new_pgs[pp.first] = new_pool;
+            }
+            else
+                new_pgs[pp.first] = pp.second;
+        }
+        auto res = kv.value.object_items();
+        res["items"] = new_pgs;
+        return res;
+    }
+
+    bool remove_osds_from_history(int base_state)
+    {
+        if (state == base_state+0)
+            goto resume_0;
+        history_updates.clear();
+        history_checks.clear();
+        for (auto & pp: parent->cli->st_cli.pool_config)
+        {
+            bool update_pg_history = false;
+            auto & pool_cfg = pp.second;
+            for (auto & pgp: pool_cfg.pg_config)
+            {
+                auto pg_num = pgp.first;
+                auto & pg_cfg = pgp.second;
+                for (int i = 0; i < pg_cfg.all_peers.size(); i++)
+                {
+                    if (to_remove.find(pg_cfg.all_peers[i]) != to_remove.end())
+                    {
+                        update_pg_history = true;
+                        pg_cfg.all_peers.erase(pg_cfg.all_peers.begin()+i, pg_cfg.all_peers.begin()+i+1);
+                        i--;
+                    }
+                }
+                for (int i = 0; i < pg_cfg.target_history.size(); i++)
+                {
+                    int hist_size = 0, hist_rm = 0;
+                    for (auto & old_osd: pg_cfg.target_history[i])
+                    {
+                        if (old_osd != 0)
+                        {
+                            hist_size++;
+                            if (to_remove.find(old_osd) != to_remove.end())
+                            {
+                                hist_rm++;
+                                old_osd = 0;
+                            }
+                        }
+                    }
+                    if (hist_rm > 0)
+                    {
+                        if (hist_size-hist_rm == 0)
+                        {
+                            pg_cfg.target_history.erase(pg_cfg.target_history.begin()+i, pg_cfg.target_history.begin()+i+1);
+                            i--;
+                        }
+                        update_pg_history = true;
+                    }
+                }
+                if (update_pg_history)
+                {
+                    std::string history_key = base64_encode(
+                        parent->cli->st_cli.etcd_prefix+"/pg/history/"+
+                        std::to_string(pool_cfg.id)+"/"+std::to_string(pg_num)
+                    );
+                    history_updates.push_back(json11::Json::object {
+                        { "request_put", json11::Json::object {
+                            { "key", history_key },
+                            { "value", base64_encode(json11::Json(json11::Json::object {
+                                { "epoch", pg_cfg.epoch },
+                                { "all_peers", pg_cfg.all_peers },
+                                { "osd_sets", pg_cfg.target_history },
+                            }).dump()) },
+                        } },
+                    });
+                    history_checks.push_back(json11::Json::object {
+                        { "target", "MOD" },
+                        { "key", history_key },
+                        { "result", "LESS" },
+                        { "mod_revision", parent->cli->st_cli.etcd_watch_revision+1 },
+                    });
+                }
+            }
+        }
+        if (history_updates.size())
+        {
+            if (retry_wait)
+            {
+                parent->waiting++;
+                parent->epmgr->tfd->set_timer(retry_wait, false, [this](int timer_id)
+                {
+                    parent->waiting--;
+                    parent->ringloop->wakeup();
+                });
+    resume_0:
+                state = base_state+0;
+                if (parent->waiting > 0)
+                    return false;
+            }
+            parent->etcd_txn(json11::Json::object {
+                { "success", history_updates },
+                { "compare", history_checks },
+            });
+        }
+        else
+            parent->etcd_result = json11::Json::object{ { "succeeded", true } };
+        return true;
+    }
+};
+
+std::function<bool(cli_result_t &)> cli_tool_t::start_rm_osd(json11::Json cfg)
+{
+    auto rm_osd = new rm_osd_t();
+    rm_osd->parent = this;
+    rm_osd->dry_run = cfg["dry_run"].bool_value();
+    rm_osd->force_dataloss = cfg["allow_data_loss"].bool_value();
+    rm_osd->force_warning = rm_osd->force_dataloss || cfg["force"].bool_value();
+    if (!cfg["etcd_tx_retries"].is_null())
+        rm_osd->etcd_tx_retries = cfg["etcd_tx_retries"].uint64_value();
+    if (!cfg["etcd_tx_retry_ms"].is_null())
+    {
+        rm_osd->etcd_tx_retry_ms = cfg["etcd_tx_retry_ms"].uint64_value();
+        if (rm_osd->etcd_tx_retry_ms < 100)
+            rm_osd->etcd_tx_retry_ms = 100;
+    }
+    if (cfg["osd_id"].is_number() || cfg["osd_id"].is_string())
+        rm_osd->osd_ids.push_back(cfg["osd_id"].uint64_value());
+    else
+    {
+        for (auto & id: cfg["osd_id"].array_items())
+            rm_osd->osd_ids.push_back(id.uint64_value());
+    }
+    return [rm_osd](cli_result_t & result)
+    {
+        rm_osd->loop();
+        if (rm_osd->is_done())
+        {
+            result = rm_osd->result;
+            delete rm_osd;
+            return true;
+        }
+        return false;
+    };
+}

src/cluster_client.cpp

@@ -221,9 +221,11 @@ void cluster_client_t::erase_op(cluster_op_t *op)
     if (op_queue_tail == op)
         op_queue_tail = op->prev;
     op->next = op->prev = NULL;
-    std::function<void(cluster_op_t*)>(op->callback)(op);
     if (!(flags & OP_IMMEDIATE_COMMIT))
         inc_wait(opcode, flags, next, -1);
+    // Call callback at the end to avoid inconsistencies in prev_wait
+    // if the callback adds more operations itself
+    std::function<void(cluster_op_t*)>(op->callback)(op);
 }
 
 void cluster_client_t::continue_ops(bool up_retry)
@@ -939,7 +941,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
                 .req = { .rw = {
                     .header = {
                         .magic = SECONDARY_OSD_OP_MAGIC,
-                        .id = op_id++,
+                        .id = next_op_id(),
                         .opcode = op->opcode == OSD_OP_READ_BITMAP ? OSD_OP_READ : op->opcode,
                     },
                     .inode = op->cur_inode,
@@ -1067,7 +1069,7 @@ void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
         .req = {
             .hdr = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
-                .id = op_id++,
+                .id = next_op_id(),
                 .opcode = OSD_OP_SYNC,
             },
         },
@@ -1179,5 +1181,5 @@ void cluster_client_t::copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *par
 
 uint64_t cluster_client_t::next_op_id()
 {
-    return op_id++;
+    return msgr.next_subop_id++;
 }

src/cluster_client.h

@@ -85,7 +85,6 @@ class cluster_client_t
     int up_wait_retry_interval = 500; // ms
 
     int retry_timeout_id = 0;
-    uint64_t op_id = 1;
     std::vector<cluster_op_t*> offline_ops;
     cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL;
     std::map<object_id, cluster_buffer_t> dirty_buffers;

src/cluster_client_list.cpp

@@ -196,7 +196,7 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
         .sec_list = {
             .header = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
-                .id = op_id++,
+                .id = next_op_id(),
                 .opcode = OSD_OP_SEC_LIST,
             },
             .list_pg = cur_list->pg->pg_num,

src/disk_tool.cpp

@@ -52,11 +52,12 @@ static const char *help_text =
     "    --disable_data_fsync 0     Disable data device cache and fsync (default off)\n"
     "    --disable_meta_fsync 0     Disable metadata device cache and fsync (default off)\n"
     "    --disable_journal_fsync 0  Disable journal device cache and fsync (default off)\n"
+    "    --hdd                      Enable HDD defaults (1M block, 1G journal, throttling)\n"
     "    --force                    Bypass partition safety checks (for emptiness and so on)\n"
     "  \n"
     "  Options (both modes):\n"
-    "    --journal_size 1G/32M      Set journal size (area or partition size)\n"
-    "    --block_size 1M/128k       Set blockstore object size\n"
+    "    --journal_size 32M/1G      Set journal size (area or partition size)\n"
+    "    --block_size 128k/1M       Set blockstore object size\n"
     "    --bitmap_granularity 4k    Set bitmap granularity\n"
     "    --data_device_block 4k     Override data device block size\n"
     "    --meta_device_block 4k     Override metadata device block size\n"
@@ -109,8 +110,16 @@ static const char *help_text =
     "  Commands are passed to systemctl with vitastor-osd@<num> units as arguments.\n"
     "  When --now is added to enable/disable, OSDs are also immediately started/stopped.\n"
     "\n"
-    "vitastor-disk read-sb <device>\n"
+    "vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]\n"
+    "  Purge Vitastor OSD(s) on specified device(s). Uses vitastor-cli rm-osd to check\n"
+    "  if deletion is possible without data loss and to actually remove metadata from etcd.\n"
+    "  --force and --allow-data-loss options may be used to ignore safety check results.\n"
+    "  \n"
+    "  Requires `vitastor-cli`, `sfdisk` and `partprobe` (from parted) utilities.\n"
+    "\n"
+    "vitastor-disk read-sb [--force] <device>\n"
     "  Try to read Vitastor OSD superblock from <device> and print it in JSON format.\n"
+    "  --force allows to ignore validation errors.\n"
     "\n"
     "vitastor-disk write-sb <device>\n"
     "  Read JSON from STDIN and write it into Vitastor OSD superblock on <device>.\n"
@@ -195,6 +204,10 @@ int main(int argc, char *argv[])
         {
             self.options["hybrid"] = "1";
         }
+        else if (!strcmp(argv[i], "--hdd"))
+        {
+            self.options["hdd"] = "1";
+        }
         else if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h"))
         {
             cmd.insert(cmd.begin(), (char*)"help");
@@ -207,6 +220,10 @@ int main(int argc, char *argv[])
         {
             self.options["force"] = "1";
         }
+        else if (!strcmp(argv[i], "--allow-data-loss"))
+        {
+            self.options["allow_data_loss"] = "1";
+        }
         else if (argv[i][0] == '-' && argv[i][1] == '-')
         {
             char *key = argv[i]+2;
@@ -339,6 +356,10 @@ int main(int argc, char *argv[])
         }
         return self.systemd_start_stop_osds(systemd_cmd, std::vector<std::string>(cmd.begin()+1, cmd.end()));
     }
+    else if (!strcmp(cmd[0], "purge"))
+    {
+        return self.purge_devices(std::vector<std::string>(cmd.begin()+1, cmd.end()));
+    }
     else if (!strcmp(cmd[0], "exec-osd"))
     {
         if (cmd.size() != 2)

src/disk_tool.h

@@ -56,7 +56,7 @@ struct disk_tool_t
     uint64_t meta_pos;
     uint64_t journal_pos, journal_calc_data_pos;
 
-    bool first, first2;
+    bool first_block, first_entry;
 
     allocator *data_alloc;
     std::map<uint64_t, uint64_t> data_remap;
@@ -108,10 +108,11 @@ struct disk_tool_t
     int read_sb(std::string device);
     int write_sb(std::string device);
     int exec_osd(std::string device);
-    int systemd_start_stop_osds(std::vector<std::string> cmd, std::vector<std::string> devices);
+    int systemd_start_stop_osds(const std::vector<std::string> & cmd, const std::vector<std::string> & devices);
     int pre_exec_osd(std::string device);
+    int purge_devices(const std::vector<std::string> & devices);
 
-    json11::Json read_osd_superblock(std::string device, bool expect_exist = true);
+    json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
     uint32_t write_osd_superblock(std::string device, json11::Json params);
 
     int prepare_one(std::map<std::string, std::string> options, int is_hdd = -1);
@@ -139,3 +140,4 @@ int write_zero(int fd, uint64_t offset, uint64_t size);
 json11::Json read_parttable(std::string dev);
 uint64_t dev_size_from_parttable(json11::Json pt);
 uint64_t free_from_parttable(json11::Json pt);
+int fix_partition_type(std::string dev_by_uuid);

src/disk_tool_journal.cpp

@@ -13,7 +13,7 @@ int disk_tool_t::dump_journal()
         fprintf(stderr, "Invalid journal block size\n");
         return 1;
     }
-    first = true;
+    first_block = true;
     if (json)
         printf("[\n");
     if (all)
@@ -38,8 +38,8 @@ int disk_tool_t::dump_journal()
             }
             if (json)
             {
-                printf("%s{\"offset\":\"0x%lx\"", first ? "" : ",\n", journal_pos);
-                first = false;
+                printf("%s{\"offset\":\"0x%lx\"", first_block ? "" : ",\n", journal_pos);
+                first_block = false;
             }
             if (s == dsk.journal_block_size)
             {
@@ -55,10 +55,10 @@ int disk_tool_t::dump_journal()
                     printf("offset %08lx:\n", journal_pos);
                 else
                     printf(",\"entries\":[\n");
-                first2 = true;
+                first_entry = true;
                 process_journal_block(journal_buf, [this](int num, journal_entry *je) { dump_journal_entry(num, je, json); });
                 if (json)
-                    printf(first2 ? "]}" : "\n]}");
+                    printf(first_entry ? "]}" : "\n]}");
             }
             else
             {
@@ -75,34 +75,30 @@ int disk_tool_t::dump_journal()
     }
     else
     {
+        first_entry = true;
         process_journal([this](void *data)
         {
-            first2 = true;
+            if (json && dump_with_blocks)
+                first_entry = true;
             if (!json)
                 printf("offset %08lx:\n", journal_pos);
             auto pos = journal_pos;
             int r = process_journal_block(data, [this, pos](int num, journal_entry *je)
             {
-                if (json && first2)
-                {
-                    if (dump_with_blocks)
-                        printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first ? "" : ",\n", pos);
-                    first = false;
-                }
+                if (json && dump_with_blocks && first_entry)
+                    printf("%s{\"offset\":\"0x%lx\",\"entries\":[\n", first_block ? "" : ",\n", pos);
                 dump_journal_entry(num, je, json);
+                first_block = false;
             });
-            if (json)
-            {
-                if (dump_with_blocks && !first2)
-                    printf("\n]}");
-            }
-            else if (r <= 0)
+            if (json && dump_with_blocks && !first_entry)
+                printf("\n]}");
+            else if (!json && r <= 0)
                 printf("end of the journal\n");
             return r;
         });
     }
     if (json)
-        printf(first ? "]\n" : "\n]\n");
+        printf(first_block ? "]\n" : "\n]\n");
     return 0;
 }
 
@@ -209,9 +205,9 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
 {
     if (json)
     {
-        if (!first2)
+        if (!first_entry)
             printf(",\n");
-        first2 = false;
+        first_entry = false;
         printf(
             "{\"crc32\":\"%08x\",\"valid\":%s,\"crc32_prev\":\"%08x\"",
             je->crc32, (je_crc32(je) == je->crc32 ? "true" : "false"), je->crc32_prev
@@ -275,10 +271,12 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
     else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
     {
         printf(
-            json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"loc\":\"0x%lx\""
-                : "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx",
+            json ? ",\"type\":\"big_write%s\",\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"ver\":\"%lu\",\"offset\":%u,\"len\":%u,\"loc\":\"0x%lx\""
+                : "je_big_write%s oid=%lx:%lx ver=%lu offset=%u len=%u loc=%08lx",
             je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
-            je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location
+            je->big_write.oid.inode, je->big_write.oid.stripe,
+            je->big_write.version, je->big_write.offset, je->big_write.len,
+            je->big_write.location
         );
         if (je->big_write.size > sizeof(journal_entry_big_write))
         {
@@ -424,6 +422,8 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                     .stripe = sscanf_json(NULL, rec["stripe"]),
                 },
                 .version = rec["ver"].uint64_value(),
+                .offset = (uint32_t)rec["offset"].uint64_value(),
+                .len = (uint32_t)rec["len"].uint64_value(),
                 .location = sscanf_json(NULL, rec["loc"]),
             };
             fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));

src/disk_tool_meta.cpp

@@ -124,14 +124,14 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v1_t *hdr)
     {
         printf("{\"version\":\"0.5\",\"meta_block_size\":%lu,\"entries\":[\n", dsk.meta_block_size);
     }
-    first = true;
+    first_entry = true;
 }
 
 void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
 {
     printf(
-#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu"
-        (first ? ENTRY_FMT : (",\n" ENTRY_FMT)),
+#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":\"0x%lx\",\"stripe\":\"0x%lx\",\"version\":%lu"
+        (first_entry ? ENTRY_FMT : (",\n" ENTRY_FMT)),
 #undef ENTRY_FMT
         block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
         entry->oid.stripe, entry->version
@@ -154,7 +154,7 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
     {
         printf("}");
     }
-    first = false;
+    first_entry = false;
 }
 
 int disk_tool_t::write_json_meta(json11::Json meta)

src/disk_tool_prepare.cpp

@@ -61,6 +61,11 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
                 fprintf(stderr, "%s already contains Vitastor OSD superblock, not creating OSD without --force\n", dev.c_str());
                 return 1;
             }
+            if (fix_partition_type(dev) != 0)
+            {
+                fprintf(stderr, "%s has incorrect type and we failed to change it to Vitastor type\n", dev.c_str());
+                return 1;
+            }
         }
     }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
@@ -317,7 +322,8 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
     {
         script += "+ "+size+" "+std::string(VITASTOR_PART_TYPE)+"\n";
     }
-    if (shell_exec({ "sfdisk", "--force", devinfo.path }, script, NULL, NULL) != 0)
+    std::string out;
+    if (shell_exec({ "sfdisk", "--no-reread", "--force", devinfo.path }, script, &out, NULL) != 0)
     {
         fprintf(stderr, "Failed to add %lu partition(s) with sfdisk\n", sizes.size());
         return {};
@@ -351,7 +357,8 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
                 {
                     iter++;
                     // Run partprobe
-                    if (iter > 1 || (r = shell_exec({ "partprobe", devinfo.path }, "", NULL, NULL)) != 0)
+                    std::string out;
+                    if (iter > 1 || (r = shell_exec({ "partprobe", devinfo.path }, "", &out, NULL)) != 0)
                     {
                         fprintf(
                             stderr, iter == 1 && r == 255
@@ -539,7 +546,7 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
             fprintf(stderr, "Device list (positional arguments) and --hybrid are incompatible with --data_device\n");
             return 1;
         }
-        return prepare_one(options);
+        return prepare_one(options, options.find("hdd") != options.end() ? 1 : 0);
     }
     if (!devices.size())
     {
@@ -549,12 +556,12 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     options.erase("data_device");
     options.erase("meta_device");
     options.erase("journal_device");
+    bool hybrid = options.find("hybrid") != options.end();
     auto devinfo = collect_devices(devices);
     if (!devinfo.size())
     {
         return 1;
     }
-    bool hybrid = options.find("hybrid") != options.end();
     uint64_t osd_per_disk = stoull_full(options["osd_per_disk"]);
     if (!osd_per_disk)
         osd_per_disk = 1;
@@ -612,7 +619,8 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
                         return 1;
                     }
                 }
-                prepare_one(options, dev.is_hdd ? 1 : 0);
+                // Treat all disks as SSDs if not in the hybrid mode
+                prepare_one(options, hybrid && dev.is_hdd ? 1 : 0);
             }
         }
     }

src/disk_tool_udev.cpp

@@ -5,6 +5,7 @@
 
 #include "disk_tool.h"
 #include "rw_blocking.h"
+#include "str_util.h"
 
 struct __attribute__((__packed__)) vitastor_disk_superblock_t
 {
@@ -54,7 +55,7 @@ int disk_tool_t::udev_import(std::string device)
 
 int disk_tool_t::read_sb(std::string device)
 {
-    json11::Json sb = read_osd_superblock(device);
+    json11::Json sb = read_osd_superblock(device, true, options.find("force") != options.end());
     if (sb.is_null())
     {
         return 1;
@@ -123,7 +124,7 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
     return sb_size;
 }
 
-json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist)
+json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_exist, bool ignore_errors)
 {
     vitastor_disk_superblock_t *sb = NULL;
     uint8_t *buf = NULL;
@@ -144,7 +145,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         goto ex;
     }
     sb = (vitastor_disk_superblock_t*)buf;
-    if (sb->magic != VITASTOR_DISK_MAGIC)
+    if (sb->magic != VITASTOR_DISK_MAGIC && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "Invalid OSD superblock on %s: magic number mismatch\n", device.c_str());
@@ -172,7 +173,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         }
         sb = (vitastor_disk_superblock_t*)buf;
     }
-    if (sb->crc32c != crc32c(0, &sb->size, sb->size - ((uint8_t*)&sb->size - buf)))
+    if (sb->crc32c != crc32c(0, &sb->size, sb->size - ((uint8_t*)&sb->size - buf)) && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "Invalid OSD superblock on %s: crc32 mismatch\n", device.c_str());
@@ -186,14 +187,14 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         goto ex;
     }
     // Validate superblock
-    if (!osd_params["osd_num"].uint64_value())
+    if (!osd_params["osd_num"].uint64_value() && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "OSD superblock on %s lacks osd_num\n", device.c_str());
         osd_params = json11::Json();
         goto ex;
     }
-    if (osd_params["data_device"].string_value() == "")
+    if (osd_params["data_device"].string_value() == "" && !ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "OSD superblock on %s lacks data_device\n", device.c_str());
@@ -226,7 +227,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
     {
         device_type = "journal";
     }
-    else
+    else if (!ignore_errors)
     {
         if (expect_exist)
             fprintf(stderr, "Invalid OSD superblock on %s: does not refer to the device itself\n", device.c_str());
@@ -246,7 +247,7 @@ ex:
     return osd_params;
 }
 
-int disk_tool_t::systemd_start_stop_osds(std::vector<std::string> cmd, std::vector<std::string> devices)
+int disk_tool_t::systemd_start_stop_osds(const std::vector<std::string> & cmd, const std::vector<std::string> & devices)
 {
     if (!devices.size())
     {
@@ -306,8 +307,7 @@ int disk_tool_t::exec_osd(std::string device)
         argv[i] = (char*)argstr[i].c_str();
     }
     argv[argstr.size()] = NULL;
-    execvpe(osd_binary.c_str(), argv, environ);
-    return 0;
+    return execvpe(osd_binary.c_str(), argv, environ);
 }
 
 static int check_disabled_cache(std::string dev)
@@ -362,3 +362,140 @@ int disk_tool_t::pre_exec_osd(std::string device)
     }
     return 0;
 }
+
+int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
+{
+    std::vector<uint64_t> osd_numbers;
+    json11::Json::array superblocks;
+    for (auto & device: devices)
+    {
+        json11::Json sb = read_osd_superblock(device);
+        if (!sb.is_null())
+        {
+            uint64_t osd_num = sb["params"]["osd_num"].uint64_value();
+            osd_numbers.push_back(osd_num);
+            superblocks.push_back(sb);
+        }
+    }
+    if (!osd_numbers.size())
+    {
+        return 0;
+    }
+    std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd" };
+    for (auto osd_num: osd_numbers)
+    {
+        rm_osd_cli.push_back(std::to_string(osd_num));
+    }
+    // Check for data loss
+    if (options["force"] != "")
+    {
+        rm_osd_cli.push_back("--force");
+    }
+    else if (options["allow_data_loss"] != "")
+    {
+        rm_osd_cli.push_back("--allow-data-loss");
+    }
+    rm_osd_cli.push_back("--dry-run");
+    std::string dry_run_ignore_stdout;
+    if (shell_exec(rm_osd_cli, "", &dry_run_ignore_stdout, NULL) != 0)
+    {
+        return 1;
+    }
+    // Disable & stop OSDs
+    std::vector<std::string> systemctl_cli = { "systemctl", "disable", "--now" };
+    for (auto osd_num: osd_numbers)
+    {
+        systemctl_cli.push_back("vitastor-osd@"+std::to_string(osd_num));
+    }
+    if (shell_exec(systemctl_cli, "", NULL, NULL) != 0)
+    {
+        return 1;
+    }
+    // Remove OSD metadata
+    rm_osd_cli.pop_back();
+    if (shell_exec(rm_osd_cli, "", NULL, NULL) != 0)
+    {
+        return 1;
+    }
+    // Destroy OSD superblocks
+    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096);
+    for (auto & sb: superblocks)
+    {
+        for (auto dev_type: std::vector<std::string>{ "data", "meta", "journal" })
+        {
+            auto dev = sb["real_"+dev_type+"_device"].string_value();
+            if (dev != "")
+            {
+                int fd = -1, r = open(dev.c_str(), O_DIRECT|O_RDWR);
+                if (r >= 0)
+                {
+                    fd = r;
+                    r = read_blocking(fd, buf, 4096);
+                    if (r == 4096)
+                    {
+                        // Clear magic and CRC
+                        memset(buf, 0, 12);
+                        r = lseek64(fd, 0, 0);
+                        if (r == 0)
+                        {
+                            r = write_blocking(fd, buf, 4096);
+                            if (r == 4096)
+                                r = 0;
+                        }
+                    }
+                }
+                if (fd >= 0)
+                    close(fd);
+                if (r != 0)
+                {
+                    fprintf(stderr, "Failed to clear OSD %lu %s device %s superblock: %s\n",
+                        sb["params"]["osd_num"].uint64_value(), dev_type.c_str(), dev.c_str(), strerror(errno));
+                }
+                else
+                {
+                    fprintf(stderr, "OSD %lu %s device %s superblock cleared\n",
+                        sb["params"]["osd_num"].uint64_value(), dev_type.c_str(), dev.c_str());
+                }
+                if (sb["params"][dev_type+"_device"].string_value().substr(0, 22) == "/dev/disk/by-partuuid/")
+                {
+                    // Delete the partition itself
+                    auto uuid_to_del = strtolower(sb["params"][dev_type+"_device"].string_value().substr(22));
+                    auto parent_dev = get_parent_device(dev);
+                    if (parent_dev == "" || parent_dev == dev)
+                    {
+                        fprintf(stderr, "Failed to delete partition %s: failed to find parent device\n", dev.c_str());
+                        continue;
+                    }
+                    auto pt = read_parttable("/dev/"+parent_dev);
+                    if (!pt.is_object())
+                        continue;
+                    json11::Json::array newpt = pt["partitions"].array_items();
+                    for (int i = 0; i < newpt.size(); i++)
+                    {
+                        if (strtolower(newpt[i]["uuid"].string_value()) == uuid_to_del)
+                        {
+                            auto old_part = newpt[i];
+                            newpt.erase(newpt.begin()+i, newpt.begin()+i+1);
+                            vitastor_dev_info_t devinfo = {
+                                .path = "/dev/"+parent_dev,
+                                .pt = json11::Json::object{ { "partitions", newpt } },
+                            };
+                            add_partitions(devinfo, {});
+                            struct stat st;
+                            if (stat(old_part["node"].string_value().c_str(), &st) == 0 ||
+                                errno != ENOENT)
+                            {
+                                std::string out;
+                                shell_exec({ "partprobe", "/dev/"+parent_dev }, "", &out, NULL);
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    free(buf);
+    buf = NULL;
+    return 0;
+}

src/disk_tool_upgrade.cpp

@@ -38,42 +38,6 @@ static std::map<std::string, std::string> read_vitastor_unit(std::string unit)
     return r;
 }
 
-static int fix_partition_type(std::string dev_by_uuid)
-{
-    auto uuid = strtolower(dev_by_uuid.substr(dev_by_uuid.rfind('/')+1));
-    std::string parent_dev = get_parent_device(realpath_str(dev_by_uuid, false));
-    if (parent_dev == "")
-        return 1;
-    auto pt = read_parttable("/dev/"+parent_dev);
-    if (pt.is_null())
-        return 1;
-    std::string script = "label: gpt\n\n";
-    for (const auto & part: pt["partitions"].array_items())
-    {
-        bool this_part = (strtolower(part["uuid"].string_value()) == uuid);
-        if (this_part && strtolower(part["type"].string_value()) == "e7009fac-a5a1-4d72-af72-53de13059903")
-        {
-            // Already correct type
-            return 0;
-        }
-        script += part["node"].string_value()+": ";
-        bool first = true;
-        for (const auto & kv: part.object_items())
-        {
-            if (kv.first != "node")
-            {
-                script += (first ? "" : ", ")+kv.first+"="+
-                    (kv.first == "type" && this_part
-                        ? "e7009fac-a5a1-4d72-af72-53de13059903"
-                        : (kv.second.is_string() ? kv.second.string_value() : kv.second.dump()));
-                first = false;
-            }
-        }
-        script += "\n";
-    }
-    return shell_exec({ "sfdisk", "--no-reread", "--force", "/dev/"+parent_dev }, script, NULL, NULL);
-}
-
 int disk_tool_t::upgrade_simple_unit(std::string unit)
 {
     if (stoull_full(unit) != 0)

src/disk_tool_utils.cpp

@@ -145,10 +145,10 @@ int disable_cache(std::string dev)
         closedir(dir);
         // Check cache_type
         scsi_disk += "/cache_type";
-        std::string cache_type = read_file(scsi_disk);
+        std::string cache_type = trim(read_file(scsi_disk));
         if (cache_type == "")
             return 1;
-        if (cache_type == "write back")
+        if (cache_type != "write through")
         {
             int fd = open(scsi_disk.c_str(), O_WRONLY);
             if (fd < 0 || write_blocking(fd, (void*)"write through", strlen("write through")) != strlen("write through"))
@@ -239,7 +239,8 @@ int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std
     {
         // Child
         dup2(child_stdin[0], 0);
-        dup2(child_stdout[1], 1);
+        if (out)
+            dup2(child_stdout[1], 1);
         if (err)
             dup2(child_stderr[1], 2);
         close(child_stdin[0]);
@@ -250,9 +251,7 @@ int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std
         close(child_stderr[1]);
         char *argv[cmd.size()+1];
         for (int i = 0; i < cmd.size(); i++)
-        {
             argv[i] = (char*)cmd[i].c_str();
-        }
         argv[cmd.size()] = NULL;
         execvp(argv[0], argv);
         std::string full_cmd;
@@ -354,3 +353,40 @@ uint64_t free_from_parttable(json11::Json pt)
         free *= pt["sectorsize"].uint64_value();
     return free;
 }
+
+int fix_partition_type(std::string dev_by_uuid)
+{
+    auto uuid = strtolower(dev_by_uuid.substr(dev_by_uuid.rfind('/')+1));
+    std::string parent_dev = get_parent_device(realpath_str(dev_by_uuid, false));
+    if (parent_dev == "")
+        return 1;
+    auto pt = read_parttable("/dev/"+parent_dev);
+    if (pt.is_null() || pt.is_bool())
+        return 1;
+    std::string script = "label: gpt\n\n";
+    for (const auto & part: pt["partitions"].array_items())
+    {
+        bool this_part = (strtolower(part["uuid"].string_value()) == uuid);
+        if (this_part && strtolower(part["type"].string_value()) == "e7009fac-a5a1-4d72-af72-53de13059903")
+        {
+            // Already correct type
+            return 0;
+        }
+        script += part["node"].string_value()+": ";
+        bool first = true;
+        for (const auto & kv: part.object_items())
+        {
+            if (kv.first != "node")
+            {
+                script += (first ? "" : ", ")+kv.first+"="+
+                    (kv.first == "type" && this_part
+                        ? "e7009fac-a5a1-4d72-af72-53de13059903"
+                        : (kv.second.is_string() ? kv.second.string_value() : kv.second.dump()));
+                first = false;
+            }
+        }
+        script += "\n";
+    }
+    std::string out;
+    return shell_exec({ "sfdisk", "--no-reread", "--force", "/dev/"+parent_dev }, script, &out, NULL);
+}

src/etcd_state_client.cpp

@@ -871,19 +871,33 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             pg_cfg.target_history.clear();
             pg_cfg.all_peers.clear();
             // Refuse to start PG if any set of the <osd_sets> has no live OSDs
-            for (auto hist_item: value["osd_sets"].array_items())
+            for (auto & hist_item: value["osd_sets"].array_items())
             {
                 std::vector<osd_num_t> history_set;
-                for (auto pg_osd: hist_item.array_items())
+                for (auto & pg_osd: hist_item.array_items())
                 {
-                    history_set.push_back(pg_osd.uint64_value());
+                    osd_num_t pg_osd_num = pg_osd.uint64_value();
+                    if (pg_osd_num != 0)
+                    {
+                        auto it = std::lower_bound(history_set.begin(), history_set.end(), pg_osd_num);
+                        if (it == history_set.end() || *it != pg_osd_num)
+                            history_set.insert(it, pg_osd_num);
+                    }
                 }
-                pg_cfg.target_history.push_back(history_set);
+                auto it = std::lower_bound(pg_cfg.target_history.begin(), pg_cfg.target_history.end(), history_set);
+                if (it == pg_cfg.target_history.end() || *it != history_set)
+                    pg_cfg.target_history.insert(it, history_set);
             }
             // Include these additional OSDs when peering the PG
             for (auto pg_osd: value["all_peers"].array_items())
             {
-                pg_cfg.all_peers.push_back(pg_osd.uint64_value());
+                osd_num_t pg_osd_num = pg_osd.uint64_value();
+                if (pg_osd_num != 0)
+                {
+                    auto it = std::lower_bound(pg_cfg.all_peers.begin(), pg_cfg.all_peers.end(), pg_osd_num);
+                    if (it == pg_cfg.all_peers.end() || *it != pg_osd_num)
+                        pg_cfg.all_peers.insert(it, pg_osd_num);
+                }
             }
             // Read epoch
             pg_cfg.epoch = value["epoch"].uint64_value();

src/lrc/Makefile

@@ -0,0 +1,2 @@
+mat: mat.c
+	gcc -O3 -I/usr/include/jerasure -o mat mat.c -lJerasure

src/lrc/mat.c

@@ -0,0 +1,291 @@
+#include <jerasure/reed_sol.h>
+#include <jerasure.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+// Generate LRC matrix: (groups*local + global) code rows with (data_drives) columns
+// w should be >= log2(data_drives + groups*local + global), but not necessary 8/16/32
+int* reed_sol_vandermonde_lrc_matrix(int data_drives, int groups, int local, int global, int w)
+{
+    if (w < 0 || w > 32 || data_drives + groups*local + global > (1<<w))
+    {
+        return NULL;
+    }
+    int *lrc_matrix = (int*)malloc(sizeof(int) * (local*groups+global));
+    int *matrix = reed_sol_vandermonde_coding_matrix(data_drives, local+global, w);
+    // Enough to transform LRC 8+2+2 GF(8) matrix into MR-LRC
+    //for (int i = 0; i < local+global; i++)
+    //{
+    //    int t = matrix[i*data_drives + 3];
+    //    matrix[i*data_drives + 3] = matrix[i*data_drives + 7];
+    //    matrix[i*data_drives + 7] = t;
+    //}
+    for (int gr = 0; gr < groups; gr++)
+    {
+        for (int l = 0; l < local; l++)
+        {
+            for (int j = 0; j < data_drives; j++)
+            {
+                lrc_matrix[(gr*local+l)*data_drives + j] = (j / (data_drives/groups)) == gr ? matrix[l*data_drives + j] : 0;
+            }
+        }
+    }
+    for (int i = 0; i < global; i++)
+    {
+        for (int j = 0; j < data_drives; j++)
+        {
+            lrc_matrix[(groups*local+i)*data_drives + j] = matrix[(local+i)*data_drives + j];
+        }
+    }
+    free(matrix);
+    return lrc_matrix;
+}
+
+struct lrc_test_result_t
+{
+    int success, impossible, failures;
+};
+
+// Check if the generated LRC with given parameters is Maximally Reconstructible (MR-LRC)
+// Example of a MR-LRC: (8, 2, 1, 2, 6, 8)
+struct lrc_test_result_t check_mr_lrc(int *lrc_matrix, int data_drives, int groups, int local, int global, int w, int log_level)
+{
+    int n = data_drives;
+    int total_rows = n + groups*local + global;
+    int impossible = 0, success = 0, failures = 0;
+    int *lost_per_group = (int*)malloc(sizeof(int) * groups);
+    int *recovered_per_group = (int*)malloc(sizeof(int) * groups);
+    int *selected_inverted = (int*)malloc(sizeof(int) * data_drives);
+    // global+1 is always recoverable
+    for (int lost = global+2; lost <= groups*local+global; lost++)
+    {
+        int *erased_matrix = (int*)malloc(sizeof(int) * (total_rows-lost)*n);
+        int *inverted_matrix = (int*)malloc(sizeof(int) * (total_rows-lost)*n);
+        int *p = (int*)malloc(sizeof(int) * (total_rows-lost));
+        for (int i = 0; i < n; i++)
+            p[i] = i;
+        int *p2 = (int*)malloc(sizeof(int) * n);
+        if (total_rows-lost > n)
+        {
+            p[n-1] = n; // skip combinations with all N data disks (0..n-1)
+            for (int i = n; i < total_rows-lost; i++)
+                p[i] = i+1;
+            p[total_rows-lost-1]--; // will be incremented on the first step
+        }
+        int inc = total_rows-lost-1;
+        while (1)
+        {
+            p[inc]++;
+            if (p[inc] >= n+groups*local+global)
+            {
+                if (inc == 0)
+                    break;
+                inc--;
+            }
+            else if (inc+1 < total_rows-lost)
+            {
+                p[inc+1] = p[inc];
+                inc++;
+            }
+            else
+            {
+                // Check if it should be recoverable
+                // Calculate count of data chunks lost in each group
+                int nsel = 0;
+                for (int gr = 0; gr < groups; gr++)
+                {
+                    lost_per_group[gr] = ((gr+1)*(n/groups) > n ? (n - gr*(n/groups)) : n/groups);
+                    recovered_per_group[gr] = 0;
+                }
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] < n)
+                    {
+                        lost_per_group[(p[j] / (n/groups))]--;
+                        selected_inverted[nsel++] = j;
+                    }
+                }
+                // Every local parity chunk is supposed to restore 1 missing chunk inside its group
+                // So, subtract local parity chunk counts from each group lost chunk count
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] >= n && p[j] < n+groups*local)
+                    {
+                        int gr = (p[j]-n)/local;
+                        if (lost_per_group[gr] > recovered_per_group[gr] && nsel < n)
+                        {
+                            selected_inverted[nsel++] = j;
+                        }
+                        recovered_per_group[gr]++;
+                    }
+                }
+                // Every global parity chunk is supposed to restore 1 chunk of all that are still missing
+                int still_missing = 0;
+                for (int gr = 0; gr < groups; gr++)
+                {
+                    int non_fixed = lost_per_group[gr] - recovered_per_group[gr];
+                    still_missing += (non_fixed > 0 ? non_fixed : 0);
+                }
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] >= n+groups*local)
+                    {
+                        if (still_missing > 0 && nsel < n)
+                        {
+                            selected_inverted[nsel++] = j;
+                        }
+                        still_missing--;
+                    }
+                }
+                if (still_missing <= 0)
+                {
+                    // We hope it can be recoverable. Try to invert it
+                    assert(nsel == n);
+                    for (int i = 0; i < n; i++)
+                    {
+                        for (int j = 0; j < n; j++)
+                        {
+                            erased_matrix[i*n+j] = lrc_matrix[p[selected_inverted[i]]*n+j];
+                        }
+                    }
+                    int invert_ok = jerasure_invert_matrix(erased_matrix, inverted_matrix, n, w);
+                    if (invert_ok < 0)
+                    {
+                        failures++;
+                        if (log_level > 0)
+                        {
+                            printf("\nFAIL: ");
+                            for (int i = 0; i < total_rows-lost; i++)
+                            {
+                                printf("%d ", p[i]);
+                            }
+                            printf("\nDIRECT:\n");
+                            for (int i = 0; i < total_rows-lost; i++)
+                            {
+                                for (int j = 0; j < n; j++)
+                                    printf("%d ", lrc_matrix[p[i]*n+j]);
+                                printf("\n");
+                            }
+                            printf("INVERSE:\n");
+                            for (int i = 0; i < total_rows-lost; i++)
+                            {
+                                for (int j = 0; j < n; j++)
+                                    printf("%d ", inverted_matrix[i*n+j]);
+                                printf("\n");
+                            }
+                        }
+                    }
+                    else
+                    {
+                        success++;
+                        if (log_level > 2)
+                        {
+                            printf("OK: ");
+                            for (int i = 0; i < total_rows-lost; i++)
+                            {
+                                printf("%d ", p[i]);
+                            }
+                            printf("\n");
+                        }
+                    }
+                }
+                else
+                {
+                    impossible++;
+                    if (log_level > 1)
+                    {
+                        printf("IMPOSSIBLE: ");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            printf("%d ", p[i]);
+                        }
+                        printf("\n");
+                    }
+                }
+            }
+        }
+        free(p2);
+        free(p);
+        free(inverted_matrix);
+        free(erased_matrix);
+    }
+    free(lost_per_group);
+    free(recovered_per_group);
+    return (struct lrc_test_result_t){
+        .success = success,
+        .impossible = impossible,
+        .failures = failures,
+    };
+}
+
+int main()
+{
+    int W = 8, MATRIX_W = 8;
+    int n = 8, groups = 2, local = 1, global = 2;
+    //n = 4, groups = 2, local = 1, global = 1;
+    int total_rows = n+groups*local+global;
+    int *matrix = reed_sol_vandermonde_lrc_matrix(n, groups, local, global, MATRIX_W);
+    int *lrc_matrix = (int*)malloc(sizeof(int) * total_rows*n);
+    // Fill identity+LRC matrix
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++)
+            lrc_matrix[i*n + j] = j == i ? 1 : 0;
+    memcpy(lrc_matrix + n*n, matrix, (total_rows-n)*n*sizeof(int));
+    free(matrix);
+    matrix = NULL;
+    // Print LRC matrix
+    for (int i = 0; i < total_rows; i++)
+    {
+        for (int j = 0; j < n; j++)
+        {
+            printf("%d ", lrc_matrix[i*n+j]);
+        }
+        printf("\n");
+    }
+    struct lrc_test_result_t t = check_mr_lrc(lrc_matrix, n, groups, local, global, W, 1);
+    printf("\n%d recovered, %d impossible, %d failures\n", t.success, t.impossible, t.failures);
+    return 0;
+}
+
+// 1 1 1 1 0 0 0 0
+// 0 0 0 0 1 1 1 1
+// 1 55 39 73 84 181 225 217
+// 1 172 70 235 143 34 200 101
+//
+// Can't recover
+// 1 2 4 5 8 9 10 11    -1
+// 2 3 4 6 8 9 10 11    -1
+// FULL:
+// 1 0 0 0 0 0 0 0 
+// 0 1 0 0 0 0 0 0 
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 1 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 1 0 0 
+// 0 0 0 0 0 0 1 0 
+// 0 0 0 0 0 0 0 1 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// FIRST UNRECOVERABLE:
+// 0 1 0 0 0 0 0 0 
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 1 0 0 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// SECOND UNRECOVERABLE:
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 1 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 0 1 0 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// Ho ho ho

src/messenger.cpp

@@ -80,12 +80,20 @@ void osd_messenger_t::init()
                     };
                     op->callback = [this, cl](osd_op_t *op)
                     {
+                        auto cl_it = clients.find(op->peer_fd);
+                        if (cl_it == clients.end() || cl_it->second != cl)
+                        {
+                            // client is already dropped
+                            delete op;
+                            return;
+                        }
                         int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
+                        auto fail_osd_num = cl->osd_num;
                         cl->ping_time_remaining = 0;
                         delete op;
                         if (fail_fd >= 0)
                         {
-                            fprintf(stderr, "Ping failed for OSD %lu (client %d), disconnecting peer\n", cl->osd_num, cl->peer_fd);
+                            fprintf(stderr, "Ping failed for OSD %lu (client %d), disconnecting peer\n", fail_osd_num, fail_fd);
                             stop_client(fail_fd, true);
                         }
                     };

src/osd.cpp

@@ -163,6 +163,9 @@ void osd_t::parse_config(const json11::Json & config, bool allow_disk_params)
     recovery_queue_depth = config["recovery_queue_depth"].uint64_value();
     if (recovery_queue_depth < 1 || recovery_queue_depth > MAX_RECOVERY_QUEUE)
         recovery_queue_depth = DEFAULT_RECOVERY_QUEUE;
+    recovery_pg_switch = config["recovery_pg_switch"].uint64_value();
+    if (recovery_pg_switch < 1)
+        recovery_pg_switch = DEFAULT_RECOVERY_PG_SWITCH;
     recovery_sync_batch = config["recovery_sync_batch"].uint64_value();
     if (recovery_sync_batch < 1 || recovery_sync_batch > MAX_RECOVERY_QUEUE)
         recovery_sync_batch = DEFAULT_RECOVERY_BATCH;

src/osd.h

@@ -34,6 +34,7 @@
 #define DEFAULT_AUTOSYNC_WRITES 128
 #define MAX_RECOVERY_QUEUE 2048
 #define DEFAULT_RECOVERY_QUEUE 4
+#define DEFAULT_RECOVERY_PG_SWITCH 128
 #define DEFAULT_RECOVERY_BATCH 16
 
 //#define OSD_STUB
@@ -108,6 +109,7 @@ class osd_t
     int autosync_interval = DEFAULT_AUTOSYNC_INTERVAL; // "emergency" sync every 5 seconds
     int autosync_writes = DEFAULT_AUTOSYNC_WRITES;
     int recovery_queue_depth = DEFAULT_RECOVERY_QUEUE;
+    int recovery_pg_switch = DEFAULT_RECOVERY_PG_SWITCH;
     int recovery_sync_batch = DEFAULT_RECOVERY_BATCH;
     int inode_vanish_time = 60;
     int log_level = 0;
@@ -135,7 +137,10 @@ class osd_t
     uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0;
     int peering_state = 0;
     std::map<object_id, osd_recovery_op_t> recovery_ops;
-    int recovery_done = 0;
+    bool recovery_last_degraded = true;
+    pool_pg_num_t recovery_last_pg;
+    object_id recovery_last_oid;
+    int recovery_pg_done = 0, recovery_done = 0;
     osd_op_t *autosync_op = NULL;
 
     // Unstable writes
@@ -200,7 +205,6 @@ class osd_t
     bool check_peer_config(osd_client_t *cl, json11::Json conf);
     void repeer_pgs(osd_num_t osd_num);
     void start_pg_peering(pg_t & pg);
-    void submit_sync_and_list_subop(osd_num_t role_osd, pg_peering_state_t *ps);
     void submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps);
     void discard_list_subop(osd_op_t *list_op);
     bool stop_pg(pg_t & pg);

src/osd_cluster.cpp

@@ -132,7 +132,7 @@ bool osd_t::check_peer_config(osd_client_t *cl, json11::Json conf)
                 this->osd_num, immediate_commit == IMMEDIATE_ALL ? "all" : "small",
                 cl->osd_num, conf["immediate_commit"].string_value().c_str()
             );
-            return true;
+            return false;
         }
         else if (conf["block_size"].uint64_value() != (uint64_t)this->bs_block_size)
         {
@@ -140,7 +140,7 @@ bool osd_t::check_peer_config(osd_client_t *cl, json11::Json conf)
                 "[OSD %lu] My block_size is %u, but peer OSD %lu has %lu. We can't work together\n",
                 this->osd_num, this->bs_block_size, cl->osd_num, conf["block_size"].uint64_value()
             );
-            return true;
+            return false;
         }
         else if (conf["bitmap_granularity"].uint64_value() != (uint64_t)this->bs_bitmap_granularity)
         {
@@ -148,7 +148,7 @@ bool osd_t::check_peer_config(osd_client_t *cl, json11::Json conf)
                 "[OSD %lu] My bitmap_granularity is %u, but peer OSD %lu has %lu. We can't work together\n",
                 this->osd_num, this->bs_bitmap_granularity, cl->osd_num, conf["bitmap_granularity"].uint64_value()
             );
-            return true;
+            return false;
         }
     }
     return true;
@@ -382,30 +382,6 @@ void osd_t::on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes
     }
 }
 
-void osd_t::on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num)
-{
-    auto pg_it = pgs.find({
-        .pool_id = pool_id,
-        .pg_num = pg_num,
-    });
-    if (pg_it != pgs.end() && pg_it->second.epoch > pg_it->second.reported_epoch &&
-        st_cli.pool_config[pool_id].pg_config[pg_num].epoch >= pg_it->second.epoch)
-    {
-        pg_it->second.reported_epoch = st_cli.pool_config[pool_id].pg_config[pg_num].epoch;
-        object_id oid = { 0 };
-        bool first = true;
-        for (auto op: pg_it->second.write_queue)
-        {
-            if (first || oid != op.first)
-            {
-                oid = op.first;
-                first = false;
-                continue_primary_write(op.second);
-            }
-        }
-    }
-}
-
 void osd_t::on_load_config_hook(json11::Json::object & global_config)
 {
     json11::Json::object osd_config = this->config;
@@ -704,13 +680,16 @@ void osd_t::apply_pg_config()
                         }
                     }
                 }
+                auto vec_all_peers = std::vector<osd_num_t>(all_peers.begin(), all_peers.end());
                 if (currently_taken)
                 {
                     if (pg_it->second.state & (PG_ACTIVE | PG_INCOMPLETE | PG_PEERING | PG_REPEERING | PG_PEERED))
                     {
-                        if (pg_it->second.target_set == pg_cfg.target_set)
+                        if (pg_it->second.target_set == pg_cfg.target_set &&
+                            pg_it->second.target_history == pg_cfg.target_history &&
+                            pg_it->second.all_peers == vec_all_peers)
                         {
-                            // No change in osd_set; history changes are ignored
+                            // No change in osd_set and history
                             continue;
                         }
                         else
@@ -761,7 +740,7 @@ void osd_t::apply_pg_config()
                     .pg_num = pg_num,
                     .reported_epoch = pg_cfg.epoch,
                     .target_history = pg_cfg.target_history,
-                    .all_peers = std::vector<osd_num_t>(all_peers.begin(), all_peers.end()),
+                    .all_peers = vec_all_peers,
                     .target_set = pg_cfg.target_set,
                 };
                 if (pg.scheme == POOL_SCHEME_EC)

src/osd_flush.cpp

@@ -226,42 +226,51 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
 
 bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
 {
-    if (!no_recovery)
+    if (!pgs.size())
     {
-        for (auto pg_it = pgs.begin(); pg_it != pgs.end(); pg_it++)
-        {
-            if ((pg_it->second.state & (PG_ACTIVE | PG_HAS_DEGRADED)) == (PG_ACTIVE | PG_HAS_DEGRADED))
-            {
-                for (auto obj_it = pg_it->second.degraded_objects.begin(); obj_it != pg_it->second.degraded_objects.end(); obj_it++)
-                {
-                    if (recovery_ops.find(obj_it->first) == recovery_ops.end())
-                    {
-                        op.degraded = true;
-                        op.oid = obj_it->first;
-                        return true;
-                    }
-                }
-            }
-        }
+        return false;
     }
-    if (!no_rebalance)
+    // Restart scanning from the same degraded/misplaced status as the last time
+    for (int tried_degraded = 0; tried_degraded < 2; tried_degraded++)
     {
-        for (auto pg_it = pgs.begin(); pg_it != pgs.end(); pg_it++)
+        if (recovery_last_degraded ? !no_recovery : !no_rebalance)
         {
             // Don't try to "recover" misplaced objects if "recovery" would make them degraded
-            if ((pg_it->second.state & (PG_ACTIVE | PG_DEGRADED | PG_HAS_MISPLACED)) == (PG_ACTIVE | PG_HAS_MISPLACED))
+            auto mask = recovery_last_degraded ? (PG_ACTIVE | PG_HAS_DEGRADED) : (PG_ACTIVE | PG_DEGRADED | PG_HAS_MISPLACED);
+            auto check = recovery_last_degraded ? (PG_ACTIVE | PG_HAS_DEGRADED) : (PG_ACTIVE | PG_HAS_MISPLACED);
+            // Restart scanning from the same PG as the last time
+            for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++)
             {
-                for (auto obj_it = pg_it->second.misplaced_objects.begin(); obj_it != pg_it->second.misplaced_objects.end(); obj_it++)
+                if ((pg_it->second.state & mask) == check)
                 {
-                    if (recovery_ops.find(obj_it->first) == recovery_ops.end())
+                    auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects;
+                    assert(src.size() > 0);
+                    // Restart scanning from the next object
+                    for (auto obj_it = src.upper_bound(recovery_last_oid); obj_it != src.end(); obj_it++)
                     {
-                        op.degraded = false;
-                        op.oid = obj_it->first;
-                        return true;
+                        if (recovery_ops.find(obj_it->first) == recovery_ops.end())
+                        {
+                            op.degraded = recovery_last_degraded;
+                            recovery_last_oid = op.oid = obj_it->first;
+                            recovery_pg_done++;
+                            // Switch to another PG after recovery_pg_switch operations
+                            // to always mix all PGs during recovery but still benefit
+                            // from recovery queue depth greater than 1
+                            if (recovery_pg_done >= recovery_pg_switch)
+                            {
+                                recovery_pg_done = 0;
+                                recovery_last_pg.pg_num++;
+                                recovery_last_oid = {};
+                            }
+                            return true;
+                        }
                     }
                 }
             }
         }
+        recovery_last_degraded = !recovery_last_degraded;
+        recovery_last_pg = {};
+        recovery_last_oid = {};
     }
     return false;
 }

src/osd_id.h

@@ -28,3 +28,13 @@ inline bool operator < (const pool_pg_num_t & a, const pool_pg_num_t & b)
 {
     return a.pool_id < b.pool_id || a.pool_id == b.pool_id && a.pg_num < b.pg_num;
 }
+
+inline bool operator == (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id == b.pool_id && a.pg_num == b.pg_num;
+}
+
+inline bool operator != (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id != b.pool_id || a.pg_num != b.pg_num;
+}

src/osd_peering.cpp

@@ -9,6 +9,8 @@
 #include "str_util.h"
 #include "osd.h"
 
+#define SELF_FD -1
+
 // Peering loop
 void osd_t::handle_peers()
 {
@@ -30,7 +32,16 @@ void osd_t::handle_peers()
                     if (p.second.state & PG_HAS_UNCLEAN)
                         peering_state = peering_state | OSD_FLUSHING_PGS;
                     else if (p.second.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
+                    {
                         peering_state = peering_state | OSD_RECOVERING;
+                        if (p.second.state & PG_HAS_DEGRADED)
+                        {
+                            // Restart recovery from degraded objects
+                            recovery_last_degraded = true;
+                            recovery_last_pg = {};
+                            recovery_last_oid = {};
+                        }
+                    }
                     ringloop->wakeup();
                     return;
                 }
@@ -300,82 +311,11 @@ void osd_t::start_pg_peering(pg_t & pg)
         {
             continue;
         }
-        submit_sync_and_list_subop(peer_osd, pg.peering_state);
+        submit_list_subop(peer_osd, pg.peering_state);
     }
     ringloop->wakeup();
 }
 
-void osd_t::submit_sync_and_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
-{
-    // Sync before listing, if not readonly
-    if (readonly)
-    {
-        submit_list_subop(role_osd, ps);
-    }
-    else if (role_osd == this->osd_num)
-    {
-        // Self
-        osd_op_t *op = new osd_op_t();
-        op->op_type = 0;
-        op->peer_fd = -1;
-        clock_gettime(CLOCK_REALTIME, &op->tv_begin);
-        op->bs_op = new blockstore_op_t();
-        op->bs_op->opcode = BS_OP_SYNC;
-        op->bs_op->callback = [this, ps, op, role_osd](blockstore_op_t *bs_op)
-        {
-            if (bs_op->retval < 0)
-            {
-                printf("Local OP_SYNC failed: %d (%s)\n", bs_op->retval, strerror(-bs_op->retval));
-                force_stop(1);
-                return;
-            }
-            add_bs_subop_stats(op);
-            delete op->bs_op;
-            op->bs_op = NULL;
-            delete op;
-            ps->list_ops.erase(role_osd);
-            submit_list_subop(role_osd, ps);
-        };
-        bs->enqueue_op(op->bs_op);
-        ps->list_ops[role_osd] = op;
-    }
-    else
-    {
-        // Peer
-        auto & cl = msgr.clients.at(msgr.osd_peer_fds.at(role_osd));
-        osd_op_t *op = new osd_op_t();
-        op->op_type = OSD_OP_OUT;
-        op->peer_fd = cl->peer_fd;
-        op->req = (osd_any_op_t){
-            .sec_sync = {
-                .header = {
-                    .magic = SECONDARY_OSD_OP_MAGIC,
-                    .id = msgr.next_subop_id++,
-                    .opcode = OSD_OP_SEC_SYNC,
-                },
-            },
-        };
-        op->callback = [this, ps, role_osd](osd_op_t *op)
-        {
-            if (op->reply.hdr.retval < 0)
-            {
-                // FIXME: Mark peer as failed and don't reconnect immediately after dropping the connection
-                printf("Failed to sync OSD %lu: %ld (%s), disconnecting peer\n", role_osd, op->reply.hdr.retval, strerror(-op->reply.hdr.retval));
-                int fail_fd = op->peer_fd;
-                ps->list_ops.erase(role_osd);
-                delete op;
-                msgr.stop_client(fail_fd);
-                return;
-            }
-            delete op;
-            ps->list_ops.erase(role_osd);
-            submit_list_subop(role_osd, ps);
-        };
-        msgr.outbox_push(op);
-        ps->list_ops[role_osd] = op;
-    }
-}
-
 void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
 {
     if (role_osd == this->osd_num)
@@ -383,7 +323,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
         // Self
         osd_op_t *op = new osd_op_t();
         op->op_type = 0;
-        op->peer_fd = -1;
+        op->peer_fd = SELF_FD;
         clock_gettime(CLOCK_REALTIME, &op->tv_begin);
         op->bs_op = new blockstore_op_t();
         op->bs_op->opcode = BS_OP_LIST;
@@ -415,8 +355,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
             op->bs_op = NULL;
             delete op;
         };
-        bs->enqueue_op(op->bs_op);
         ps->list_ops[role_osd] = op;
+        bs->enqueue_op(op->bs_op);
     }
     else
     {
@@ -463,14 +403,14 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
             ps->list_ops.erase(role_osd);
             delete op;
         };
-        msgr.outbox_push(op);
         ps->list_ops[role_osd] = op;
+        msgr.outbox_push(op);
     }
 }
 
 void osd_t::discard_list_subop(osd_op_t *list_op)
 {
-    if (list_op->peer_fd == 0)
+    if (list_op->peer_fd == SELF_FD)
     {
         // Self
         list_op->bs_op->callback = [list_op](blockstore_op_t *bs_op)
@@ -549,13 +489,17 @@ void osd_t::report_pg_state(pg_t & pg)
         pg.history_changed = true;
         pg.target_history.clear();
         pg.all_peers = pg.target_set;
+        std::sort(pg.all_peers.begin(), pg.all_peers.end());
         pg.cur_peers = pg.target_set;
     }
     else if (pg.state == (PG_ACTIVE|PG_LEFT_ON_DEAD))
     {
         // Clear history of active+left_on_dead PGs, but leave dead OSDs in all_peers
-        pg.history_changed = true;
-        pg.target_history.clear();
+        if (pg.target_history.size())
+        {
+            pg.history_changed = true;
+            pg.target_history.clear();
+        }
         std::set<osd_num_t> dead_peers;
         for (auto pg_osd: pg.all_peers)
         {
@@ -572,8 +516,12 @@ void osd_t::report_pg_state(pg_t & pg)
                 dead_peers.insert(pg_osd);
             }
         }
-        pg.all_peers.clear();
-        pg.all_peers.insert(pg.all_peers.begin(), dead_peers.begin(), dead_peers.end());
+        auto new_all_peers = std::vector<osd_num_t>(dead_peers.begin(), dead_peers.end());
+        if (pg.all_peers != new_all_peers)
+        {
+            pg.history_changed = true;
+            pg.all_peers = new_all_peers;
+        }
         pg.cur_peers.clear();
         for (auto pg_osd: pg.target_set)
         {

src/osd_peering_pg.cpp

@@ -86,18 +86,9 @@ void pg_obj_state_check_t::walk()
     }
     if (pg->pg_cursize < pg->pg_size)
     {
-        // Report PG history and activate
+        // Activate as degraded
+        // Current OSD set will be added into target_history on first write
         pg->state |= PG_DEGRADED | PG_PEERED;
-        std::vector<osd_num_t> history_set;
-        for (auto peer_osd: pg->cur_set)
-        {
-            if (peer_osd != 0)
-            {
-                history_set.push_back(peer_osd);
-            }
-        }
-        pg->target_history.push_back(history_set);
-        pg->history_changed = true;
     }
     else
     {
@@ -435,11 +426,35 @@ void pg_t::calc_object_states(int log_level)
     std::sort(st.list.begin(), st.list.end());
     // Walk over it and check object states
     st.walk();
-    if (this->state & (PG_DEGRADED|PG_LEFT_ON_DEAD))
+    if (this->state != PG_ACTIVE)
     {
         assert(epoch != (((uint64_t)1 << PG_EPOCH_BITS)-1));
         epoch++;
     }
+    if (log_level > 0)
+    {
+        std::string osd_set_desc;
+        for (auto & osd_num: target_set)
+        {
+            osd_set_desc += (osd_set_desc == "" ? "" : ", ")+std::to_string(osd_num);
+        }
+        printf(
+            "[PG %u/%u] %lu clean objects on target OSD set %s\n",
+            pool_id, pg_num, clean_count, osd_set_desc.c_str()
+        );
+        for (auto & stp: state_dict)
+        {
+            osd_set_desc = "";
+            for (auto & loc: stp.first)
+            {
+                osd_set_desc += (osd_set_desc == "" ? "" : ", ")+
+                    std::to_string(loc.osd_num)+
+                    (st.replicated ? "" : "("+std::to_string(loc.role)+")")+
+                    (loc.outdated ? "(old)" : "");
+            }
+            printf("[PG %u/%u] %lu objects on OSD set %s\n", pool_id, pg_num, stp.second.object_count, osd_set_desc.c_str());
+        }
+    }
 }
 
 void pg_t::print_state()

src/osd_primary.cpp

@@ -228,7 +228,7 @@ resume_1:
 resume_2:
     if (op_data->errors > 0)
     {
-        finish_op(cur_op, op_data->epipe > 0 ? -EPIPE : -EIO);
+        finish_op(cur_op, op_data->errcode);
         return;
     }
     cur_op->reply.rw.version = op_data->fact_ver;
@@ -350,7 +350,7 @@ resume_2:
 resume_3:
     if (op_data->errors > 0)
     {
-        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
+        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
         return;
     }
     // Check CAS version
@@ -371,7 +371,7 @@ resume_4:
 resume_5:
     if (op_data->errors > 0)
     {
-        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
+        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
         return;
     }
     // Remove version override

src/osd_primary.h

@@ -22,9 +22,9 @@ struct osd_primary_op_data_t
     pg_num_t pg_num;
     object_id oid;
     uint64_t target_ver;
-    uint64_t fact_ver = 0;
+    uint64_t orig_ver = 0, fact_ver = 0;
     uint64_t scheme = 0;
-    int n_subops = 0, done = 0, errors = 0, epipe = 0;
+    int n_subops = 0, done = 0, errors = 0, errcode = 0;
     int degraded = 0, pg_size, pg_data_size;
     osd_rmw_stripe_t *stripes;
     osd_op_t *subops = NULL;

src/osd_primary_chain.cpp

@@ -42,7 +42,7 @@ resume_4:
     {
         free(op_data->chain_reads);
         op_data->chain_reads = NULL;
-        finish_op(cur_op, op_data->epipe > 0 ? -EPIPE : -EIO);
+        finish_op(cur_op, op_data->errcode);
         return;
     }
     send_chained_read_results(pg, cur_op);

src/osd_primary_subops.cpp

@@ -122,7 +122,7 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
         zero_read = -1;
     osd_op_t *subops = new osd_op_t[n_subops];
     op_data->fact_ver = 0;
-    op_data->done = op_data->errors = 0;
+    op_data->done = op_data->errors = op_data->errcode = 0;
     op_data->n_subops = n_subops;
     op_data->subops = subops;
     int sent = submit_primary_subop_batch(submit_type, op_data->oid.inode, op_version, op_data->stripes, osd_set, cur_op, 0, zero_read);
@@ -263,9 +263,11 @@ void osd_t::handle_primary_bs_subop(osd_op_t *subop)
     blockstore_op_t *bs_op = subop->bs_op;
     int expected = bs_op->opcode == BS_OP_READ || bs_op->opcode == BS_OP_WRITE
         || bs_op->opcode == BS_OP_WRITE_STABLE ? bs_op->len : 0;
-    if (bs_op->retval != expected && bs_op->opcode != BS_OP_READ)
+    if (bs_op->retval != expected && bs_op->opcode != BS_OP_READ &&
+        (bs_op->opcode != BS_OP_WRITE && bs_op->opcode != BS_OP_WRITE_STABLE ||
+        bs_op->retval != -ENOSPC))
     {
-        // die
+        // die on any error except ENOSPC
         throw std::runtime_error(
             "local blockstore modification failed (opcode = "+std::to_string(bs_op->opcode)+
             " retval = "+std::to_string(bs_op->retval)+")"
@@ -276,6 +278,8 @@ void osd_t::handle_primary_bs_subop(osd_op_t *subop)
     subop->reply.hdr.retval = bs_op->retval;
     if (bs_op->opcode == BS_OP_READ || bs_op->opcode == BS_OP_WRITE || bs_op->opcode == BS_OP_WRITE_STABLE)
     {
+        subop->req.sec_rw.oid = bs_op->oid;
+        subop->req.sec_rw.version = bs_op->version;
         subop->req.sec_rw.len = bs_op->len;
         subop->reply.sec_rw.version = bs_op->version;
     }
@@ -337,14 +341,17 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
                 osd_op_names[opcode], subop->peer_fd, retval, expected
             );
         }
-        if (retval == -EPIPE)
+        // Error priority: EIO > ENOSPC > EPIPE
+        if (op_data->errcode == 0 || retval == -EIO ||
+            retval == -ENOSPC && op_data->errcode == -EPIPE)
         {
-            op_data->epipe++;
+            op_data->errcode = retval;
         }
         op_data->errors++;
-        if (subop->peer_fd >= 0)
+        if (subop->peer_fd >= 0 && (opcode != OSD_OP_SEC_WRITE && opcode != OSD_OP_SEC_WRITE_STABLE ||
+            retval != -ENOSPC))
         {
-            // Drop connection on any error
+            // Drop connection on any error expect ENOSPC
             msgr.stop_client(subop->peer_fd);
         }
     }
@@ -408,7 +415,8 @@ void osd_t::cancel_primary_write(osd_op_t *cur_op)
         // are sent to peer OSDs, so we can't just throw them away.
         // Mark them with an extra EPIPE.
         cur_op->op_data->errors++;
-        cur_op->op_data->epipe++;
+        if (cur_op->op_data->errcode == 0)
+            cur_op->op_data->errcode = -EPIPE;
         cur_op->op_data->done--; // Caution: `done` must be signed because may become -1 here
     }
     else
@@ -460,7 +468,7 @@ void osd_t::submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     op_data->n_subops = chunks_to_delete_count;
-    op_data->done = op_data->errors = 0;
+    op_data->done = op_data->errors = op_data->errcode = 0;
     if (!op_data->n_subops)
     {
         return;
@@ -523,7 +531,7 @@ int osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int n_osds = op_data->dirty_osd_count;
     osd_op_t *subops = new osd_op_t[n_osds];
-    op_data->done = op_data->errors = 0;
+    op_data->done = op_data->errors = op_data->errcode = 0;
     op_data->n_subops = n_osds;
     op_data->subops = subops;
     std::map<uint64_t, int>::iterator peer_it;
@@ -579,7 +587,7 @@ void osd_t::submit_primary_stab_subops(osd_op_t *cur_op)
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int n_osds = op_data->unstable_write_osds->size();
     osd_op_t *subops = new osd_op_t[n_osds];
-    op_data->done = op_data->errors = 0;
+    op_data->done = op_data->errors = op_data->errcode = 0;
     op_data->n_subops = n_osds;
     op_data->subops = subops;
     for (int i = 0; i < n_osds; i++)

src/osd_primary_sync.cpp

@@ -240,7 +240,7 @@ resume_8:
     }
     if (op_data->errors > 0)
     {
-        finish_op(cur_op, op_data->epipe > 0 ? -EPIPE : -EIO);
+        finish_op(cur_op, op_data->errcode);
     }
     else
     {

src/osd_primary_write.cpp

@@ -93,7 +93,7 @@ resume_2:
 resume_3:
     if (op_data->errors > 0)
     {
-        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
+        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
         return;
     }
     // Check CAS version
@@ -138,6 +138,7 @@ resume_3:
         }
     }
     // Send writes
+    op_data->orig_ver = op_data->fact_ver;
     if ((op_data->fact_ver >> (64-PG_EPOCH_BITS)) < pg.epoch)
     {
         op_data->target_ver = ((uint64_t)pg.epoch << (64-PG_EPOCH_BITS)) | 1;
@@ -154,17 +155,36 @@ resume_3:
     if (pg.epoch > pg.reported_epoch)
     {
         // Report newer epoch before writing
-        // FIXME: We may report only one PG state here...
+        // FIXME: We don't have to report all changed PG states here
         this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
-        pg.history_changed = true;
+        if (pg.state != PG_ACTIVE)
+        {
+            // Check that current OSD set is in history and/or add it there
+            std::vector<osd_num_t> history_set;
+            for (auto peer_osd: pg.cur_set)
+                if (peer_osd != 0)
+                    history_set.push_back(peer_osd);
+            std::sort(history_set.begin(), history_set.end());
+            auto it = std::lower_bound(pg.target_history.begin(), pg.target_history.end(), history_set);
+            if (it == pg.target_history.end() || *it != history_set)
+                pg.target_history.insert(it, history_set);
+            pg.history_changed = true;
+        }
         report_pg_states();
 resume_10:
         if (pg.epoch > pg.reported_epoch)
         {
-            op_data->st = 10;
+#define PG_EPOCH_WAIT_STATE 10
+            op_data->st = PG_EPOCH_WAIT_STATE;
             return;
         }
     }
+    // Recheck PG state after reporting history - maybe it's already stopping/restarting
+    if (pg.state & (PG_STOPPING|PG_REPEERING))
+    {
+        pg_cancel_write_queue(pg, cur_op, op_data->oid, -EPIPE);
+        return;
+    }
     submit_primary_subops(SUBMIT_WRITE, op_data->target_ver, pg.cur_set.data(), cur_op);
 resume_4:
     op_data->st = 4;
@@ -177,7 +197,7 @@ resume_5:
     }
     if (op_data->errors > 0)
     {
-        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
+        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
         return;
     }
     if (op_data->object_state)
@@ -194,7 +214,7 @@ resume_7:
     {
         return;
     }
-    if (op_data->fact_ver == 1)
+    if (op_data->orig_ver == 0)
     {
         // Object is created
         pg.clean_count++;
@@ -254,7 +274,7 @@ resume_8:
 resume_9:
                 if (op_data->errors > 0)
                 {
-                    pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
+                    pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
                     return;
                 }
             }
@@ -286,6 +306,50 @@ continue_others:
     }
 }
 
+void osd_t::on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num)
+{
+    auto pg_it = pgs.find({
+        .pool_id = pool_id,
+        .pg_num = pg_num,
+    });
+    if (pg_it == pgs.end())
+    {
+        return;
+    }
+    auto & pg = pg_it->second;
+    if (pg.epoch > pg.reported_epoch &&
+        st_cli.pool_config[pool_id].pg_config[pg_num].epoch >= pg.epoch)
+    {
+        pg.reported_epoch = st_cli.pool_config[pool_id].pg_config[pg_num].epoch;
+        std::vector<object_id> resume_oids;
+        for (auto & op: pg.write_queue)
+        {
+            if (op.second->op_data->st == PG_EPOCH_WAIT_STATE)
+            {
+                // Run separately to prevent side effects
+                resume_oids.push_back(op.first);
+            }
+        }
+        for (auto & oid: resume_oids)
+        {
+            auto pg_it = pgs.find({
+                .pool_id = pool_id,
+                .pg_num = pg_num,
+            });
+            if (pg_it != pgs.end())
+            {
+                auto & pg = pg_it->second;
+                auto op_it = pg.write_queue.find(oid);
+                if (op_it != pg.write_queue.end() &&
+                    op_it->second->op_data->st == PG_EPOCH_WAIT_STATE)
+                {
+                    continue_primary_write(op_it->second);
+                }
+            }
+        }
+    }
+}
+
 bool osd_t::remember_unstable_write(osd_op_t *cur_op, pg_t & pg, pg_osd_set_t & loc_set, int base_state)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
@@ -336,7 +400,7 @@ resume_7:
             op_data->unstable_write_osds = NULL;
             if (op_data->errors > 0)
             {
-                pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
+                pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
                 return false;
             }
         }

src/osd_rmw.cpp

@@ -154,6 +154,8 @@ struct reed_sol_matrix_t
     int refs = 0;
     int *je_data;
     uint8_t *isal_data;
+    // 32 bytes = 256/8 = max pg_size/8
+    std::map<std::array<uint8_t, 32>, void*> subdata;
     std::map<reed_sol_erased_t, void*> decodings;
 };
 
@@ -194,6 +196,12 @@ void use_ec(int pg_size, int pg_minsize, bool use)
         free(rs_it->second.je_data);
         if (rs_it->second.isal_data)
             free(rs_it->second.isal_data);
+        for (auto sub_it = rs_it->second.subdata.begin(); sub_it != rs_it->second.subdata.end();)
+        {
+            void *data = sub_it->second;
+            rs_it->second.subdata.erase(sub_it++);
+            free(data);
+        }
         for (auto dec_it = rs_it->second.decodings.begin(); dec_it != rs_it->second.decodings.end();)
         {
             void *data = dec_it->second;
@@ -294,6 +302,47 @@ static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size
     return dec_it->second;
 }
 
+#ifndef WITH_ISAL
+#define JERASURE_ALIGNMENT 16
+
+// jerasure requires 16-byte alignment for SSE...
+// FIXME: jerasure/gf-complete should probably be patched to automatically choose non-sse version for unaligned buffers
+static void jerasure_matrix_encode_unaligned(int k, int m, int w, int *matrix, char **data_ptrs, char **coding_ptrs, int size)
+{
+    bool unaligned = false;
+    for (int i = 0; i < k; i++)
+        if (((unsigned long)data_ptrs[i]) % JERASURE_ALIGNMENT)
+            unaligned = true;
+    for (int i = 0; i < m; i++)
+        if (((unsigned long)coding_ptrs[i]) % JERASURE_ALIGNMENT)
+            unaligned = true;
+    if (!unaligned)
+    {
+        jerasure_matrix_encode(k, m, w, matrix, data_ptrs, coding_ptrs, size);
+        return;
+    }
+    int aligned_size = ((size+JERASURE_ALIGNMENT-1)/JERASURE_ALIGNMENT)*JERASURE_ALIGNMENT;
+    int copy_size = aligned_size*(k+m);
+    char local_data[copy_size > 4096 ? 0 : copy_size];
+    char *data_copy = copy_size > 4096 || (unsigned long)local_data % JERASURE_ALIGNMENT
+        ? (char*)memalign_or_die(JERASURE_ALIGNMENT, aligned_size*(k+m))
+        : local_data;
+    char *aligned_ptrs[k+m];
+    for (int i = 0; i < k; i++)
+    {
+        memcpy(data_copy + i*aligned_size, data_ptrs[i], size);
+        aligned_ptrs[i] = data_copy + i*aligned_size;
+    }
+    for (int i = 0; i < m; i++)
+        aligned_ptrs[k+i] = data_copy + (k+i)*aligned_size;
+    jerasure_matrix_encode(k, m, w, matrix, aligned_ptrs, aligned_ptrs+k, size);
+    for (int i = 0; i < m; i++)
+        memcpy(coding_ptrs[i], aligned_ptrs[k+i], size);
+    if (copy_size > 4096 || (unsigned long)local_data % JERASURE_ALIGNMENT)
+        free(data_copy);
+}
+#endif
+
 #ifdef WITH_ISAL
 void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, uint32_t bitmap_size)
 {
@@ -357,10 +406,12 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi
     {
         data_ptrs[role] = NULL;
     }
+    bool recovered = false;
     for (int role = 0; role < pg_minsize; role++)
     {
         if (stripes[role].read_end != 0 && stripes[role].missing)
         {
+            recovered = true;
             if (stripes[role].read_end > stripes[role].read_start)
             {
                 for (int other = 0; other < pg_size; other++)
@@ -378,18 +429,64 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi
                     data_ptrs, data_ptrs+pg_minsize, stripes[role].read_end - stripes[role].read_start
                 );
             }
-            for (int other = 0; other < pg_size; other++)
+        }
+    }
+    if (recovered && bitmap_size > 0)
+    {
+        bool unaligned = false;
+        for (int role = 0; role < pg_size; role++)
+        {
+            if (stripes[role].read_end != 0)
             {
-                if (stripes[other].read_end != 0 && !stripes[other].missing)
+                data_ptrs[role] = (char*)stripes[role].bmp_buf;
+                if (((unsigned long)stripes[role].bmp_buf) % JERASURE_ALIGNMENT)
+                    unaligned = true;
+            }
+        }
+        if (!unaligned)
+        {
+            for (int role = 0; role < pg_minsize; role++)
+            {
+                if (stripes[role].read_end != 0 && stripes[role].missing)
                 {
-                    data_ptrs[other] = (char*)(stripes[other].bmp_buf);
+                    jerasure_matrix_dotprod(
+                        pg_minsize, OSD_JERASURE_W, decoding_matrix+(role*pg_minsize), dm_ids, role,
+                        data_ptrs, data_ptrs+pg_minsize, bitmap_size
+                    );
                 }
             }
-            data_ptrs[role] = (char*)stripes[role].bmp_buf;
-            jerasure_matrix_dotprod(
-                pg_minsize, OSD_JERASURE_W, decoding_matrix+(role*pg_minsize), dm_ids, role,
-                data_ptrs, data_ptrs+pg_minsize, bitmap_size
-            );
+        }
+        else
+        {
+            // jerasure_matrix_dotprod requires 16-byte alignment for SSE...
+            int aligned_size = ((bitmap_size+JERASURE_ALIGNMENT-1)/JERASURE_ALIGNMENT)*JERASURE_ALIGNMENT;
+            int copy_size = aligned_size*pg_size;
+            char local_data[copy_size > 4096 ? 0 : copy_size];
+            bool alloc_copy = copy_size > 4096 || (unsigned long)local_data % JERASURE_ALIGNMENT;
+            char *data_copy = alloc_copy
+                ? (char*)memalign_or_die(JERASURE_ALIGNMENT, copy_size)
+                : local_data;
+            for (int role = 0; role < pg_size; role++)
+            {
+                if (stripes[role].read_end != 0)
+                {
+                    data_ptrs[role] = data_copy + role*aligned_size;
+                    memcpy(data_ptrs[role], stripes[role].bmp_buf, bitmap_size);
+                }
+            }
+            for (int role = 0; role < pg_size; role++)
+            {
+                if (stripes[role].read_end != 0 && stripes[role].missing)
+                {
+                    jerasure_matrix_dotprod(
+                        pg_minsize, OSD_JERASURE_W, decoding_matrix+(role*pg_minsize), dm_ids, role,
+                        data_ptrs, data_ptrs+pg_minsize, bitmap_size
+                    );
+                    memcpy(stripes[role].bmp_buf, data_ptrs[role], bitmap_size);
+                }
+            }
+            if (alloc_copy)
+                free(data_copy);
         }
     }
 }
@@ -478,15 +575,18 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
     {
         if (write_osd_set[role] != 0)
         {
-            write_parity = 1;
+            write_parity++;
             if (write_osd_set[role] != read_osd_set[role])
             {
                 start = 0;
                 end = chunk_size;
                 for (int r2 = pg_minsize; r2 < role; r2++)
                 {
-                    stripes[r2].write_start = start;
-                    stripes[r2].write_end = end;
+                    if (write_osd_set[r2] != 0)
+                    {
+                        stripes[r2].write_start = start;
+                        stripes[r2].write_end = end;
+                    }
                 }
             }
             stripes[role].write_start = start;
@@ -555,7 +655,7 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
         }
     }
     // Allocate read buffers
-    void *rmw_buf = alloc_read_buffer(stripes, pg_size, (write_parity ? pg_size-pg_minsize : 0) * (end - start));
+    void *rmw_buf = alloc_read_buffer(stripes, pg_size, write_parity * (end - start));
     // Position write buffers
     uint64_t buf_pos = 0, in_pos = 0;
     for (int role = 0; role < pg_size; role++)
@@ -804,14 +904,57 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
     calc_rmw_parity_copy_mod(stripes, pg_size, pg_minsize, read_osd_set, write_osd_set, chunk_size, bitmap_granularity, start, end);
     if (end != 0)
     {
-        int i;
-        for (i = pg_minsize; i < pg_size; i++)
+        int write_parity = 0;
+        bool is_seq = true;
+        for (int i = pg_size-1; i >= pg_minsize; i--)
         {
             if (write_osd_set[i] != 0)
-                break;
+                write_parity++;
+            else if (write_parity != 0)
+                is_seq = false;
         }
-        if (i < pg_size)
+        if (write_parity > 0)
         {
+            // First get the coding matrix or sub-matrix
+            void *matrix_data =
+#ifdef WITH_ISAL
+                matrix->isal_data;
+#else
+                matrix->je_data;
+#endif
+            if (!is_seq)
+            {
+                // We need a coding sub-matrix
+                std::array<uint8_t, 32> missing_parity = {};
+                for (int i = pg_minsize; i < pg_size; i++)
+                {
+                    if (!write_osd_set[i])
+                        missing_parity[(i-pg_minsize) >> 3] |= (1 << ((i-pg_minsize) & 0x7));
+                }
+                auto sub_it = matrix->subdata.find(missing_parity);
+                if (sub_it == matrix->subdata.end())
+                {
+                    int item_size =
+#ifdef WITH_ISAL
+                        32;
+#else
+                        sizeof(int);
+#endif
+                    void *subm = malloc_or_die(item_size * write_parity * pg_minsize);
+                    for (int i = pg_minsize, j = 0; i < pg_size; i++)
+                    {
+                        if (write_osd_set[i])
+                        {
+                            memcpy(subm + item_size*pg_minsize*j, matrix_data + item_size*pg_minsize*(i-pg_minsize), item_size*pg_minsize);
+                            j++;
+                        }
+                    }
+                    matrix->subdata[missing_parity] = subm;
+                    matrix_data = subm;
+                }
+                else
+                    matrix_data = sub_it->second;
+            }
             // Calculate new coding chunks
             buf_len_t bufs[pg_size][3];
             int nbuf[pg_size], curbuf[pg_size];
@@ -830,57 +973,67 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
             }
             for (int i = pg_minsize; i < pg_size; i++)
             {
-                bufs[i][nbuf[i]++] = { .buf = stripes[i].write_buf, .len = end-start };
-                positions[i] = start;
+                if (write_osd_set[i] != 0)
+                {
+                    bufs[i][nbuf[i]++] = { .buf = stripes[i].write_buf, .len = end-start };
+                    positions[i] = start;
+                }
             }
             uint32_t pos = start;
             while (pos < end)
             {
                 uint32_t next_end = end;
-                for (int i = 0; i < pg_size; i++)
+                for (int i = 0, j = 0; i < pg_size; i++)
                 {
-                    assert(curbuf[i] < nbuf[i]);
-                    assert(bufs[i][curbuf[i]].buf);
-                    data_ptrs[i] = (uint8_t*)bufs[i][curbuf[i]].buf + pos-positions[i];
-                    uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
-                    if (next_end > this_end)
-                        next_end = this_end;
+                    if (i < pg_minsize || write_osd_set[i] != 0)
+                    {
+                        assert(curbuf[i] < nbuf[i]);
+                        assert(bufs[i][curbuf[i]].buf);
+                        data_ptrs[j++] = (uint8_t*)bufs[i][curbuf[i]].buf + pos-positions[i];
+                        uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
+                        if (next_end > this_end)
+                            next_end = this_end;
+                    }
                 }
                 assert(next_end > pos);
                 for (int i = 0; i < pg_size; i++)
                 {
-                    uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
-                    if (next_end >= this_end)
+                    if (i < pg_minsize || write_osd_set[i] != 0)
                     {
-                        positions[i] += bufs[i][curbuf[i]].len;
-                        curbuf[i]++;
+                        uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
+                        if (next_end >= this_end)
+                        {
+                            positions[i] += bufs[i][curbuf[i]].len;
+                            curbuf[i]++;
+                        }
                     }
                 }
 #ifdef WITH_ISAL
                 ec_encode_data(
-                    next_end-pos, pg_minsize, pg_size-pg_minsize, matrix->isal_data,
+                    next_end-pos, pg_minsize, write_parity, (uint8_t*)matrix_data,
                     (uint8_t**)data_ptrs, (uint8_t**)data_ptrs+pg_minsize
                 );
 #else
                 jerasure_matrix_encode(
-                    pg_minsize, pg_size-pg_minsize, OSD_JERASURE_W, matrix->je_data,
+                    pg_minsize, write_parity, OSD_JERASURE_W, (int*)matrix_data,
                     (char**)data_ptrs, (char**)data_ptrs+pg_minsize, next_end-pos
                 );
 #endif
                 pos = next_end;
             }
-            for (int i = 0; i < pg_size; i++)
+            for (int i = 0, j = 0; i < pg_size; i++)
             {
-                data_ptrs[i] = stripes[i].bmp_buf;
+                if (i < pg_minsize || write_osd_set[i] != 0)
+                    data_ptrs[j++] = stripes[i].bmp_buf;
             }
 #ifdef WITH_ISAL
             ec_encode_data(
-                bitmap_size, pg_minsize, pg_size-pg_minsize, matrix->isal_data,
+                bitmap_size, pg_minsize, write_parity, (uint8_t*)matrix_data,
                 (uint8_t**)data_ptrs, (uint8_t**)data_ptrs+pg_minsize
             );
 #else
-            jerasure_matrix_encode(
-                pg_minsize, pg_size-pg_minsize, OSD_JERASURE_W, matrix->je_data,
+            jerasure_matrix_encode_unaligned(
+                pg_minsize, write_parity, OSD_JERASURE_W, (int*)matrix_data,
                 (char**)data_ptrs, (char**)data_ptrs+pg_minsize, bitmap_size
             );
 #endif

src/osd_rmw_test.cpp

@@ -3,6 +3,10 @@
 
 #define RMW_DEBUG
 
+#ifdef NO_ISAL
+#undef WITH_ISAL
+#endif
+
 #include <string.h>
 #include "osd_rmw.cpp"
 #include "test_pattern.h"
@@ -20,6 +24,8 @@ void test11();
 void test12();
 void test13();
 void test14();
+void test15();
+void test16();
 
 int main(int narg, char *args[])
 {
@@ -47,6 +53,10 @@ int main(int narg, char *args[])
     test13();
     // Test 14
     test14();
+    // Test 15
+    test15();
+    // Test 16
+    test16();
     // End
     printf("all ok\n");
     return 0;
@@ -706,7 +716,7 @@ void test13()
 
 /***
 
-13. basic jerasure 2+1 test
+14. basic jerasure 2+1 test
    calc_rmw(offset=128K-4K, len=8K, osd_set=[1,2,0], write_set=[1,2,3])
    = {
      read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ] ],
@@ -727,13 +737,13 @@ void test14()
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
     osd_rmw_stripe_t stripes[3] = {};
     unsigned bitmaps[3] = { 0 };
-    // Test 13.0
+    // Test 14.0
     void *write_buf = malloc_or_die(8192);
     split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
     assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
     assert(stripes[1].req_start == 0 && stripes[1].req_end == 4096);
     assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
-    // Test 13.1
+    // Test 14.1
     void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, bmp);
     for (int i = 0; i < 3; i++)
         stripes[i].bmp_buf = bitmaps+i;
@@ -750,7 +760,7 @@ void test14()
     assert(stripes[0].write_buf == write_buf);
     assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
-    // Test 13.2 - encode
+    // Test 14.2 - encode
     set_pattern(write_buf, 8192, PATTERN3);
     set_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
     set_pattern(stripes[1].read_buf, 128*1024-4096, PATTERN2);
@@ -767,7 +777,7 @@ void test14()
     assert(stripes[0].write_buf == write_buf);
     assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
-    // Test 13.3 - decode and verify
+    // Test 14.3 - decode and verify
     osd_num_t read_osd_set[4] = { 0, 2, 3 };
     memset(stripes, 0, sizeof(stripes));
     split_stripes(2, 128*1024, 0, 128*1024, stripes);
@@ -802,3 +812,177 @@ void test14()
     free(write_buf);
     use_ec(3, 2, false);
 }
+
+/***
+
+15. EC 2+2 partial overwrite with 1 missing stripe
+   calc_rmw(offset=64K+28K, len=4K, osd_set=[1,2,3,0], write_set=[1,2,3,0])
+   = {
+     read: [ [ 28K, 32K ], [ 0, 0 ], [ 0, 0 ], [ 0, 0 ] ],
+     write: [ [ 0, 0 ], [ 28K, 32K ], [ 28K, 32K ], [ 0, 0 ] ],
+     input buffer: [ write1 ],
+     rmw buffer: [ write2, read0 ],
+   }
+
+***/
+
+void test15()
+{
+    const int bmp = 64*1024 / 4096 / 8;
+    use_ec(4, 2, true);
+    osd_num_t osd_set[4] = { 1, 2, 3, 0 };
+    osd_num_t write_osd_set[4] = { 1, 2, 3, 0 };
+    osd_rmw_stripe_t stripes[4] = {};
+    unsigned bitmaps[4] = { 0 };
+    // Test 15.0
+    void *write_buf = malloc_or_die(4096);
+    split_stripes(2, 64*1024, (64+28)*1024, 4096, stripes);
+    assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
+    assert(stripes[1].req_start == 28*1024 && stripes[1].req_end == 32*1024);
+    assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
+    assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
+    // Test 15.1
+    void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 4, 2, 3, write_osd_set, 64*1024, bmp);
+    for (int i = 0; i < 4; i++)
+        stripes[i].bmp_buf = bitmaps+i;
+    assert(rmw_buf);
+    assert(stripes[0].read_start == 28*1024 && stripes[0].read_end == 32*1024);
+    assert(stripes[1].read_start == 0 && stripes[1].read_end == 0);
+    assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
+    assert(stripes[3].read_start == 0 && stripes[3].read_end == 0);
+    assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
+    assert(stripes[1].write_start == 28*1024 && stripes[1].write_end == 32*1024);
+    assert(stripes[2].write_start == 28*1024 && stripes[2].write_end == 32*1024);
+    assert(stripes[3].write_start == 0 && stripes[3].write_end == 0);
+    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+4*1024);
+    assert(stripes[1].read_buf == NULL);
+    assert(stripes[2].read_buf == NULL);
+    assert(stripes[3].read_buf == NULL);
+    assert(stripes[0].write_buf == NULL);
+    assert(stripes[1].write_buf == (uint8_t*)write_buf);
+    assert(stripes[2].write_buf == rmw_buf);
+    assert(stripes[3].write_buf == NULL);
+    // Test 15.2 - encode
+    set_pattern(write_buf, 4*1024, PATTERN1);
+    set_pattern(stripes[0].read_buf, 4*1024, PATTERN2);
+    memset(stripes[0].bmp_buf, 0, bmp);
+    memset(stripes[1].bmp_buf, 0, bmp);
+    calc_rmw_parity_ec(stripes, 4, 2, osd_set, write_osd_set, 64*1024, bmp);
+    assert(*(uint32_t*)stripes[2].bmp_buf == 0x80);
+    assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
+    assert(stripes[1].write_start == 28*1024 && stripes[1].write_end == 32*1024);
+    assert(stripes[2].write_start == 28*1024 && stripes[2].write_end == 32*1024);
+    assert(stripes[3].write_start == 0 && stripes[3].write_end == 0);
+    assert(stripes[0].write_buf == NULL);
+    assert(stripes[1].write_buf == (uint8_t*)write_buf);
+    assert(stripes[2].write_buf == rmw_buf);
+    assert(stripes[3].write_buf == NULL);
+    check_pattern(stripes[2].write_buf, 4*1024, PATTERN1^PATTERN2); // first parity is always xor :)
+    // Done
+    free(rmw_buf);
+    free(write_buf);
+    use_ec(3, 2, false);
+}
+
+/***
+
+16. EC 2+2 write one parity block with another missing
+   calc_rmw(offset=0, len=0, osd_set=[1,2,0,0], write_set=[1,2,0,3])
+   = {
+     read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ], [ 0, 0 ] ],
+     write: [ [ 0, 0 ], [ 0, 0 ], [ 0, 0 ], [ 0, 128K ] ],
+     input buffer: [],
+     rmw buffer: [ write3, read0, read1 ],
+   }
+
+***/
+
+void test16()
+{
+    const int bmp = 128*1024 / 4096 / 8;
+    use_ec(4, 2, true);
+    osd_num_t osd_set[4] = { 1, 2, 0, 0 };
+    osd_num_t write_osd_set[4] = { 1, 2, 0, 3 };
+    osd_rmw_stripe_t stripes[4] = {};
+    unsigned bitmaps[4] = { 0 };
+    // Test 16.0
+    void *write_buf = NULL;
+    split_stripes(2, 128*1024, 0, 0, stripes);
+    assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
+    assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
+    assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
+    assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
+    // Test 16.1
+    void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 4, 2, 3, write_osd_set, 128*1024, bmp);
+    for (int i = 0; i < 4; i++)
+        stripes[i].bmp_buf = bitmaps+i;
+    assert(rmw_buf);
+    assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
+    assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
+    assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
+    assert(stripes[3].read_start == 0 && stripes[3].read_end == 0);
+    assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
+    assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
+    assert(stripes[2].write_start == 0 && stripes[2].write_end == 0);
+    assert(stripes[3].write_start == 0 && stripes[3].write_end == 128*1024);
+    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+256*1024);
+    assert(stripes[2].read_buf == NULL);
+    assert(stripes[3].read_buf == NULL);
+    assert(stripes[0].write_buf == NULL);
+    assert(stripes[1].write_buf == NULL);
+    assert(stripes[2].write_buf == NULL);
+    assert(stripes[3].write_buf == rmw_buf);
+    // Test 16.2 - encode
+    set_pattern(stripes[0].read_buf, 128*1024, PATTERN1);
+    set_pattern(stripes[1].read_buf, 128*1024, PATTERN2);
+    memset(stripes[0].bmp_buf, 0xff, bmp);
+    memset(stripes[1].bmp_buf, 0xff, bmp);
+    calc_rmw_parity_ec(stripes, 4, 2, osd_set, write_osd_set, 128*1024, bmp);
+    assert(*(uint32_t*)stripes[2].bmp_buf == 0);
+    assert(*(uint32_t*)stripes[3].bmp_buf == 0xF1F1F1F1);
+    assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
+    assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
+    assert(stripes[2].write_start == 0 && stripes[2].write_end == 0);
+    assert(stripes[3].write_start == 0 && stripes[3].write_end == 128*1024);
+    assert(stripes[0].write_buf == NULL);
+    assert(stripes[1].write_buf == NULL);
+    assert(stripes[2].write_buf == NULL);
+    assert(stripes[3].write_buf == rmw_buf);
+    check_pattern(stripes[3].write_buf, 128*1024, 0x7eb9ae9cd8e652c3); // 2nd EC chunk
+    // Test 16.3 - decode and verify
+    osd_num_t read_osd_set[4] = { 0, 2, 0, 3 };
+    memset(stripes, 0, sizeof(stripes));
+    split_stripes(2, 128*1024, 0, 256*1024, stripes);
+    assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
+    assert(stripes[1].req_start == 0 && stripes[1].req_end == 128*1024);
+    assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
+    assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
+    for (int role = 0; role < 4; role++)
+    {
+        stripes[role].read_start = stripes[role].req_start;
+        stripes[role].read_end = stripes[role].req_end;
+    }
+    assert(extend_missing_stripes(stripes, read_osd_set, 2, 4) == 0);
+    assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
+    assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
+    assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
+    assert(stripes[3].read_start == 0 && stripes[3].read_end == 128*1024);
+    void *read_buf = alloc_read_buffer(stripes, 4, 0);
+    for (int i = 0; i < 4; i++)
+        stripes[i].bmp_buf = bitmaps+i;
+    assert(read_buf);
+    assert(stripes[0].read_buf == read_buf);
+    assert(stripes[1].read_buf == (uint8_t*)read_buf+128*1024);
+    assert(stripes[3].read_buf == (uint8_t*)read_buf+2*128*1024);
+    set_pattern(stripes[1].read_buf, 128*1024, PATTERN2);
+    memcpy(stripes[3].read_buf, rmw_buf, 128*1024);
+    reconstruct_stripes_ec(stripes, 4, 2, bmp);
+    assert(bitmaps[0] == 0xFFFFFFFF);
+    check_pattern(stripes[0].read_buf, 128*1024, PATTERN1);
+    free(read_buf);
+    // Done
+    free(rmw_buf);
+    free(write_buf);
+    use_ec(3, 2, false);
+}

src/qemu_driver.c

@@ -206,6 +206,25 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
     }
 }
 
+static void vitastor_aio_set_fd_handler(void *ctx, int fd, int unused1, IOHandler *fd_read, IOHandler *fd_write, void *unused2, void *opaque)
+{
+    aio_set_fd_handler(ctx, fd,
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 5 || QEMU_VERSION_MAJOR >= 3
+        0 /*is_external*/,
+#endif
+        fd_read, fd_write,
+#if QEMU_VERSION_MAJOR == 1 && QEMU_VERSION_MINOR <= 6 || QEMU_VERSION_MAJOR < 1
+        NULL /*io_flush*/,
+#endif
+#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 9 || QEMU_VERSION_MAJOR >= 3
+        NULL /*io_poll*/,
+#endif
+#if QEMU_VERSION_MAJOR >= 7
+        NULL /*io_poll_ready*/,
+#endif
+        opaque);
+}
+
 static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, Error **errp)
 {
     VitastorClient *client = bs->opaque;
@@ -221,7 +240,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
     client->rdma_gid_index = qdict_get_try_int(options, "rdma-gid-index", 0);
     client->rdma_mtu = qdict_get_try_int(options, "rdma-mtu", 0);
     client->proxy = vitastor_c_create_qemu(
-        (QEMUSetFDHandler*)aio_set_fd_handler, bdrv_get_aio_context(bs), client->config_path, client->etcd_host, client->etcd_prefix,
+        vitastor_aio_set_fd_handler, bdrv_get_aio_context(bs), client->config_path, client->etcd_host, client->etcd_prefix,
         client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0
     );
     client->image = g_strdup(qdict_get_try_str(options, "image"));
@@ -238,9 +257,9 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
         }
         else
         {
-            qemu_coroutine_enter(qemu_coroutine_create((void(*)(void*))vitastor_co_get_metadata, &task));
+            bdrv_coroutine_enter(bs, qemu_coroutine_create((void(*)(void*))vitastor_co_get_metadata, &task));
+            BDRV_POLL_WHILE(bs, !task.complete);
         }
-        BDRV_POLL_WHILE(bs, !task.complete);
         client->watch = (void*)task.ret;
         client->readonly = client->readonly || vitastor_c_inode_get_readonly(client->watch);
         client->size = vitastor_c_inode_get_size(client->watch);
@@ -428,7 +447,13 @@ static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version)
     vitastor_co_generic_bh_cb(opaque, retval);
 }
 
-static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *iov, int flags)
+static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
+#if QEMU_VERSION_MAJOR >= 7 || QEMU_VERSION_MAJOR == 6 && QEMU_VERSION_MINOR >= 2
+    int64_t offset, int64_t bytes, QEMUIOVector *iov, BdrvRequestFlags flags
+#else
+    uint64_t offset, uint64_t bytes, QEMUIOVector *iov, int flags
+#endif
+)
 {
     VitastorClient *client = bs->opaque;
     VitastorRPC task;
@@ -448,7 +473,13 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs, uint64_t offset
     return task.ret;
 }
 
-static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *iov, int flags)
+static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
+#if QEMU_VERSION_MAJOR >= 7 || QEMU_VERSION_MAJOR == 6 && QEMU_VERSION_MINOR >= 2
+    int64_t offset, int64_t bytes, QEMUIOVector *iov, BdrvRequestFlags flags
+#else
+    uint64_t offset, uint64_t bytes, QEMUIOVector *iov, int flags
+#endif
+)
 {
     VitastorClient *client = bs->opaque;
     VitastorRPC task;

src/str_util.cpp

@@ -56,6 +56,16 @@ std::string base64_decode(const std::string &in)
     return out;
 }
 
+std::string strtoupper(const std::string & in)
+{
+    std::string s = in;
+    for (int i = 0; i < s.length(); i++)
+    {
+        s[i] = toupper(s[i]);
+    }
+    return s;
+}
+
 std::string strtolower(const std::string & in)
 {
     std::string s = in;

src/str_util.h

@@ -8,6 +8,7 @@
 std::string base64_encode(const std::string &in);
 std::string base64_decode(const std::string &in);
 uint64_t parse_size(std::string size_str, bool *ok = NULL);
+std::string strtoupper(const std::string & in);
 std::string strtolower(const std::string & in);
 std::string trim(const std::string & in, const char *rm_chars = " \n\r\t");
 std::string str_replace(const std::string & in, const std::string & needle, const std::string & replacement);

src/test_crc32.cpp

@@ -0,0 +1,28 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdint.h>
+
+#include "malloc_or_die.h"
+#include "errno.h"
+#include "crc32c.h"
+
+int main(int narg, char *args[])
+{
+    int bufsize = 65536;
+    uint8_t *buf = (uint8_t*)malloc_or_die(bufsize);
+    uint32_t csum = 0;
+    while (1)
+    {
+        int r = read(0, buf, bufsize);
+        if (r <= 0 && errno != EAGAIN && errno != EINTR)
+            break;
+        csum = crc32c(csum, buf, r);
+    }
+    free(buf);
+    printf("%08x\n", csum);
+    return 0;
+}

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.8.0
+Version: 0.8.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/vitastor_c.h

@@ -24,6 +24,7 @@ typedef void VitastorIOHandler(void *opaque, long retval);
 
 // QEMU
 typedef void IOHandler(void *opaque);
+// is_external and poll_fn are not required, but are here for compatibility
 typedef void QEMUSetFDHandler(void *ctx, int fd, int is_external, IOHandler *fd_read, IOHandler *fd_write, void *poll_fn, void *opaque);
 
 vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,

tests/run_3osds.sh

@@ -61,15 +61,31 @@ fi
 POOLCFG='"name":"testpool","failure_domain":"osd",'$POOLCFG
 $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}'
 
-sleep 2
+wait_up()
+{
+    local sec=$1
+    local i=0
+    local configured=0
+    while [[ $i -lt $sec ]]; do
+        if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] |
+            select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT; then
+            configured=1
+            if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then
+                break
+            fi
+        fi
+        sleep 1
+        i=$((i+1))
+        if [ $i -eq $sec ]; then
+            if [[ $configured -ne 0 ]]; then
+                format_error "FAILED: $PG_COUNT PG(s) NOT CONFIGURED"
+            fi
+            format_error "FAILED: $PG_COUNT PG(s) NOT UP"
+        fi
+    done
+}
 
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == '$PG_SIZE') | length) == '$PG_COUNT); then
-    format_error "FAILED: $PG_COUNT PGS NOT CONFIGURED"
-fi
-
-if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT); then
-    format_error "FAILED: $PG_COUNT PGS NOT UP"
-fi
+wait_up 60
 
 try_reweight()
 {

tests/test_add_osd.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -ex
 
-PG_COUNT=16
+PG_COUNT=2048
 
 . `dirname $0`/run_3osds.sh
 
@@ -10,8 +10,7 @@ LD_PRELOAD="build/src/libfio_vitastor.so" \
 
 for i in 4; do
     dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-disk simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
-    eval OSD${i}_PID=$!
+    start_osd $i
 done
 
 sleep 2
@@ -33,4 +32,28 @@ if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '(
     format_error "FAILED: $PG_COUNT PGS NOT ACTIVE"
 fi
 
+sleep 1
+kill -9 $OSD4_PID
+sleep 1
+build/src/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4
+
+sleep 2
+
+for i in {1..10}; do
+    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+        jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \
+        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \
+        break
+    sleep 1
+done
+
+if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then
+    format_error "FAILED: OSD NOT REMOVED FROM DISTRIBUTION"
+fi
+
+if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT''); then
+    format_error "FAILED: $PG_COUNT PGS NOT ACTIVE"
+fi
+
 format_green OK