Support reading parameters automatically from the superblock in vitastor-disk {dump,write}-{meta,journal}

OSDs could crash with the following "assertion failed" message (crash didn't affect data
and was caused by OSD thinking upper blocks are full while they weren't). Reproduction
without introducing artificial delays is hard because you have to force OSD to read an
object with enqueued but not handled write which fills previously non-full bitmap. O_o.

```
vitastor-osd: ./src/osd/osd_primary_chain.cpp:613: void osd_t::send_chained_read_results(pg_t&, osd_op_t*): Assertion `stripes[role].read_buf' failed.
```

.gitea/workflows/test.yml

@@ -828,6 +828,24 @@ jobs:
           echo ""
         done
 
+  test_snapshot_pool2:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_pool2.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_osd_tags:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.9.0")
+set(VITASTOR_VERSION "1.9.1")
 
 add_subdirectory(src)

README-ru.md

@@ -1,4 +1,4 @@
-## Vitastor
+# Vitastor
 
 [Read English version](README.md)
 
@@ -22,7 +22,7 @@ TCP и RDMA и на хорошем железе может достигать з
 Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, OpenNebula, Proxmox, Kubernetes.
 Другие драйверы могут также быть легко реализованы.
 
-Подробности смотрите в документации по ссылкам ниже.
+Подробности смотрите в документации по ссылкам. Можете начать отсюда: [Быстрый старт](docs/intro/quickstart.ru.md).
 
 ## Презентации и записи докладов
 
@@ -51,7 +51,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
   - Параметры
     - [Общие](docs/config/common.ru.md)
     - [Сетевые](docs/config/network.ru.md)
-    - [Клиентский код](docs/config/client.en.md)
+    - [Клиентский код](docs/config/client.ru.md)
     - [Глобальные дисковые параметры](docs/config/layout-cluster.ru.md)
     - [Дисковые параметры OSD](docs/config/layout-osd.ru.md)
     - [Прочие параметры OSD](docs/config/osd.ru.md)

README.md

@@ -22,7 +22,7 @@ or internal systems of public clouds.
 Vitastor supports QEMU, NBD, NFS protocols, OpenStack, OpenNebula, Proxmox, Kubernetes drivers.
 More drivers may be created easily.
 
-Read more details below in the documentation.
+Read more details in the documentation. You can start from here: [Quick Start](docs/intro/quickstart.en.md).
 
 ## Talks and presentations
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.9.0
+VITASTOR_VERSION ?= v1.9.1
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.9.0
+          image: vitalif/vitastor-csi:v1.9.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.9.0
+          image: vitalif/vitastor-csi:v1.9.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.9.0"
+    vitastorCSIDriverVersion = "1.9.1"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.9.0-1) unstable; urgency=medium
+vitastor (1.9.1-1) unstable; urgency=medium
 
   * Bugfixes
 

docs/config/layout-cluster.en.md

@@ -106,8 +106,8 @@ SSD cache or "media-cache" - for example, a lot of Seagate EXOS drives have
 it (they have internal SSD cache even though it's not stated in datasheets).
 
 Setting this parameter to "all" or "small" in OSD parameters requires enabling
-[disable_journal_fsync](layout-osd.en.yml#disable_journal_fsync) and
-[disable_meta_fsync](layout-osd.en.yml#disable_meta_fsync), setting it to
-"all" also requires enabling [disable_data_fsync](layout-osd.en.yml#disable_data_fsync).
+[disable_journal_fsync](layout-osd.en.md#disable_journal_fsync) and
+[disable_meta_fsync](layout-osd.en.md#disable_meta_fsync), setting it to
+"all" also requires enabling [disable_data_fsync](layout-osd.en.md#disable_data_fsync).
 vitastor-disk tried to do that by default, first checking/disabling drive cache.
 If it can't disable drive cache, OSD get initialized with "none".

docs/config/layout-cluster.ru.md

@@ -112,6 +112,6 @@ HDD-дисках с внутренним SSD или "медиа" кэшем - н
 указано в спецификациях).
 
 Указание "all" или "small" в настройках / командной строке OSD требует
-включения [disable_journal_fsync](layout-osd.ru.yml#disable_journal_fsync) и
-[disable_meta_fsync](layout-osd.ru.yml#disable_meta_fsync), значение "all"
-также требует включения [disable_data_fsync](layout-osd.ru.yml#disable_data_fsync).
+включения [disable_journal_fsync](layout-osd.ru.md#disable_journal_fsync) и
+[disable_meta_fsync](layout-osd.ru.md#disable_meta_fsync), значение "all"
+также требует включения [disable_data_fsync](layout-osd.ru.md#disable_data_fsync).

docs/config/pool.en.md

@@ -55,7 +55,7 @@ Examples:
 OSD placement tree is set in a separate etcd key `/vitastor/config/node_placement`
 in the following JSON format:
 
-`
+```
 {
   "<node name or OSD number>": {
     "level": "<level>",
@@ -63,7 +63,7 @@ in the following JSON format:
   },
   ...
 }
-`
+```
 
 Here, if a node name is a number then it is assumed to refer to an OSD.
 Level of the OSD is always "osd" and cannot be overriden. You may only

docs/config/pool.ru.md

@@ -54,7 +54,7 @@
 Дерево размещения OSD задаётся в отдельном ключе etcd `/vitastor/config/node_placement`
 в следующем JSON-формате:
 
-`
+```
 {
   "<имя узла или номер OSD>": {
     "level": "<уровень>",
@@ -62,7 +62,7 @@
   },
   ...
 }
-`
+```
 
 Здесь, если название узла - число, считается, что это OSD. Уровень OSD
 всегда равен "osd" и не может быть переопределён. Для OSD вы можете только

docs/config/src/layout-cluster.yml

@@ -97,9 +97,9 @@
     it (they have internal SSD cache even though it's not stated in datasheets).
 
     Setting this parameter to "all" or "small" in OSD parameters requires enabling
-    [disable_journal_fsync](layout-osd.en.yml#disable_journal_fsync) and
-    [disable_meta_fsync](layout-osd.en.yml#disable_meta_fsync), setting it to
-    "all" also requires enabling [disable_data_fsync](layout-osd.en.yml#disable_data_fsync).
+    [disable_journal_fsync](layout-osd.en.md#disable_journal_fsync) and
+    [disable_meta_fsync](layout-osd.en.md#disable_meta_fsync), setting it to
+    "all" also requires enabling [disable_data_fsync](layout-osd.en.md#disable_data_fsync).
     vitastor-disk tried to do that by default, first checking/disabling drive cache.
     If it can't disable drive cache, OSD get initialized with "none".
   info_ru: |
@@ -156,6 +156,6 @@
     указано в спецификациях).
 
     Указание "all" или "small" в настройках / командной строке OSD требует
-    включения [disable_journal_fsync](layout-osd.ru.yml#disable_journal_fsync) и
-    [disable_meta_fsync](layout-osd.ru.yml#disable_meta_fsync), значение "all"
-    также требует включения [disable_data_fsync](layout-osd.ru.yml#disable_data_fsync).
+    включения [disable_journal_fsync](layout-osd.ru.md#disable_journal_fsync) и
+    [disable_meta_fsync](layout-osd.ru.md#disable_meta_fsync), значение "all"
+    также требует включения [disable_data_fsync](layout-osd.ru.md#disable_data_fsync).

docs/installation/opennebula.en.md

@@ -4,6 +4,8 @@
 
 [Читать на русском](opennebula.ru.md)
 
+# OpenNebula
+
 ## Automatic Installation
 
 OpenNebula plugin is packaged as `vitastor-opennebula` Debian and RPM package since Vitastor 1.9.0. So:

docs/installation/opennebula.ru.md

@@ -4,6 +4,8 @@
 
 [Read in English](opennebula.en.md)
 
+# OpenNebula
+
 ## Автоматическая установка
 
 Плагин OpenNebula Vitastor распространяется как Debian и RPM пакет `vitastor-opennebula`, начиная с версии Vitastor 1.9.0. Так что:

docs/usage/admin.en.md

@@ -169,6 +169,18 @@ Upgrading is performed without stopping clients (VMs/containers), you just need
 upgrade and restart servers one by one. However, ideally you should restart VMs too
 to make them use the new version of the client library.
 
+### 1.7.x to 1.8.0
+
+After upgrading version <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
+(VMs and so on), otherwise they will hang when monitor clears old PG configuration key,
+which happens 24 hours after upgrade.
+
+This is fixed in 1.9.1. So, after upgrading version <= 1.7.x directly to version >= 1.9.1,
+you DO NOT have to restart all old clients immediately - they will work like before until
+you decide to upgrade them too. The downside is that you'll have to remove the old PG
+configuration key (`/vitastor/config/pgs`) from etcd by hand when you make sure that all
+your clients are restarted.
+
 ### 1.1.x to 1.2.0
 
 Upgrading version <= 1.1.x to version >= 1.2.0, if you use EC n+k with k>=2, is recommended

docs/usage/admin.ru.md

@@ -166,6 +166,18 @@ done
 достаточно обновлять серверы по одному. Однако, конечно, чтобы запущенные виртуальные машины
 начали использовать новую версию клиентской библиотеки, их тоже нужно перезапустить.
 
+### 1.7.x -> 1.8.0
+
+После обновления с версий <= 1.7.x до версий >= 1.8.0, НО <= 1.9.0: перезапустите всех
+клиентов (процессы виртуальных машин можно перезапустить путём миграции на другой сервер),
+иначе они зависнут, когда монитор удалит старый ключ конфигурации PG, что происходит через
+24 часа после обновления.
+
+Однако, это исправлено в 1.9.1. Так что, если вы обновляетесь с <= 1.7.x сразу до >= 1.9.1,
+вам НЕ нужно сразу перезапускать всех клиентов - они будут работать, как раньше. Минус,
+правда, в том, что старый ключ конфигурации PG (`/vitastor/config/pgs`) будет нужно удалить
+вам из etcd вручную - после того, как вы убедитесь, что все клиенты перезапущены.
+
 ### 1.1.x -> 1.2.0
 
 Обновляться с версий <= 1.1.x до версий >= 1.2.0, если вы используете EC n+k и k>=2,

mon/mon.js

@@ -218,6 +218,10 @@ class Mon
             }
             else if (key != '/stats' && key.substr(0, 13) != '/inode/stats/')
             {
+                if (key == '/config/pgs' && !kv.value)
+                {
+                    this.old_pg_config = false;
+                }
                 changed = true;
             }
             if (this.config.verbose)
@@ -422,30 +426,6 @@ class Mon
                     throw new Error('Failed to duplicate old PG config to new PG config');
             }
             this.old_pg_config = true;
-            this.old_pg_config_timer = setInterval(() => this.check_clear_old_config().catch(console.error),
-                this.config.old_pg_config_clear_interval||3600000);
-        }
-    }
-
-    async check_clear_old_config()
-    {
-        if (this.old_pg_config && this.old_pg_stats_seen)
-        {
-            this.old_pg_stats_seen = false;
-            return;
-        }
-        if (this.old_pg_config)
-        {
-            await this.etcd.etcd_call('/kv/txn', { success: [
-                { requestDeleteRange: { key: b64(this.config.etcd_prefix+'/config/pgs') } },
-                { requestDeleteRange: { key: b64(this.config.etcd_prefix+'/pg/stats/'), range_end: b64(this.config.etcd_prefix+'/pg/stats0') } },
-            ] }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
-            this.old_pg_config = false;
-        }
-        if (this.old_pg_config_timer)
-        {
-            clearInterval(this.old_pg_config_timer);
-            this.old_pg_config_timer = null;
         }
     }
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

opennebula/install.sh

@@ -3,7 +3,9 @@
 set -e
 
 reapply_patch() {
-	if ! patch -f --dry-run -F 0 -R $1 < $2 >/dev/null; then
+	if ! [[ -e $1 ]]; then
+		echo "$1 does not exist, OpenNebula is not installed"
+	elif ! patch -f --dry-run -F 0 -R $1 < $2 >/dev/null; then
 		already_applied=0
 		if ! patch --no-backup-if-mismatch -r - -F 0 -f $1 < $2; then
 			applied_ok=0
@@ -15,8 +17,13 @@ echo "Reapplying Vitastor patches to OpenNebula's oned.conf, vmm_execrc and down
 already_applied=1
 applied_ok=1
 reapply_patch /var/lib/one/remotes/datastore/downloader.sh /var/lib/one/remotes/datastore/vitastor/downloader-vitastor.sh.diff
-reapply_patch /etc/one/oned.conf /var/lib/one/remotes/datastore/vitastor/oned.conf.diff
 reapply_patch /etc/one/vmm_exec/vmm_execrc /var/lib/one/remotes/datastore/vitastor/vmm_execrc.diff
+if [[ -e /etc/one/oned.conf ]]; then
+	if ! /var/lib/one/remotes/datastore/vitastor/patch-oned-conf.py /etc/one/oned.conf; then
+		applied_ok=0
+		already_applied=0
+	fi
+fi
 if [[ "$already_applied" = 1 ]]; then
 	echo "OK: Vitastor OpenNebula patches are already applied"
 elif [[ "$applied_ok" = 1 ]]; then

opennebula/remotes/datastore/vitastor/oned.conf.diff

@@ -54,20 +54,20 @@ index be02d646a8..27f876ec36 100644
  TM_MAD_CONF = [
      NAME = "iscsi_libvirt", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
      DS_MIGRATE = "NO", DRIVER = "raw"
-@@ -1219,9 +1230,16 @@ DS_MAD_CONF = [
+@@ -1263,9 +1274,16 @@ DS_MAD_CONF = [
      NAME = "ceph",
      REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
      PERSISTENT_ONLY = "NO",
      MARKETPLACE_ACTIONS = "export"
- ]
- 
++]
++
 +DS_MAD_CONF = [
 +    NAME = "vitastor",
 +    REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
 +    PERSISTENT_ONLY = "NO",
 +    MARKETPLACE_ACTIONS = "export"
-+]
-+
+ ]
+ 
  DS_MAD_CONF = [
      NAME = "dev", REQUIRED_ATTRS = "DISK_TYPE", PERSISTENT_ONLY = "YES"
  ]

opennebula/remotes/datastore/vitastor/patch-oned-conf.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+# Patch /etc/one/oned.conf for Vitastor support
+# -s = also enable save.vitastor/restore.vitastor overrides
+
+import re
+import os
+import sys
+
+class Fixer:
+    save_restore = 0
+
+    def require_sub_cb(self, m, cb):
+        self.found = 1
+        return cb(m)
+
+    def require_sub(self, regexp, cb, text, error):
+        self.found = 0
+        new_text = re.sub(regexp, lambda m: self.require_sub_cb(m, cb), text)
+        if not self.found and error:
+            self.errors.append(error)
+        return new_text
+
+    def fix(self, oned_conf):
+        self.errors = []
+        self.kvm_found = 0
+        oned_conf = self.require_sub(r'((?:^|\n)[ \t]*VM_MAD\s*=\s*\[)([^\]]+)\]', lambda m: m.group(1)+self.fix_vm_mad(m.group(2))+']', oned_conf, 'VM_MAD not found')
+        if not self.kvm_found:
+            self.errors.append("VM_MAD[NAME=kvm].ARGUMENTS not found")
+        oned_conf = self.require_sub(r'((?:^|\n)[ \t]*TM_MAD\s*=\s*\[)([^\]]+)\]', lambda m: m.group(1)+self.fix_tm_mad(m.group(2))+']', oned_conf, 'TM_MAD not found')
+        oned_conf = self.require_sub(r'((?:^|\n)[ \t]*DATASTORE_MAD\s*=\s*\[)([^\]]+)\]', lambda m: m.group(1)+self.fix_datastore_mad(m.group(2))+']', oned_conf, 'DATASTORE_MAD not found')
+        if oned_conf[-1:] != '\n':
+            oned_conf += '\n'
+        if not re.compile(r'(^|\n)[ \t]*INHERIT_DATASTORE_ATTR\s*=\s*"VITASTOR_CONF"').search(oned_conf):
+            oned_conf += '\nINHERIT_DATASTORE_ATTR="VITASTOR_CONF"\n'
+        if not re.compile(r'(^|\n)[ \t]*INHERIT_DATASTORE_ATTR\s*=\s*"IMAGE_PREFIX"').search(oned_conf):
+            oned_conf += '\nINHERIT_DATASTORE_ATTR="IMAGE_PREFIX"\n'
+        if not re.compile(r'(^|\n)[ \t]*TM_MAD_CONF\s*=\s*\[[^\]]*NAME\s*=\s*"vitastor"').search(oned_conf):
+            oned_conf += ('\nTM_MAD_CONF = [\n'+
+                '    NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",\n'+
+                '    DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",\n'+
+                '    TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",\n'+
+                '    DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",\n'+
+                '    CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"\n'+
+                ']\n')
+        if not re.compile(r'(^|\n)[ \t]*DS_MAD_CONF\s*=\s*\[[^\]]*NAME\s*=\s*"vitastor"').search(oned_conf):
+            oned_conf += ('\nDS_MAD_CONF = [\n'+
+                '    NAME = "vitastor",\n'+
+                '    REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",\n'+
+                '    PERSISTENT_ONLY = "NO",\n'+
+                '    MARKETPLACE_ACTIONS = "export"\n'+
+                ']\n')
+        return oned_conf
+
+    def fix_vm_mad(self, vm_mad_params):
+        if re.compile(r'\bNAME\s*=\s*"kvm"').search(vm_mad_params):
+            vm_mad_params = re.sub(r'\b(ARGUMENTS\s*=\s*")([^"]+)"', lambda m: m.group(1)+self.fix_vm_mad_args(m.group(2))+'"', vm_mad_params)
+            self.kvm_found = 1
+        return vm_mad_params
+
+    def fix_vm_mad_args(self, args):
+        args = self.fix_vm_mad_override(args, 'deploy')
+        if self.save_restore:
+            args = self.fix_vm_mad_override(args, 'save')
+            args = self.fix_vm_mad_override(args, 'restore')
+        return args
+
+    def fix_vm_mad_override(self, args, override):
+        m = re.compile(r'-l (\S+)').search(args)
+        if m and re.compile(override+'='+override+'.vitastor').search(m.group(1)):
+            return args
+        elif m and re.compile(override+'=').search(m.group(1)):
+            self.errors.append(override+"= is already overridden in -l option in VM_MAD[NAME=kvm].ARGUMENTS")
+            return args
+        elif m:
+            return self.require_sub(r'-l (\S+)', lambda m: '-l '+m.group(1)+','+override+'='+override+'.vitastor', args, '-l option not found in VM_MAD[NAME=kvm].ARGUMENTS')
+        else:
+            return args+' -l '+override+'='+override+'.vitastor'
+
+    def fix_tm_mad(self, params):
+        return self.require_sub(r'\b(ARGUMENTS\s*=\s*")([^"]+)"', lambda m: m.group(1)+self.fix_tm_mad_args('d', m.group(2), "TM_MAD")+'"', params, "TM_MAD.ARGUMENTS not found")
+
+    def fix_tm_mad_args(self, opt, args, v):
+        return self.require_sub('(-'+opt+r') (\S+)', lambda m: self.fix_tm_mad_arg(m), args, "-"+opt+" option not found in "+v+".ARGUMENTS")
+
+    def fix_tm_mad_arg(self, m):
+        a = m.group(2).split(',')
+        if 'vitastor' not in a:
+            a += [ 'vitastor' ]
+        return m.group(1)+' '+(','.join(a))
+
+    def fix_datastore_mad(self, params):
+        params = self.require_sub(r'\b(ARGUMENTS\s*=\s*")([^"]+)"', lambda m: m.group(1)+self.fix_tm_mad_args('d', m.group(2), "DATASTORE_MAD")+'"', params, "DATASTORE_MAD.ARGUMENTS not found")
+        return self.require_sub(r'\b(ARGUMENTS\s*=\s*")([^"]+)"', lambda m: m.group(1)+self.fix_tm_mad_args('s', m.group(2), "DATASTORE_MAD")+'"', params, "")
+
+fixer = Fixer()
+oned_conf_file = ''
+for arg in sys.argv[1:]:
+    if arg == '-s':
+        fixer.save_restore = 1
+    else:
+        oned_conf_file = arg
+        break
+if not oned_conf_file:
+    sys.stderr.write("USAGE: ./patch-oned-conf.py [-s] /etc/one/oned.conf\n-s means also enable save.vitastor/restore.vitastor overrides\n")
+    sys.exit(1)
+with open(oned_conf_file, 'r') as fd:
+    oned_conf = fd.read()
+new_conf = fixer.fix(oned_conf)
+if new_conf != oned_conf:
+    os.rename(oned_conf_file, oned_conf_file+'.bak')
+    with open(oned_conf_file, 'w') as fd:
+        fd.write(new_conf)
+if len(fixer.errors) > 0:
+    sys.stderr.write("ERROR: Failed to patch "+oned_conf_file+", patch it manually. Errors:\n- "+('\n- '.join(fixer.errors))+'\n')
+    sys.exit(1)

opennebula/remotes/vmm/kvm/deploy.vitastor

@@ -4,12 +4,15 @@
 # Copyright (c) Vitaliy Filippov, 2024+
 # License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
 
+set -e
+
 DRIVER_PATH=$(dirname $0)
 DEP_FILE=$1
 DEP_FILE_LOCATION=$(dirname $DEP_FILE)
+HOST=$2
 
 cat > $DEP_FILE
 
 python3 $DRIVER_PATH/deploy_vitastor.py $DEP_FILE $DEP_FILE_LOCATION/vm.xml
 
-cat $DEP_FILE | $DRIVER_PATH/deploy $@
+cat "$DEP_FILE" | ssh "$HOST" "'${SCRIPTS_REMOTE_DIR:-/var/tmp/one}/vmm/kvm/deploy' '$DEP_FILE'"

opennebula/remotes/vmm/kvm/deploy_vitastor.py

@@ -54,5 +54,4 @@ for disk in dep.findall('./devices/disk[@type="file"]'):
     except Exception as e:
         print("Error: {}".format(e), file=stderr)
 
-if changed:
-    ET.ElementTree(dep).write(dep_file)
+ET.ElementTree(dep).write(dep_file)

opennebula/remotes/vmm/kvm/restore.vitastor

@@ -10,7 +10,7 @@ source $DRIVER_PATH/../../etc/vmm/kvm/kvmrc
 source $DRIVER_PATH/../../scripts_common.sh
 
 FILE=$1
-HOST=$2
+DST_HOST=$2
 DEPLOY_ID=$3
 VM_ID=$4
 DS_ID=$5
@@ -33,7 +33,8 @@ fi
 
 SRC_IMAGE="${IMAGE_PREFIX}-sys-${VM_ID}-checkpoint"
 
-exec_and_log "$CLI dd iimg=$SRC_IMAGE of=$FILE" "Error exporting checkpoint into from $SRC_IMAGE to $FILE"
-exec_and_log "$CLI rm $SRC_IMAGE" "Error removing checkpoint $SRC_IMAGE"
+ssh_exec_and_log "$DST_HOST" "$CLI dd iimg=$SRC_IMAGE of=$FILE" "Error exporting checkpoint into from $SRC_IMAGE to $FILE"
+ssh_exec_and_log "$DST_HOST" "$CLI rm $SRC_IMAGE" "Error removing checkpoint $SRC_IMAGE"
 
-"$DRIVER_PATH"/restore $@
+set -e
+ssh "$DST_HOST" "'${SCRIPTS_REMOTE_DIR:-/var/tmp/one}/vmm/kvm/restore' $@"

opennebula/remotes/vmm/kvm/save.vitastor

@@ -11,11 +11,14 @@ source $DRIVER_PATH/../../scripts_common.sh
 
 DEPLOY_ID=$1
 FILE=$2
+SRC_HOST=$3
 VM_ID=$4
 DS_ID=$5
 
-rm -f "$FILE"
-"$DRIVER_PATH"/save $@
+set -e
+ssh "$SRC_HOST" 'rm -f "$FILE"'
+ssh "$SRC_HOST" "'${SCRIPTS_REMOTE_DIR:-/var/tmp/one}/vmm/kvm/save' $@"
+set +e
 
 XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
 
@@ -42,7 +45,7 @@ fi
 
 DST_IMAGE="${IMAGE_PREFIX}-sys-${VM_ID}-checkpoint"
 
-exec_and_log "$CLI dd if=$FILE oimg=$DST_IMAGE conv=trunc" "Error importing checkpoint into $DST_IMAGE"
-exec_and_log "$RM -f $FILE" "Error removing checkpoint ($FILE)"
+ssh_exec_and_log "$SRC_HOST" "$CLI dd if=$FILE oimg=$DST_IMAGE conv=trunc" "Error importing checkpoint into $DST_IMAGE"
+ssh_exec_and_log "$SRC_HOST" "$RM -f $FILE" "Error removing checkpoint ($FILE)"
 
 exit 0

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.9.0'
+VITASTOR_VERSION = '1.9.1'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.0
+Version:        1.9.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.0.el7.tar.gz
+Source0:        vitastor-1.9.1.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.0
+Version:        1.9.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.0.el8.tar.gz
+Source0:        vitastor-1.9.1.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.0
+Version:        1.9.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.0.el9.tar.gz
+Source0:        vitastor-1.9.1.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.9.0")
+add_definitions(-DVITASTOR_VERSION="1.9.1")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_read.cpp

@@ -993,7 +993,8 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
     {
         while (dirty_it->first.oid == oid)
         {
-            if (target_version >= dirty_it->first.version)
+            // Condition has to be the same as in dequeue_read()
+            if (!IS_IN_FLIGHT(dirty_it->second.state) && target_version >= dirty_it->first.version)
             {
                 if (result_version)
                     *result_version = dirty_it->first.version;

src/client/CMakeLists.txt

@@ -10,7 +10,7 @@ endif (IBVERBS_LIBRARIES)
 add_library(vitastor_common STATIC
 	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
 	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
-	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ${MSGR_RDMA}
+	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA}
 )
 target_link_libraries(vitastor_common pthread)
 target_compile_options(vitastor_common PUBLIC -fPIC)

src/client/cluster_client.cpp

@@ -955,7 +955,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
             ? (stripe + pg_block_size) : (op->offset + op->len);
         op->parts[i].iov.reset();
         op->parts[i].flags = 0;
-        if (op->cur_inode != op->inode || op->opcode == OSD_OP_READ && dirty_copied)
+        if (op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->cur_inode != op->inode || op->opcode == OSD_OP_READ && dirty_copied)
         {
             // Read remaining parts from upper layers
             uint64_t prev = begin, cur = begin;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.9.0
+Version: 1.9.1
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_dd.cpp

@@ -369,6 +369,7 @@ struct cli_dd_t
 {
     cli_tool_t *parent;
 
+    std::vector<std::string> conv, iflag, oflag;
     dd_in_info_t iinfo;
     dd_out_info_t oinfo;
 
@@ -766,6 +767,49 @@ struct cli_dd_t
             goto resume_3;
         else if (state == 4)
             goto resume_4;
+        for (int i = 0; i < conv.size(); i++)
+        {
+            if (conv[i] == "nofsync")
+                oinfo.end_fsync = false;
+            else if (conv[i] == "trunc")
+                oinfo.out_trunc = true;
+            else if (conv[i] == "nocreat")
+                oinfo.out_create = false;
+            else if (conv[i] == "noerror")
+                ignore_errors = true;
+            else if (conv[i] == "nosparse")
+                write_zero = true;
+            else
+            {
+                result = (cli_result_t){ .err = EINVAL, .text = "Unknown option conv="+conv[i] };
+                state = 100;
+                return;
+            }
+        }
+        for (int i = 0; i < iflag.size(); i++)
+        {
+            if (iflag[i] == "direct")
+                iinfo.in_direct = true;
+            else
+            {
+                result = (cli_result_t){ .err = EINVAL, .text = "Unknown option iflag="+iflag[i] };
+                state = 100;
+                return;
+            }
+        }
+        for (int i = 0; i < oflag.size(); i++)
+        {
+            if (oflag[i] == "direct")
+                oinfo.out_direct = true;
+            else if (oflag[i] == "append")
+                oinfo.out_append = true;
+            else
+            {
+                result = (cli_result_t){ .err = EINVAL, .text = "Unknown option oflag="+oflag[i] };
+                state = 100;
+                return;
+            }
+        }
         if ((oinfo.oimg != "" && oinfo.ofile != "") || (iinfo.iimg != "" && iinfo.ifile != ""))
         {
             result = (cli_result_t){ .err = EINVAL, .text = "Image and file can't be specified at the same time" };
@@ -908,6 +952,18 @@ static uint64_t parse_blocks(json11::Json v, uint64_t bs, uint64_t def)
     return res;
 }
 
+static std::vector<std::string> explode_json(const std::string & sep, json11::Json opt)
+{
+    if (opt.is_array())
+    {
+        std::vector<std::string> arr;
+        for (auto & item: opt.array_items())
+            arr.push_back(item.as_string());
+        return arr;
+    }
+    return explode(sep, opt.as_string(), true);
+}
+
 std::function<bool(cli_result_t &)> cli_tool_t::start_dd(json11::Json cfg)
 {
     auto dd = new cli_dd_t();
@@ -923,7 +979,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_dd(json11::Json cfg)
     dd->oseek = parse_blocks(cfg["oseek"], dd->blocksize, 0);
     if (!dd->oseek)
         dd->oseek = parse_blocks(cfg["seek"], dd->blocksize, 0);
-    dd->iseek = parse_blocks(cfg["oseek"], dd->blocksize, 0);
+    dd->iseek = parse_blocks(cfg["iseek"], dd->blocksize, 0);
     if (!dd->iseek)
         dd->iseek = parse_blocks(cfg["skip"], dd->blocksize, 0);
     dd->iodepth = cfg["iodepth"].uint64_value();
@@ -935,25 +991,9 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_dd(json11::Json cfg)
         progress = true;
     dd->iinfo.detect_size = cfg["size"].is_null();
     dd->oinfo.out_size = parse_size(cfg["size"].as_string());
-    std::vector<std::string> conv = explode(",", cfg["conv"].string_value(), true);
-    if (std::find(conv.begin(), conv.end(), "nofsync") != conv.end())
-        dd->oinfo.end_fsync = false;
-    if (std::find(conv.begin(), conv.end(), "trunc") != conv.end())
-        dd->oinfo.out_trunc = true;
-    if (std::find(conv.begin(), conv.end(), "nocreat") != conv.end())
-        dd->oinfo.out_create = false;
-    if (std::find(conv.begin(), conv.end(), "noerror") != conv.end())
-        dd->ignore_errors = true;
-    if (std::find(conv.begin(), conv.end(), "nosparse") != conv.end())
-        dd->write_zero = true;
-    conv = explode(",", cfg["iflag"].string_value(), true);
-    if (std::find(conv.begin(), conv.end(), "direct") != conv.end())
-        dd->iinfo.in_direct = true;
-    conv = explode(",", cfg["oflag"].string_value(), true);
-    if (std::find(conv.begin(), conv.end(), "direct") != conv.end())
-        dd->oinfo.out_direct = true;
-    if (std::find(conv.begin(), conv.end(), "append") != conv.end())
-        dd->oinfo.out_append = true;
+    dd->conv = explode_json(",", cfg["conv"]);
+    dd->iflag = explode_json(",", cfg["iflag"]);
+    dd->oflag = explode_json(",", cfg["oflag"]);
     return [dd](cli_result_t & result)
     {
         dd->loop();

src/disk_tool/CMakeLists.txt

@@ -6,7 +6,7 @@ project(vitastor)
 add_executable(vitastor-disk
 	disk_tool.cpp disk_simple_offsets.cpp
 	disk_tool_journal.cpp disk_tool_meta.cpp disk_tool_prepare.cpp disk_tool_resize.cpp disk_tool_udev.cpp disk_tool_utils.cpp disk_tool_upgrade.cpp
-	../util/crc32c.c ../util/str_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp
+	../util/crc32c.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp
 )
 target_link_libraries(vitastor-disk
 	tcmalloc_minimal

src/disk_tool/disk_tool.cpp

@@ -143,8 +143,10 @@ static const char *help_text =
     "  For now, this only checks that device cache is in write-through mode if fsync is disabled.\n"
     "  Intended for use from startup scripts (i.e. from systemd units).\n"
     "\n"
+    "vitastor-disk dump-journal [OPTIONS] <osd_device>\n"
     "vitastor-disk dump-journal [OPTIONS] <journal_file> <journal_block_size> <offset> <size>\n"
-    "  Dump journal in human-readable or JSON (if --json is specified) format.\n"
+    "  Dump journal in text or JSON (if --json is specified) format.\n"
+    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
     "  Options:\n"
     "  --all             Scan the whole journal area for entries and dump them, even outdated ones\n"
     "  --json            Dump journal in JSON format\n"
@@ -152,16 +154,21 @@ static const char *help_text =
     "  --format data     Same as \"entries\", but also include small write data\n"
     "  --format blocks   Dump as an array of journal blocks each containing array of entries\n"
     "\n"
+    "vitastor-disk write-journal <osd_device>\n"
     "vitastor-disk write-journal <journal_file> <journal_block_size> <bitmap_size> <offset> <size>\n"
     "  Write journal from JSON taken from standard input in the same format as produced by\n"
     "  `dump-journal --json --format data`.\n"
+    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
     "\n"
+    "vitastor-disk dump-meta <osd_device>\n"
     "vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
     "  Dump metadata in JSON format.\n"
+    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
     "\n"
+    "vitastor-disk write-meta <osd_device>\n"
     "vitastor-disk write-meta <meta_file> <offset> <size>\n"
-    "  Write metadata from JSON taken from standard input in the same format as produced by\n"
-    "  `dump-meta`. Intended for debugging.\n"
+    "  Write metadata from JSON taken from standard input in the same format as produced by `dump-meta`.\n"
+    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
     "\n"
     "vitastor-disk simple-offsets <device>\n"
     "  Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n"
@@ -249,29 +256,49 @@ int main(int argc, char *argv[])
     }
     if (!strcmp(cmd[0], "dump-journal"))
     {
-        if (cmd.size() < 5)
+        if (cmd.size() != 2 && cmd.size() < 5)
         {
             print_help(help_text, aliased ? "vitastor-dump-journal" : "vitastor-disk", cmd[0], false);
             return 1;
         }
         self.dsk.journal_device = cmd[1];
-        self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10);
-        self.dsk.journal_offset = strtoull(cmd[3], NULL, 10);
-        self.dsk.journal_len = strtoull(cmd[4], NULL, 10);
+        if (cmd.size() > 2)
+        {
+            self