Add Hugo-based (https://gohugo.io) documentation

Bugfix release, would be 1.7.2, but etcd layout changes mandate it to be 1.8.0. :-)

- Change etcd layout: /config/pgs is now /pg/config, /pg/stats/* is now /pgstats/*.
  This is required to fix a rare PG history tracking issue caused by non-atomic
  delivery of etcd events sometimes resulting in `incomplete` objects in EC pools
  after mass OSD restarts. Upgrading can be performed freely, downgrade requires
  additional action: [1.8.0 to 1.7.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/usage/admin.en.md#1-8-0-to-1-7-1)
- Fix a rare client hang on PG primary OSD switch
- Fix vitastor-nfs started using mount command sometimes not stopping automatically after unmount
- Fix vitastor-nfs mounts started using mount command sometimes hanging after daemonizing
- Fix merge/flatten into a pool with different object size (image migration between pools case)
- Do not print extra "PG disappeared after reload" verbose log messages for non-existing PGs
- Fix clustered Antietcd support and persistence filter
- Do not try to purge the same OSD multiple times if its multiple devices are passed to purge
- Various node.js binding fixes

.gitea/workflows/test.yml

@@ -558,6 +558,24 @@ jobs:
           echo ""
         done
 
+  test_dd:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_dd.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_root_node:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.7.1")
+set(VITASTOR_VERSION "1.8.0")
 
 add_subdirectory(src)

README-ru.md

@@ -19,7 +19,7 @@ Vitastor нацелен в первую очередь на SSD и SSD+HDD кл
 TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
 что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
 
-Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, Proxmox, Kubernetes.
+Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, OpenNebula, Proxmox, Kubernetes.
 Другие драйверы могут также быть легко реализованы.
 
 Подробности смотрите в документации по ссылкам ниже.
@@ -42,6 +42,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
 - Установка
   - [Пакеты](docs/installation/packages.ru.md)
   - [Proxmox](docs/installation/proxmox.ru.md)
+  - [OpenNebula](docs/installation/opennebula.ru.md)
   - [OpenStack](docs/installation/openstack.ru.md)
   - [Kubernetes CSI](docs/installation/kubernetes.ru.md)
   - [Сборка из исходных кодов](docs/installation/source.ru.md)

README.md

@@ -19,7 +19,7 @@ supports TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1
 with proper hardware which is ~10 times faster than other popular SDS's like Ceph
 or internal systems of public clouds.
 
-Vitastor supports QEMU, NBD, NFS protocols, OpenStack, Proxmox, Kubernetes drivers.
+Vitastor supports QEMU, NBD, NFS protocols, OpenStack, OpenNebula, Proxmox, Kubernetes drivers.
 More drivers may be created easily.
 
 Read more details below in the documentation.
@@ -42,6 +42,7 @@ Read more details below in the documentation.
 - Installation
   - [Packages](docs/installation/packages.en.md)
   - [Proxmox](docs/installation/proxmox.en.md)
+  - [OpenNebula](docs/installation/opennebula.en.md)
   - [OpenStack](docs/installation/openstack.en.md)
   - [Kubernetes CSI](docs/installation/kubernetes.en.md)
   - [Building from Source](docs/installation/source.en.md)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.7.1
+VITASTOR_VERSION ?= v1.8.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.7.1
+          image: vitalif/vitastor-csi:v1.8.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.7.1
+          image: vitalif/vitastor-csi:v1.8.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.7.1"
+    vitastorCSIDriverVersion = "1.8.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.7.1-1) unstable; urgency=medium
+vitastor (1.8.0-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/control

@@ -53,3 +53,9 @@ Architecture: amd64
 Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version})
 Description: Vitastor Proxmox Virtual Environment storage plugin
  Vitastor storage plugin for Proxmox Virtual Environment.
+
+Package: vitastor-opennebula
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client, patch, python3, jq
+Description: Vitastor OpenNebula storage plugin
+ Vitastor storage plugin for OpenNebula.

debian/vitastor-opennebula.install

@@ -0,0 +1,3 @@
+opennebula/remotes var/lib/one/
+opennebula/sudoers.d etc/
+opennebula/install.sh var/lib/one/remotes/datastore/vitastor/

debian/vitastor-opennebula.postinst

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+if [ "$1" = "configure" ]; then
+	/var/lib/one/remotes/datastore/vitastor/install.sh
+fi

debian/vitastor-opennebula.triggers

@@ -0,0 +1,4 @@
+interest /var/lib/one/remotes/datastore/downloader.sh
+interest /etc/one/oned.conf
+interest /etc/one/vmm_exec/vmm_execrc
+interest /etc/apparmor.d/local/abstractions/libvirt-qemu

docs/gen-docs.js

@@ -0,0 +1,55 @@
+#!/usr/bin/nodejs
+
+const fs = require('fs');
+const yaml = require('yaml');
+
+const L = {
+    en: {},
+    ru: {
+        Type: 'Тип',
+        Default: 'Значение по умолчанию',
+        Minimum: 'Минимальное значение',
+    },
+};
+const types = {
+    en: {
+        string: 'string',
+        bool: 'boolean',
+        int: 'integer',
+        sec: 'seconds',
+        ms: 'milliseconds',
+        us: 'microseconds',
+    },
+    ru: {
+        string: 'строка',
+        bool: 'булево (да/нет)',
+        int: 'целое число',
+        sec: 'секунды',
+        ms: 'миллисекунды',
+        us: 'микросекунды',
+    },
+};
+const params_files = fs.readdirSync(__dirname+'/params')
+    .filter(f => f.substr(-4) == '.yml')
+    .map(f => f.substr(0, f.length-4));
+
+for (const file of params_files)
+{
+    const cfg = yaml.parse(fs.readFileSync(__dirname+'/params/'+file+'.yml', { encoding: 'utf-8' }));
+    for (const lang in types)
+    {
+        let out = '\n\n{{< toc >}}';
+        for (const c of cfg)
+        {
+            out += `\n\n## ${c.name}\n\n`;
+            out += `- ${L[lang]['Type'] || 'Type'}: ${c["type_"+lang] || types[lang][c.type] || c.type}\n`;
+            if (c.default !== undefined)
+                out += `- ${L[lang]['Default'] || 'Default'}: ${c.default}\n`;
+            if (c.min !== undefined)
+                out += `- ${L[lang]['Minimum'] || 'Minimum'}: ${c.min}\n`;
+            out += `\n`+(c["info_"+lang] || c["info"]).replace(/\s+$/, '');
+        }
+        const head = fs.readFileSync(__dirname+'/params/head/'+file+'.'+lang+'.md', { encoding: 'utf-8' });
+        fs.writeFileSync(__dirname+'/hugo/content/config/'+file+'.'+lang+'.md', head.replace(/\s+$/, '')+out+"\n");
+    }
+}

docs/installation/opennebula.en.md

@@ -0,0 +1,184 @@
+[Documentation](../../README.md#documentation) → Installation → OpenNebula
+
+-----
+
+[Читать на русском](opennebula.ru.md)
+
+## Automatic Installation
+
+OpenNebula plugin is packaged as `vitastor-opennebula` Debian and RPM package since Vitastor 1.9.0. So:
+
+- Run `apt-get install vitastor-opennebula` or `yum install vitastor-opennebula` after installing OpenNebula on all nodes
+- Check that it prints "OK, Vitastor OpenNebula patches successfully applied" or "OK, Vitastor OpenNebula patches are already applied"
+- If it does not, refer to [Manual Installation](#manual-installation) and apply configuration file changes manually
+- Make sure that Vitastor patched versions of QEMU and libvirt are installed
+  (`dpkg -l qemu-system-x86`, `dpkg -l | grep libvirt`, `rpm -qa | grep qemu`, `rpm -qa | grep qemu`, `rpm -qa | grep libvirt-libs` should show "vitastor" in version names)
+- [Block VM access to Vitastor cluster](#block-vm-access-to-vitastor-cluster)
+
+## Manual Installation
+
+Install OpenNebula. Then, on each node:
+
+- Copy [opennebula/remotes](../../opennebula/remotes) into `/var/lib/one` recursively: `cp -r opennebula/remotes /var/lib/one/`
+- Copy [opennebula/sudoers.d](../../opennebula/sudoers.d) to `/etc`: `cp -r opennebula/sudoers.d /etc/`
+- Apply [downloader-vitastor.sh.diff](../../opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff) to `/var/lib/one/remotes/datastore/downloader.sh`:
+  `patch /var/lib/one/remotes/datastore/downloader.sh < opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff` - or read the patch and apply the same change manually
+- Add `kvm-vitastor` to `LIVE_DISK_SNAPSHOTS` in `/etc/one/vmm_exec/vmm_execrc`
+- If on Debian or Ubuntu (and AppArmor is used), add Vitastor config file path(s) to `/etc/apparmor.d/local/abstractions/libvirt-qemu`: for example,
+  `echo '  "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu`
+- Apply changes to `/etc/one/oned.conf`
+
+### oned.conf changes
+
+1. Add deploy script override in kvm VM_MAD: add `-l deploy.vitastor` to ARGUMENTS.
+
+```diff
+ VM_MAD = [
+     NAME           = "kvm",
+     SUNSTONE_NAME  = "KVM",
+     EXECUTABLE     = "one_vmm_exec",
+-    ARGUMENTS      = "-t 15 -r 0 kvm -p",
++    ARGUMENTS      = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
+     DEFAULT        = "vmm_exec/vmm_exec_kvm.conf",
+     TYPE           = "kvm",
+     KEEP_SNAPSHOTS = "yes",
+     LIVE_RESIZE    = "yes",
+     SUPPORT_SHAREABLE    = "yes",
+     IMPORTED_VMS_ACTIONS = "terminate, terminate-hard, hold, release, suspend,
+         resume, delete, reboot, reboot-hard, resched, unresched, disk-attach,
+         disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete,
+         resize, updateconf, update"
+ ]
+```
+
+Optional: if you also want to save VM RAM checkpoints to Vitastor, use
+`-l deploy=deploy.vitastor,save=save.vitastor,restore=restore.vitastor`
+instead of just `-l deploy=deploy.vitastor`.
+
+2. Add `vitastor` to TM_MAD.ARGUMENTS and DATASTORE_MAD.ARGUMENTS:
+
+```diff
+ TM_MAD = [
+     EXECUTABLE = "one_tm",
+-    ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
++    ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
+ ]
+
+ DATASTORE_MAD = [
+     EXECUTABLE = "one_datastore",
+-    ARGUMENTS  = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
++    ARGUMENTS  = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
+ ]
+```
+
+3. Add INHERIT_DATASTORE_ATTR for two Vitastor attributes:
+
+```
+INHERIT_DATASTORE_ATTR = "VITASTOR_CONF"
+INHERIT_DATASTORE_ATTR = "IMAGE_PREFIX"
+```
+
+4. Add TM_MAD_CONF and DS_MAD_CONF for Vitastor:
+
+```
+TM_MAD_CONF = [
+    NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
+    DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
+    TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
+    DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
+    CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
+]
+
+DS_MAD_CONF = [
+    NAME = "vitastor",
+    REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
+    PERSISTENT_ONLY = "NO",
+    MARKETPLACE_ACTIONS = "export"
+]
+```
+
+## Create Datastores
+
+Example Image and System Datastore definitions:
+[opennebula/vitastor-imageds.conf](../../opennebula/vitastor-imageds.conf) and
+[opennebula/vitastor-systemds.conf](../../opennebula/vitastor-systemds.conf).
+
+Change parameters to your will:
+
+- POOL_NAME is Vitastor pool name to store images.
+- IMAGE_PREFIX is a string prepended to all Vitastor image names.
+- BRIDGE_LIST is a list of hosts with access to Vitastor cluster, mostly used for image (not system) datastore operations.
+- VITASTOR_CONF is the path to cluster configuration. Note that it should be also added to `/etc/apparmor.d/local/abstractions/libvirt-qemu` if you use AppArmor.
+- STAGING_DIR is a temporary directory used when importing external images. Should have free space sufficient for downloading external images.
+
+Then create datastores using `onedatastore create vitastor-imageds.conf` and `onedatastore create vitastor-systemds.conf` (or use UI).
+
+## Block VM access to Vitastor cluster
+
+Vitastor doesn't support any authentication yet, so you MUST block VM guest access to the Vitastor cluster at the network level.
+
+If you use VLAN networking for VMs - make sure you use different VLANs for VMs and hypervisor/storage network and
+block access between them using your firewall/switch configuration.
+
+If you use something more stupid like bridged networking, you probably have to use manual firewall/iptables setup
+to only allow access to Vitastor from hypervisor IPs.
+
+Also you need to switch network to "Bridged & Security Groups" and enable IP spoofing filters in OpenNebula.
+Problem is that OpenNebula's IP spoofing filter doesn't affect local interfaces of the hypervisor i.e. when
+it's enabled a VM can't talk to other VMs or to the outer world using a spoofed IP, but it CAN talk to the
+hypervisor if it takes an IP from its subnet. To fix that you also need some more iptables.
+
+So the complete "stupid" bridged network filter setup could look like the following
+(here `10.0.3.0/24` is the VM subnet and `10.0.2.0/24` is the hypervisor subnet):
+
+```
+# Allow incoming traffic from physical device
+iptables -A INPUT -m physdev --physdev-in eth0 -j ACCEPT
+# Do not allow incoming traffic from VMs, but not from VM subnet
+iptables -A INPUT ! -s 10.0.3.0/24 -i onebr0 -j DROP
+# Drop traffic from VMs to hypervisor/storage subnet
+iptables -I FORWARD 1 -s 10.0.3.0/24 -d 10.0.2.0/24 -j DROP
+```
+
+## Testing
+
+The OpenNebula plugin includes quite a bit of bash scripts, so here's their description to get an idea about what they actually do.
+
+| Script                  | Action                                    | How to Test                                                                          |
+| ----------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------ |
+| vmm/kvm/deploy.vitastor | Start a VM                                | Create and start a VM with Vitastor disk(s): persistent / non-persistent / volatile. |
+| vmm/kvm/save.vitastor   | Save VM memory checkpoint                 | Stop a VM using "Stop" command.                                                      |
+| vmm/kvm/restore.vitastor| Restore VM memory checkpoint              | Start a VM back after stopping it.                                                   |
+| datastore/clone         | Copy an image as persistent               | Create a VM template and instantiate it as persistent.                               |
+| datastore/cp            | Import an external image                  | Import a VM template with images from Marketplace.                                   |
+| datastore/export        | Export an image as URL                    | Probably: export a VM template with images to Marketplace.                           |
+| datastore/mkfs          | Create an image with FS                   | Storage → Images → Create → Type: Datablock, Location: Empty disk image, Filesystem: Not empty. |
+| datastore/monitor       | Monitor used space in image datastore     | Check reported used/free space in image datastore list.                              |
+| datastore/rm            | Remove a persistent image                 | Storage → Images → Select an image → Delete.                                         |
+| datastore/snap_delete   | Delete a snapshot of a persistent image   | Storage → Images → Select an image → Select a snapshot → Delete; <br> To create an image with snapshot: attach a persistent image to a VM; create a snapshot; detach the image. |
+| datastore/snap_flatten  | Revert an image to snapshot and delete other snapshots | Storage → Images → Select an image → Select a snapshot → Flatten.       |
+| datastore/snap_revert   | Revert an image to snapshot               | Storage → Images → Select an image → Select a snapshot → Revert.                     |
+| datastore/stat          | Get virtual size of an image in MB        | No idea. Seems to be unused both in Vitastor and Ceph datastores.                    |
+| tm/clone                | Clone a non-persistent image to a VM disk | Attach a non-persistent image to a VM.                                               |
+| tm/context              | Generate a contextualisation VM disk      | Create a VM with enabled contextualisation (default). Common host FS-based version is used in Vitastor and Ceph datastores. |
+| tm/cpds                 | Copy a VM disk / its snapshot to an image | Select a VM → Select a disk → Optionally select a snapshot → Save as.                |
+| tm/delete               | Delete a cloned or volatile VM disk       | Detach a volatile disk or a non-persistent image from a VM.                          |
+| tm/failmigrate          | Handle live migration failure             | No action. Script is empty in Vitastor and Ceph. In other datastores, should roll back actions done by tm/premigrate. |
+| tm/ln                   | Attach a persistent image to a VM         | No action. Script is empty in Vitastor and Ceph.                                     |
+| tm/mkimage              | Create a volatile disk, maybe with FS     | Attach a volatile disk to a VM, with or without file system.                         |
+| tm/mkswap               | Create a volatile swap disk               | Attach a volatile disk to a VM, formatted as swap.                                   |
+| tm/monitor              | Monitor used space in system datastore    | Check reported used/free space in system datastore list.                             |
+| tm/mv                   | Move a migrated VM disk between hosts     | Migrate a VM between hosts. In Vitastor and Ceph datastores, doesn't do any storage action. |
+| tm/mvds                 | Detach a persistent image from a VM       | No action. The opposite of tm/ln. Script is empty in Vitastor and Ceph. In other datastores, script may copy the image from VM host back to the datastore. |
+| tm/postbackup           | Executed after backup                     | Seems that the script just removes temporary files after backup. Perform a VM backup and check that temporary files are cleaned up. |
+| tm/postbackup_live      | Executed after backup of a running VM     | Same as tm/postbackup, but for a running VM.                                         |
+| tm/postmigrate          | Executed after VM live migration          | No action. Only executed for system datastore, so the script tries to call other TMs for other disks. Except that, the script does nothing in Vitastor and Ceph datastores. |
+| tm/prebackup            | Actual backup script: backup VM disks     | Set up "rsync" backup datastore → Backup a VM to it.                                 |
+| tm/prebackup_live       | Backup VM disks of a running VM           | Same as tm/prebackup, but also does fsfreeze/thaw. So perform a live backup, restore it and check that disks are consistent. |
+| tm/premigrate           | Executed before live migration            | No action. Only executed for system datastore, so the script tries to call other TMs for other disks. Except that, the script does nothing in Vitastor and Ceph datastores. |
+| tm/resize               | Resize a VM disk                          | Select a VM → Select a non-persistent disk → Resize.                                 |
+| tm/restore              | Restore VM disks from backup              | Set up "rsync" backup datastore → Backup a VM to it → Restore it back.               |
+| tm/snap_create          | Create a VM disk snapshot                 | Select a VM → Select a disk → Create snapshot.                                       |
+| tm/snap_create_live     | Create a VM disk snapshot for a live VM   | Select a running VM → Select a disk → Create snapshot.                               |
+| tm/snap_delete          | Delete a VM disk snapshot                 | Select a VM → Select a disk → Select a snapshot → Delete.                            |
+| tm/snap_revert          | Revert a VM disk to a snapshot            | Select a VM → Select a disk → Select a snapshot → Revert.                            |

docs/installation/opennebula.ru.md

@@ -0,0 +1,187 @@
+[Документация](../../README-ru.md#документация) → Установка → OpenNebula
+
+-----
+
+[Read in English](opennebula.en.md)
+
+## Автоматическая установка
+
+Плагин OpenNebula Vitastor распространяется как Debian и RPM пакет `vitastor-opennebula`, начиная с версии Vitastor 1.9.0. Так что:
+
+- Запустите `apt-get install vitastor-opennebula` или `yum install vitastor-opennebula` после установки OpenNebula на всех серверах
+- Проверьте, что он выводит "OK, Vitastor OpenNebula patches successfully applied" или "OK, Vitastor OpenNebula patches are already applied" в процессе установки
+- Если сообщение не выведено, пройдите по шагам инструкцию [Ручная установка](#ручная-установка) и примените правки файлов конфигурации вручную
+- Удостоверьтесь, что установлены версии QEMU и libvirt с изменениями Vitastor
+  (`dpkg -l qemu-system-x86`, `dpkg -l | grep libvirt`, `rpm -qa | grep qemu`, `rpm -qa | grep qemu`, `rpm -qa | grep libvirt-libs` должны показывать "vitastor" в номере версии)
+- [Заблокируйте доступ виртуальных машин в Vitastor](#блокировка-доступа-вм-в-vitastor)
+
+## Ручная установка
+
+Сначала установите саму OpenNebula. После этого, на каждом сервере:
+
+- Скопируйте директорию [opennebula/remotes](../../opennebula/remotes) в `/var/lib/one`: `cp -r opennebula/remotes /var/lib/one/`
+- Скопируйте директорию [opennebula/sudoers.d](../../opennebula/sudoers.d) в `/etc`: `cp -r opennebula/sudoers.d /etc/`
+- Примените патч [downloader-vitastor.sh.diff](../../opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff) к `/var/lib/one/remotes/datastore/downloader.sh`:
+  `patch /var/lib/one/remotes/datastore/downloader.sh < opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff` - либо прочитайте патч и примените изменение вручную
+- Добавьте `kvm-vitastor` в список `LIVE_DISK_SNAPSHOTS` в файле `/etc/one/vmm_exec/vmm_execrc`
+- Если вы используете Debian или Ubuntu (и AppArmor), добавьте пути к файлу(ам) конфигурации Vitastor в файл `/etc/apparmor.d/local/abstractions/libvirt-qemu`: например,
+  `echo '  "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu`
+- Примените изменения `/etc/one/oned.conf`
+
+### Изменения oned.conf
+
+1. Добавьте переопределение скрипта deploy в VM_MAD kvm, добавив `-l deploy.vitastor` в `ARGUMENTS`:
+
+```diff
+ VM_MAD = [
+     NAME           = "kvm",
+     SUNSTONE_NAME  = "KVM",
+     EXECUTABLE     = "one_vmm_exec",
+-    ARGUMENTS      = "-t 15 -r 0 kvm -p",
++    ARGUMENTS      = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
+     DEFAULT        = "vmm_exec/vmm_exec_kvm.conf",
+     TYPE           = "kvm",
+     KEEP_SNAPSHOTS = "yes",
+     LIVE_RESIZE    = "yes",
+     SUPPORT_SHAREABLE    = "yes",
+     IMPORTED_VMS_ACTIONS = "terminate, terminate-hard, hold, release, suspend,
+         resume, delete, reboot, reboot-hard, resched, unresched, disk-attach,
+         disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete,
+         resize, updateconf, update"
+ ]
+```
+
+Опционально: если вы хотите также сохранять снимки памяти ВМ в Vitastor, добавьте
+`-l deploy=deploy.vitastor,save=save.vitastor,restore=restore.vitastor`
+вместо просто `-l deploy=deploy.vitastor`.
+
+2. Добавьте `vitastor` в значения TM_MAD.ARGUMENTS и DATASTORE_MAD.ARGUMENTS:
+
+```diff
+ TM_MAD = [
+     EXECUTABLE = "one_tm",
+-    ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
++    ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
+ ]
+
+ DATASTORE_MAD = [
+     EXECUTABLE = "one_datastore",
+-    ARGUMENTS  = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
++    ARGUMENTS  = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
+ ]
+```
+
+3. Добавьте строчки с INHERIT_DATASTORE_ATTR для двух атрибутов Vitastor-хранилищ:
+
+```
+INHERIT_DATASTORE_ATTR = "VITASTOR_CONF"
+INHERIT_DATASTORE_ATTR = "IMAGE_PREFIX"
+```
+
+4. Добавьте TM_MAD_CONF и DS_MAD_CONF для Vitastor:
+
+```
+TM_MAD_CONF = [
+    NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
+    DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
+    TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
+    DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
+    CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
+]
+
+DS_MAD_CONF = [
+    NAME = "vitastor",
+    REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
+    PERSISTENT_ONLY = "NO",
+    MARKETPLACE_ACTIONS = "export"
+]
+```
+
+## Создайте хранилища
+
+Примеры настроек хранилищ образов (image) и дисков ВМ (system):
+[opennebula/vitastor-imageds.conf](../../opennebula/vitastor-imageds.conf) и
+[opennebula/vitastor-systemds.conf](../../opennebula/vitastor-systemds.conf).
+
+Скопируйте настройки и поменяйте следующие параметры так, как вам необходимо:
+
+- POOL_NAME - имя пула Vitastor для сохранения образов дисков.
+- IMAGE_PREFIX - строка, добавляемая в начало имён образов дисков.
+- BRIDGE_LIST - список серверов с доступом к кластеру Vitastor, используемых для операций с хранилищем образов (image, не system).
+- VITASTOR_CONF - путь к конфигурации Vitastor. Имейте в виду, что этот путь также надо добавить в `/etc/apparmor.d/local/abstractions/libvirt-qemu`, если вы используете AppArmor.
+- STAGING_DIR - путь к временному каталогу, используемому при импорте внешних образов. Должен иметь достаточно свободного места, чтобы вмещать скачанные образы.
+
+После этого создайте хранилища с помощью команд `onedatastore create vitastor-imageds.conf` и `onedatastore create vitastor-systemds.conf` (либо через UI).
+
+## Блокировка доступа ВМ в Vitastor
+
+Vitastor пока не поддерживает никакую аутентификацию, так что вы ДОЛЖНЫ заблокировать доступ гостевых ВМ
+в кластер Vitastor на сетевом уровне.
+
+Если вы используете VLAN-сети для ВМ - удостоверьтесь, что ВМ и гипервизор/сеть хранения помещены в разные
+изолированные друг от друга VLAN-ы.
+
+Если вы используете что-то более примитивное, например, мосты (bridge), вам, скорее всего, придётся вручную
+настроить iptables / межсетевой экран, чтобы разрешить доступ к Vitastor только с IP гипервизоров.
+
+Также в этом случае нужно будет переключить обычные мосты на "Bridged & Security Groups" и включить фильтр
+спуфинга IP в OpenNebula. Правда, реализация этого фильтра пока не полная, и она не блокирует доступ к
+локальным интерфейсам гипервизора. То есть, включённый фильтр спуфинга IP запрещает ВМ отправлять трафик
+с чужими IP к другим ВМ или во внешний мир, но не запрещает отправлять его напрямую гипервизору. Чтобы
+исправить это, тоже нужны дополнительные правила iptables.
+
+Таким образом, более-менее полная блокировка при использовании простой сети на сетевых мостах может
+выглядеть так (здесь `10.0.3.0/24` - подсеть ВМ, `10.0.2.0/24` - подсеть гипервизора):
+
+```
+# Разрешаем входящий трафик с физического устройства
+iptables -A INPUT -m physdev --physdev-in eth0 -j ACCEPT
+# Запрещаем трафик со всех ВМ, но с IP не из подсети ВМ
+iptables -A INPUT ! -s 10.0.3.0/24 -i onebr0 -j DROP
+# Запрещаем трафик от ВМ к сети гипервизора
+iptables -I FORWARD 1 -s 10.0.3.0/24 -d 10.0.2.0/24 -j DROP
+```
+
+## Тестирование
+
+Плагин OpenNebula по большей части состоит из bash-скриптов, и чтобы было понятнее, что они
+вообще делают - ниже приведены описания процедур, которыми можно протестировать каждый из них.
+
+| Скрипт                  | Описание                                      | Как протестировать                                                                   |
+| ----------------------- | --------------------------------------------- | ------------------------------------------------------------------------------------ |
+| vmm/kvm/deploy.vitastor | Запустить виртуальную машину                  | Создайте и запустите виртуальную машину с дисками Vitastor: постоянным / непостоянным / волатильным (временным). |
+| vmm/kvm/save.vitastor   | Сохранить снимок памяти ВМ                    | Остановите виртуальную машину командой "Остановить".                                 |
+| vmm/kvm/restore.vitastor| Восстановить снимок памяти ВМ                 | Запустите ВМ после остановки обратно.                                                |
+| datastore/clone         | Скопировать образ как "постоянный"            | Создайте шаблон ВМ и создайте из него постоянную ВМ.                                 |
+| datastore/cp            | Импортировать внешний образ                   | Импортируйте шаблон ВМ с образами дисков из Магазина OpenNebula.                     |
+| datastore/export        | Экспортировать образ как URL                  | Вероятно: экспортируйте шаблон ВМ с образами в Магазин.                              |
+| datastore/mkfs          | Создать образ с файловой системой             | Хранилище → Образы → Создать → Тип: базовый блок данных, Расположение: пустой образ диска, Файловая система: любая непустая. |
+| datastore/monitor       | Вывод статистики места в хранилище образов    | Проверьте статистику свободного/занятого места в списке хранилищ образов.            |
+| datastore/rm            | Удалить "постоянный" образ                    | Хранилище → Образы → Выберите образ → Удалить.                                       |
+| datastore/snap_delete   | Удалить снимок "постоянного" образа           | Хранилище → Образы → Выберите образ → Выберите снимок → Удалить; <br> Чтобы создать образ со снимком: подключите постоянный образ к ВМ, создайте снимок, отключите образ. |
+| datastore/snap_flatten  | Откатить образ к снимку, удалив другие снимки | Хранилище → Образы → Выберите образ → Выберите снимок → "Выровнять" (flatten).       |
+| datastore/snap_revert   | Откатить образ к снимку                       | Хранилище → Образы → Выберите образ → Выберите снимок → Откатить.                    |
+| datastore/stat          | Показать виртуальный размер образа в МБ       | Неизвестно. По-видимому, в плагинах Vitastor и Ceph не используется.                 |
+| tm/clone                | Клонировать "непостоянный" образ в диск ВМ    | Подключите "непостоянный" образ к ВМ.                                                |
+| tm/context              | Создать диск контекстуализации ВМ             | Создайте ВМ с контекстуализацией, как обычно. Но тестировать особенно нечего: в плагинах Vitastor и Ceph образ контекста хранится в локальной ФС гипервизора. |
+| tm/cpds                 | Копировать диск ВМ/его снимок в новый образ   | Выберите ВМ → Выберите диск → Опционально выберите снимок → "Сохранить как".         |
+| tm/delete               | Удалить диск-клон или волатильный диск ВМ     | Отключите волатильный или не-постоянный диск от ВМ.                                  |
+| tm/failmigrate          | Обработать неудачную миграцию                 | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. В других плагинах скрипт должен откатывать действия tm/premigrate. |
+| tm/ln                   | Подключить "постоянный" образ к ВМ            | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph.                          |
+| tm/mkimage              | Создать волатильный диск, без или с ФС        | Подключите волатильный диск к ВМ, с или без файловой системы.                        |
+| tm/mkswap               | Создать волатильный диск подкачки             | Подключите волатильный диск к ВМ, форматированный как диск подкачки (swap).          |
+| tm/monitor              | Вывод статистики места в хранилище дисков ВМ  | Проверьте статистику свободного/занятого места в списке хранилищ дисков ВМ.          |
+| tm/mv                   | Мигрировать диск ВМ между хостами             | Мигрируйте ВМ между серверами. Правда, с точки зрения хранилища в плагинах Vitastor и Ceph этот скрипт ничего не делает. |
+| tm/mvds                 | Отключить "постоянный" образ от ВМ            | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. В целом же скрипт обратный к tm/ln и в других хранилищах он может, например, копировать образ ВМ с диска гипервизора обратно в хранилище. |
+| tm/postbackup           | Выполняется после бэкапа                      | По-видимому, скрипт просто удаляет временные файлы после резервного копирования. Так что можно провести его и проверить, что на серверах не осталось временных файлов. |
+| tm/postbackup_live      | Выполняется после бэкапа запущенной ВМ        | То же, что tm/postbackup, но для запущенной ВМ.                                      |
+| tm/postmigrate          | Выполняется после миграции ВМ                 | Тестировать нечего. Однако, OpenNebula запускает скрипт только для системного хранилища, поэтому он вызывает аналогичные скрипты для хранилищ других дисков той же ВМ. Помимо этого в плагинах Vitastor и Ceph скрипт ничего не делает. |
+| tm/prebackup            | Выполнить резервное копирование дисков ВМ     | Создайте хранилище резервных копий типа "rsync" → Забэкапьте в него ВМ.              |
+| tm/prebackup_live       | То же самое для запущенной ВМ                 | То же, что tm/prebackup, но запускает fsfreeze/thaw (остановку доступа к дискам). Так что смысл теста - проведите резервное копирование и проверьте, что данные скопировались консистентно. |
+| tm/premigrate           | Выполняется перед миграцией ВМ                | Тестировать нечего. Аналогично tm/postmigrate запускается только для системного хранилища. |
+| tm/resize               | Изменить размер диска ВМ                      | Выберите ВМ → Выберите непостоянный диск → Измените его размер.                      |
+| tm/restore              | Восстановить диски ВМ из бэкапа               | Создайте хранилище резервных копий → Забэкапьте в него ВМ → Восстановите её обратно. |
+| tm/snap_create          | Создать снимок диска ВМ                       | Выберите ВМ → Выберите диск → Создайте снимок.                                       |
+| tm/snap_create_live     | Создать снимок диска запущенной ВМ            | Выберите запущенную ВМ → Выберите диск → Создайте снимок.                            |
+| tm/snap_delete          | Удалить снимок диска ВМ                       | Выберите ВМ → Выберите диск → Выберите снимок → Удалить.                             |
+| tm/snap_revert          | Откатить диск ВМ к снимку                     | Выберите ВМ → Выберите диск → Выберите снимок → Откатить.                            |

docs/installation/proxmox.en.md

@@ -17,10 +17,10 @@ To enable Vitastor support in Proxmox Virtual Environment (6.4-8.1 are supported
 - Restart pvedaemon: `systemctl restart pvedaemon`
 
 `/etc/pve/storage.cfg` example (the only required option is vitastor_pool, all others
-are listed below with their default values):
+are listed below with their default values; `vitastor_ssd` is Proxmox storage pool id):
 
 ```
-vitastor: vitastor
+vitastor: vitastor_ssd
     # pool to put new images into
     vitastor_pool testpool
     # path to the configuration file

docs/installation/proxmox.ru.md

@@ -16,10 +16,10 @@
 - Перезапустите демон Proxmox: `systemctl restart pvedaemon`
 
 Пример `/etc/pve/storage.cfg` (единственная обязательная опция - vitastor_pool, все остальные
-перечислены внизу для понимания значений по умолчанию):
+перечислены внизу для понимания значений по умолчанию; `vitastor_ssd` - имя хранилища в Proxmox):
 
 ```
-vitastor: vitastor
+vitastor: vitastor_ssd
     # Пул, в который будут помещаться образы дисков
     vitastor_pool testpool
     # Путь к файлу конфигурации

docs/intro/features.en.md

@@ -39,6 +39,10 @@
 
 ## Plugins and tools
 
+- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
+- [OpenNebula storage plugin](../installation/opennebula.en.md)
+- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
+- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
 - [Debian and CentOS packages](../installation/packages.en.md)
 - [Image management CLI (vitastor-cli)](../usage/cli.en.md)
 - [Disk management CLI (vitastor-disk)](../usage/disk.en.md)
@@ -46,9 +50,6 @@
 - [Native QEMU driver](../usage/qemu.en.md)
 - [Loadable fio engine for benchmarks](../usage/fio.en.md)
 - [NBD proxy for kernel mounts](../usage/nbd.en.md)
-- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
-- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
-- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
 - [Simplified NFS proxy for file-based image access emulation (suitable for VMWare)](../usage/nfs.en.md#pseudo-fs)
 
 ## Roadmap
@@ -58,7 +59,6 @@ The following features are planned for the future:
 - Control plane optimisation
 - Other administrative tools
 - Web GUI
-- OpenNebula plugin
 - iSCSI and NVMeoF gateways
 - Multi-threaded client
 - Faster failover

docs/intro/features.ru.md

@@ -41,6 +41,10 @@
 
 ## Драйверы и инструменты
 
+- [Плагин для Proxmox](../installation/proxmox.ru.md)
+- [Плагин для OpenNebula](../installation/opennebula.ru.md)
+- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
+- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
 - [Пакеты для Debian и CentOS](../installation/packages.ru.md)
 - [Консольный интерфейс управления образами (vitastor-cli)](../usage/cli.ru.md)
 - [Инструмент управления дисками (vitastor-disk)](../usage/disk.ru.md)
@@ -48,9 +52,6 @@
 - [Драйвер диска для QEMU](../usage/qemu.ru.md)
 - [Драйвер диска для утилиты тестирования производительности fio](../usage/fio.ru.md)
 - [NBD-прокси для монтирования образов ядром](../usage/nbd.ru.md) ("блочное устройство в режиме пользователя")
-- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
-- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
-- [Плагин для Proxmox](../installation/proxmox.ru.md)
 - [Упрощённая NFS-прокси для эмуляции файлового доступа к образам (подходит для VMWare)](../usage/nfs.ru.md#псевдо-фс)
 
 ## Планы развития
@@ -58,7 +59,6 @@
 - Оптимизация слоя управления
 - Другие инструменты администрирования
 - Web-интерфейс
-- Плагин для OpenNebula
 - iSCSI и NVMeoF прокси
 - Многопоточный клиент
 - Более быстрое переключение при отказах

docs/params/head/common.en.md

@@ -0,0 +1,6 @@
+---
+title: Common Parameters
+weight: 1
+---
+
+These are the most common parameters which apply to all components of Vitastor.

docs/params/head/common.ru.md

@@ -0,0 +1,6 @@
+---
+title: Общие параметры
+weight: 1
+---
+
+Это наиболее общие параметры, используемые всеми компонентами Vitastor.

docs/params/head/layout-cluster.en.md

@@ -0,0 +1,7 @@
+---
+title: Cluster-Wide Disk Layout Parameters
+weight: 2
+---
+
+These parameters apply to clients and OSDs, are fixed at the moment of OSD drive
+initialization and can't be changed after it without losing data.

docs/params/head/layout-cluster.ru.md

@@ -0,0 +1,7 @@
+---
+title: Дисковые параметры уровня кластера
+weight: 2
+---
+
+Данные параметры используются клиентами и OSD, задаются в момент инициализации
+диска OSD и не могут быть изменены после этого без потери данных.

docs/params/head/layout-osd.en.md

@@ -0,0 +1,7 @@
+---
+title: OSD Disk Layout Parameters
+weight: 3
+---
+
+These parameters apply to OSDs, are fixed at the moment of OSD drive
+initialization and can't be changed after it without losing data.

docs/params/head/layout-osd.ru.md

@@ -0,0 +1,8 @@
+---
+title: Дисковые параметры OSD
+weight: 3
+---
+
+Данные параметры используются только OSD и, также как и общекластерные
+дисковые параметры, задаются в момент инициализации дисков OSD и не могут быть
+изменены после этого без потери данных.

docs/params/head/monitor.en.md

@@ -0,0 +1,6 @@
+---
+title: Monitor Parameters
+weight: 6
+---
+
+These parameters only apply to Monitors.

docs/params/head/monitor.ru.md

@@ -0,0 +1,6 @@
+---
+title: Параметры мониторов
+weight: 6
+---
+
+Данные параметры используются только мониторами Vitastor.

docs/params/head/network.en.md

@@ -0,0 +1,7 @@
+---
+title: Network Protocol Parameters
+weight: 4
+---
+
+These parameters apply to clients and OSDs and affect network connection logic
+between clients, OSDs and etcd.

docs/params/head/network.ru.md

@@ -0,0 +1,7 @@
+---
+title: Параметры сетевого протокола
+weight: 4
+---
+
+Данные параметры используются клиентами и OSD и влияют на логику сетевого
+взаимодействия между клиентами, OSD, а также etcd.

docs/params/head/osd.en.md

@@ -0,0 +1,7 @@
+---
+title: Runtime OSD Parameters
+weight: 5
+---
+
+These parameters only apply to OSDs, are not fixed at the moment of OSD drive
+initialization and can be changed with an OSD restart.

docs/params/head/osd.ru.md

@@ -0,0 +1,8 @@
+---
+title: Изменяемые параметры OSD
+weight: 5
+---
+
+Данные параметры используются только OSD, но, в отличие от дисковых параметров,
+не фиксируются в момент инициализации дисков OSD и могут быть изменены в любой
+момент с перезапуском OSD.

docs/usage/cli.en.md

@@ -16,6 +16,7 @@ It supports the following commands:
 - [create](#create)
 - [snap-create](#create)
 - [modify](#modify)
+- [dd](#dd)
 - [rm](#rm)
 - [flatten](#flatten)
 - [rm-data](#rm-data)
@@ -148,19 +149,60 @@ You should resize file system in the image, if present, before shrinking it.
 * `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
 * `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
 
+## dd
+
+```
+vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
+    [count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress] \
+    [conv=nocreat,noerror,nofsync,trunc,nosparse] [iflag=direct] [oflag=direct,append]
+```
+
+Copy data between Vitastor images, files and pipes.
+
+Options can be specified in classic dd style (`key=value`) or like usual (`--key value`).
+
+| <!-- -->        | <!-- -->                                                                |
+|-----------------|-------------------------------------------------------------------------|
+| `iimg=<image>`  | Copy from Vitastor image `<image>`                                      |
+| `if=<file>`     | Copy from file `<file>`                                                 |
+| `oimg=<image>`  | Copy to Vitastor image `<image>`                                        |
+| `of=<file>`     | Copy to file `<file>`                                                   |
+| `bs=1M`         | Set copy block size                                                     |
+| `count=N`       | Copy only N input blocks. If N ends in B it counts bytes, not blocks    |
+| `seek/oseek=N`  | Skip N output blocks. If N ends in B it counts bytes, not blocks        |
+| `skip/iseek=N`  | Skip N input blocks. If N ends in B it counts bytes, not blocks         |
+| `iodepth=N`     | Send N reads or writes in parallel (default 4)                          |
+| `status=LEVEL`  | The LEVEL of information to print to stderr: none/noxfer/progress       |
+| `size=N`        | Specify size for the created output file/image (defaults to input size) |
+| `iflag=direct`  | For input files only: use direct I/O                                    |
+| `oflag=direct`  | For output files only: use direct I/O                                   |
+| `oflag=append`  | For files only: append to output file                                   |
+| `conv=nocreat`  | Do not create output file/image                                         |
+| `conv=trunc`    | Truncate output file/image                                              |
+| `conv=noerror`  | Continue copying after errors                                           |
+| `conv=nofsync`  | Do not call fsync before finishing (default behaviour is fsync)         |
+| `conv=nosparse` | Write all output blocks including all-zero blocks                       |
+
 ## rm
 
 `vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
 
-Remove `<from>` or all layers between `<from>` and `<to>` (`<to>` must be a child of `<from>`),
-rebasing all their children accordingly. --writers-stopped allows merging to be a bit
-more effective in case of a single 'slim' read-write child and 'fat' removed parent:
-the child is merged into parent and parent is renamed to child in that case.
-In other cases parent layers are always merged into children.
+`vitastor-cli rm (--exact|--matching) <glob> ...`
 
-Other options:
+Remove layer(s) and rebase all their children accordingly.
 
-* `--down-ok` - Continue deletion/merging even if some data will be left on unavailable OSDs.
+In the first form, remove `<from>` or layers between `<from>` and its child `<to>`.
+
+In the second form, remove all images with exact or pattern-matched names.
+
+Options:
+
+* `--writers-stopped` allows optimised removal in case of a single 'slim' read-write
+  child and 'fat' removed parent: the child is merged into parent and parent is renamed
+  to child in that case. In other cases parent layers are always merged into children.
+* `--exact` - remove multiple images with names matching given glob patterns.
+* `--matching` - remove multiple images with given names
+* `--down-ok` - continue deletion/merging even if some data will be left on unavailable OSDs.
 
 ## flatten
 
@@ -307,7 +349,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 `vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
 
-Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config.pool.en.md#osd-settings).
+Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config/pool.en.md#osd-settings).
 
 ## pg-list
 

docs/usage/cli.ru.md

@@ -17,6 +17,7 @@ vitastor-cli - интерфейс командной строки для адм
 - [create](#create)
 - [snap-create](#create)
 - [modify](#modify)
+- [dd](#dd)
 - [rm](#rm)
 - [flatten](#flatten)
 - [rm-data](#rm-data)
@@ -151,23 +152,61 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 * `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
 * `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
 
+## dd
+
+```
+vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
+    [count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress] \
+    [conv=nocreat,noerror,nofsync,trunc,nosparse] [iflag=direct] [oflag=direct,append]
+```
+
+Копировать данные между образами Vitastor, файлами и каналами.
+
+Опции можно передавать в классическом стиле dd (`key=value`) или как обычно (`--key value`).
+
+| <!-- -->        | <!-- -->                                                                |
+|-----------------|-------------------------------------------------------------------------|
+| `iimg=<image>`  | Копировать из образа Vitastor `<image>`                                 |
+| `if=<file>`     | Копировать из файла `<file>`                                            |
+| `oimg=<image>`  | Копировать в образ Vitastor `<image>`                                   |
+| `of=<file>`     | Копировать в файл `<file>`                                              |
+| `bs=1M`         | Задать размер блока копирования                                         |
+| `count=N`       | Копировать не более N блоков. Если N заканчивается на B - то N байт.    |
+| `seek/oseek=N`  | Пропустить N выходных блоков. Если N заканчивается на B - то N байт.    |
+| `skip/iseek=N`  | Пропустить N входных блоков. Если N заканчивается на B - то N байт.     |
+| `iodepth=N`     | Отправлять N чтений/записей параллельно (по умолчанию 4).               |
+| `status=LEVEL`  | Уровень вывода в консоль: none/noxfer/progress                          |
+| `size=N`        | Задать размер выходного файла/образа (по умолчанию равен размеру входа).|
+| `iflag=direct`  | Только для входного файла: использовать прямой ввод-вывод               |
+| `oflag=direct`  | Только для выходного файла: использовать прямой ввод-вывод              |
+| `oflag=append`  | Только для файлов: дописывать в конец выходного файла                   |
+| `conv=nocreat`  | Не создавать выходной файл/образ                                        |
+| `conv=trunc`    | Обрезать выходной файл/образ до размера входа                           |
+| `conv=noerror`  | Продолжать копирование после ошибок                                     |
+| `conv=nofsync`  | Не вызывать fsync перед завершением                                     |
+| `conv=nosparse` | Записывать все выходные блоки, включая пустые                           |
+
 ## rm
 
 `vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
 
-Удалить образ `<from>` или все слои от `<from>` до `<to>` (`<to>` должен быть дочерним
-образом `<from>`), одновременно меняя родительские образы их клонов (если таковые есть).
+`vitastor-cli rm (--exact|--matching) <glob> ...`
 
-`--writers-stopped` позволяет чуть более эффективно удалять образы в частом случае, когда
-у удаляемой цепочки есть только один дочерний образ, содержащий небольшой объём данных.
-В этом случае дочерний образ вливается в родительский и удаляется, а родительский
-переименовывается в дочерний.
+Удалить образ(ы), корректно перебазируя их дочерние образы.
 
-В других случаях родительские слои вливаются в дочерние.
+В первой форме удаляет один образ `<from>` или все слои между `<from>` и его дочерним `<to>`.
 
-Другие опции:
+Во второй форме, удаляет все образы с точными именами или именами, подходящими под шаблон(ы).
 
-* `--down-ok` - Продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
+Опции:
+
+* `--writers-stopped` позволяет чуть более эффективно удалять образы в частом случае, когда
+  у удаляемой цепочки есть только один дочерний образ, содержащий небольшой объём данных.
+  В этом случае дочерний образ вливается в родительский и удаляется, а родительский
+  переименовывается в дочерний.
+* `--exact` - удалить все образы с именами, подходящими под переданные glob-шаблоны.
+* `--matching` - удалить все образы с точно заданными именами.
+* `--down-ok` - продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
 
 ## flatten
 
@@ -324,7 +363,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 `vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
 
-Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config.pool.ru.md#настройки-osd).
+Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config/pool.ru.md#настройки-osd).
 
 ## pg-list
 

hugo/archetypes/default.md

@@ -0,0 +1,6 @@
+---
+title: "{{ replace .Name "-" " " | title }}"
+date: {{ .Date }}
+draft: true
+---
+

hugo/config.yaml

@@ -0,0 +1,35 @@
+baseURL: http://localhost
+title: Vitastor
+theme: hugo-geekdoc
+#languageCode: en-us
+
+pluralizeListTitles: false
+
+# Geekdoc required configuration
+pygmentsUseClasses: true
+pygmentsCodeFences: true
+disablePathToLower: true
+
+# Required if you want to render robots.txt template
+enableRobotsTXT: true
+
+defaultContentLanguage: en
+languages:
+  en:
+    weight: 1
+    languageName: English
+  ru:
+    weight: 1
+    languageName: Русский
+
+markup:
+  goldmark:
+    renderer:
+      # Needed for mermaid shortcode
+      unsafe: true
+  tableOfContents:
+    startLevel: 1
+    endLevel: 9
+
+taxonomies:
+  tag: tags

hugo/content/_index.md

@@ -0,0 +1,6 @@
+## The Idea
+
+Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+architecturally similar to Ceph which means strong consistency, primary-replication,
+symmetric clustering and automatic data distribution over any number of drives
+of any size with configurable redundancy (replication or erasure codes/XOR).

hugo/content/config/_index.en.md

@@ -0,0 +1,5 @@
+---
+title: Parameter Reference
+weight: 1
+---
+

hugo/content/config/_index.ru.md

@@ -0,0 +1,5 @@
+---
+title: Перечень настроек
+weight: 1
+---
+

hugo/content/config/pool.en.md

@@ -0,0 +1,5 @@
+---
+title: Pool configuration
+weight: 100
+---
+

hugo/content/installation/packages.md

@@ -0,0 +1,5 @@
+---
+title: Packages
+weight: 2
+---
+

hugo/content/installation/quickstart.md

@@ -0,0 +1,5 @@
+---
+title: Quick Start
+weight: 1
+---
+

hugo/content/installation/source.md

@@ -0,0 +1,5 @@
+---
+title: Building from Source
+weight: 3
+---
+

hugo/content/introduction/_index.md

@@ -0,0 +1,4 @@
+---
+title: Introduction
+weight: -1
+---

hugo/content/introduction/architecture.md

@@ -0,0 +1,5 @@
+---
+title: Architecture
+weight: 3
+---
+

hugo/content/introduction/author.md

@@ -0,0 +1,5 @@
+---
+title: Author and License
+weight: 3
+---
+

hugo/content/introduction/features.md

@@ -0,0 +1,5 @@
+---
+title: Features
+weight: 1
+---
+

hugo/content/performance/comparison1.md

@@ -0,0 +1,5 @@
+---
+title: Example Comparison with Ceph
+weight: 4
+---
+

hugo/content/performance/theoretical.md

@@ -0,0 +1,5 @@
+---
+title: Vitastor's Theoretical Maximum Performance
+weight: 3
+---
+

hugo/content/performance/tuning.md

@@ -0,0 +1,5 @@
+---
+title: Tuning
+weight: 2
+---
+

hugo/content/performance/understanding.md

@@ -0,0 +1,5 @@
+---
+title: Understanding Storage Performance
+weight: 1
+---
+

hugo/content/usage/cli.md

@@ -0,0 +1,7 @@
+---
+title: Vitastor CLI
+weight: 1
+---
+
+{{< toc >}}
+

hugo/content/usage/nbd.md

@@ -0,0 +1,5 @@
+---
+title: NBD
+weight: 6
+---
+

hugo/content/usage/qemu.md

@@ -0,0 +1,5 @@
+---
+title: QEMU and qemu-img
+weight: 2
+---
+

hugo/i18n/ru.yaml

@@ -0,0 +1,37 @@
+---
+nav_navigation: Навигация
+nav_tags: Теги
+nav_more: Подробнее
+nav_top: К началу
+
+form_placeholder_search: Поиск
+
+error_page_title: Открыта несуществующая страница
+error_message_title: Потерялись?
+error_message_code: Ошибка 404
+error_message_text: >
+  Похоже, страница, которую вы открыли, не существует. Попробуйте найти
+  нужную информацию с <a class="gdoc-error__link" href="{{ . }}">главной страницы</a>.
+
+button_toggle_dark: Переключить тёмный/светлый/авто режим
+button_nav_open: Показать навигацию
+button_nav_close: Скрыть навигацию
+button_menu_open: Открыть меню
+button_menu_close: Закрыть меню
+button_homepage: На главную
+
+title_anchor_prefix: "Ссылка на:"
+
+posts_read_more: Читать подробнее
+posts_read_time:
+  one: "Одна минута на чтение"
+  other: "{{ . }} минут(ы) на чтение"
+posts_update_prefix: Обновлено
+
+footer_build_with: >
+  Сделано на <a href="https://gohugo.io/" class="gdoc-footer__link">Hugo</a> с
+  <svg class="icon gdoc_heart"><use xlink:href="#gdoc_heart"></use></svg>
+footer_legal_notice: Правовая информация
+footer_privacy_policy: Приватность
+
+language_switch_no_tranlation_prefix: "Страница не переведена:"

hugo/layouts/partials/site-footer.html

@@ -0,0 +1,34 @@
+<footer class="gdoc-footer">
+    <div class="container flex">
+        <div class="flex flex-wrap" style="flex: 1">
+            <span class="gdoc-footer__item gdoc-footer__item--row">
+                &copy; Vitaliy Filippov, 2021+
+            </span>
+        </div>
+        <div class="flex flex-wrap">
+            {{ with .Site.Params.GeekdocLegalNotice }}
+            <span class="gdoc-footer__item gdoc-footer__item--row">
+                <a href="{{ . | relURL }}" class="gdoc-footer__link">{{ i18n "footer_legal_notice" }}</a>
+            </span>
+            {{ end }}
+            {{ with .Site.Params.GeekdocPrivacyPolicy }}
+            <span class="gdoc-footer__item gdoc-footer__item--row">
+                <a href="{{ . | relURL }}" class="gdoc-footer__link">{{ i18n "footer_privacy_policy" }}</a>
+            </span>
+            {{ end }}
+        </div>
+        {{ if (default true .Site.Params.GeekdocBackToTop) }}
+        <div class="flex flex-25 justify-end">
+            <span class="gdoc-footer__item gdoc-footer__item--row" style="margin-right: 50px">
+                {{ i18n "footer_build_with" | safeHTML }}
+            </span>
+            <span class="gdoc-footer__item">
+                <a class="gdoc-footer__link fake-link" href="#" aria-label="{{ i18n "nav_top" }}">
+                    <svg class="icon gdoc_keyboard_arrow_up"><use xlink:href="#gdoc_keyboard_arrow_up"></use></svg>
+                    <span class="hidden-mobile">{{ i18n "nav_top" }}</span>
+                </a>
+            </span>
+        </div>
+        {{ end }}
+    </div>
+</footer>

hugo/static/brand.svg

@@ -0,0 +1,215 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:osb="http://www.openswatchbook.org/uri/2009/osb"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   sodipodi:docname="logo_only2.svg"
+   inkscape:version="1.0.2 (e86c870879, 2021-01-15)"
+   id="svg1340"
+   version="1.1"
+   viewBox="0 0 100 86.80192"
+   height="86.801918mm"
+   width="100mm"
+   inkscape:export-filename="/var/home/vitali/SVN/vitastor/presentation/logos/logo_only.png"
+   inkscape:export-xdpi="92.889999"
+   inkscape:export-ydpi="92.889999">
+  <defs
+     id="defs1334">
+    <linearGradient
+       osb:paint="gradient"
+       id="linearGradient866">
+      <stop
+         id="stop862"
+         offset="0"
+         style="stop-color:#c0c0c0;stop-opacity:1" />
+      <stop
+         id="stop864"
+         offset="1"
+         style="stop-color:#000000;stop-opacity:0" />
+    </linearGradient>
+    <linearGradient
+       id="linearGradient846"
+       osb:paint="gradient">
+      <stop
+         style="stop-color:#ffd42a;stop-opacity:1"
+         offset="0"
+         id="stop842" />
+      <stop
+         style="stop-color:#ffa200;stop-opacity:1"
+         offset="1"
+         id="stop844" />
+    </linearGradient>
+    <radialGradient
+       r="50"
+       fy="159.11139"
+       fx="202.36813"
+       cy="159.11139"
+       cx="202.36813"
+       gradientTransform="matrix(1.2462942,-1.2279529,0.77712408,0.78873143,-190.96813,230.1331)"
+       gradientUnits="userSpaceOnUse"
+       id="radialGradient1530"
+       xlink:href="#linearGradient1352"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient1352">
+      <stop
+         style="stop-color:#00c9e6;stop-opacity:1"
+         offset="0"
+         id="stop1348" />
+      <stop
+         style="stop-color:#5240d3;stop-opacity:1"
+         offset="1"
+         id="stop1350" />
+    </linearGradient>
+    <linearGradient
+       y2="62.555599"
+       x2="51.484566"
+       y1="62.555599"
+       x1="38.105473"
+       gradientTransform="rotate(-16.930773,271.11609,-412.42594)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient1508"
+       xlink:href="#linearGradient1323"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient1323">
+      <stop
+         style="stop-color:#000000;stop-opacity:0.47178105"
+         offset="0"
+         id="stop1319" />
+      <stop
+         style="stop-color:#eeaaff;stop-opacity:0;"
+         offset="1"
+         id="stop1321" />
+    </linearGradient>
+    <radialGradient
+       r="21.541935"
+       fy="24.614815"
+       fx="45.312912"
+       cy="24.614815"
+       cx="45.312912"
+       gradientTransform="matrix(1.0933447,0.13113705,-0.12664108,1.0558599,-1.082187,93.974708)"
+       gradientUnits="userSpaceOnUse"
+       id="radialGradient1504"
+       xlink:href="#linearGradient846"
+       inkscape:collect="always" />
+    <filter
+       style="color-interpolation-filters:sRGB"
+       inkscape:label="Drop Shadow"
+       id="filter1497"
+       width="2"
+       height="2"
+       x="-0.5"
+       y="-0.5">
+      <feFlood
+         flood-opacity="0.498039"
+         flood-color="rgb(0,0,0)"
+         result="flood"
+         id="feFlood1487" />
+      <feComposite
+         in="flood"
+         in2="SourceGraphic"
+         operator="in"
+         result="composite1"
+         id="feComposite1489" />
+      <feGaussianBlur
+         in="composite1"
+         stdDeviation="6"
+         result="blur"
+         id="feGaussianBlur1491" />
+      <feOffset
+         dx="0"
+         dy="6"
+         result="offset"
+         id="feOffset1493" />
+      <feComposite
+         in="offset"
+         in2="offset"
+         operator="atop"
+         result="composite2"
+         id="feComposite1495" />
+    </filter>
+    <radialGradient
+       r="21.541935"
+       fy="24.614815"
+       fx="45.312912"
+       cy="24.614815"
+       cx="45.312912"
+       gradientTransform="matrix(1.0933447,0.13113705,-0.12664108,1.0558599,-1.082187,93.974708)"
+       gradientUnits="userSpaceOnUse"
+       id="radialGradient1506"
+       xlink:href="#linearGradient846"
+       inkscape:collect="always" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:window-maximized="1"
+     inkscape:window-y="0"
+     inkscape:window-x="0"
+     inkscape:window-height="992"
+     inkscape:window-width="1920"
+     fit-margin-bottom="0"
+     fit-margin-right="0"
+     fit-margin-left="0"
+     fit-margin-top="-30"
+     showgrid="false"
+     inkscape:document-rotation="0"
+     inkscape:current-layer="layer1"
+     inkscape:document-units="mm"
+     inkscape:cy="47.914558"
+     inkscape:cx="-103.69646"
+     inkscape:zoom="0.7"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="1"
+     borderopacity="1.0"
+     bordercolor="#666666"
+     pagecolor="#000000"
+     id="base" />
+  <metadata
+     id="metadata1337">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(-133.26969,-52.101187)"
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Слой 1">
+    <path
+       style="fill:url(#radialGradient1530);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 133.26969,59.089473 50,75.000087 50,-75.000087 z"
+       id="path1528"
+       sodipodi:nodetypes="cccc" />
+    <path
+       d="m 194.29572,89.403603 -8.41706,2.562119 -2.50682,7.49308 7.17785,23.579008 9.60097,-14.40173 z"
+       style="fill:url(#linearGradient1508);fill-opacity:1;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.501961"
+       id="path1459" />
+    <g
+       transform="translate(135.70225,-49.385894)"
+       id="g1465">
+      <path
+         id="path1461"
+         style="fill:url(#radialGradient1504);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;filter:url(#filter1497)"
+         d="m 28.817436,101.36529 c 3.112699,10.74423 6.225077,21.48892 9.333984,32.23438 2.519532,0 5.039063,0 7.558594,0 -0.985406,8.09729 -2.085815,16.18202 -2.951172,24.29297 -0.06053,0.88723 1.098131,1.61652 1.76,0.9155 1.007514,-1.05482 1.676008,-2.3829 2.528566,-3.56053 7.51538,-11.37722 14.987447,-22.78299 22.482919,-34.17333 -3.239584,0 -6.479167,0 -9.71875,0 2.887267,-6.79562 5.775365,-13.59088 8.662109,-20.38672 -13.284505,0 -26.56901,0 -39.853516,0 0.06576,0.22591 0.131511,0.45182 0.197266,0.67773 z" />
+      <path
+         sodipodi:nodetypes="cccccccc"
+         id="path1463"
+         d="m 30.735882,102.2764 h 35.342242 l -8.662729,20.3854 h 9.173783 l -22.106472,33.62346 3.027029,-24.27377 H 39.34604 Z"
+         style="fill:url(#radialGradient1506);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
+    </g>
+  </g>
+</svg>

hugo/static/custom.css

@@ -0,0 +1,138 @@
+/* Global customization */
+
+:root {
+  --code-max-height: 60rem;
+}
+
+/* Light mode theming */
+:root,
+:root[color-mode="light"] {
+  --header-background: #404050;
+  --header-font-color: #ffffff;
+
+  --body-background: #ffffff;
+  --body-font-color: #343a40;
+
+  --button-background: #62cb97;
+  --button-border-color: #4ec58a;
+
+  --link-color: #c54e8a;
+  --link-color-visited: #c54e8a;
+
+  --code-background: #f5f6f8;
+  --code-accent-color: #e3e7eb;
+  --code-accent-color-lite: #eff1f3;
+
+  --accent-color: #e9ecef;
+  --accent-color-lite: #f8f9fa;
+
+  --control-icons: #b2bac1;
+
+  --footer-background: #606070;
+  --footer-font-color: #ffffff;
+  --footer-link-color: #ffcc5c;
+  --footer-link-color-visited: #ffcc5c;
+}
+@media (prefers-color-scheme: light) {
+  :root {
+    --header-background: #404050;
+    --header-font-color: #ffffff;
+
+    --body-background: #ffffff;
+    --body-font-color: #343a40;
+
+    --button-background: #62cb97;
+    --button-border-color: #4ec58a;
+
+    --link-color: #c54e8a;
+    --link-color-visited: #c54e8a;
+
+    --code-background: #f5f6f8;
+    --code-accent-color: #e3e7eb;
+    --code-accent-color-lite: #eff1f3;
+
+    --accent-color: #e9ecef;
+    --accent-color-lite: #f8f9fa;
+
+    --control-icons: #b2bac1;
+
+    --footer-background: #606070;
+    --footer-font-color: #ffffff;
+    --footer-link-color: #ffcc5c;
+    --footer-link-color-visited: #ffcc5c;
+  }
+}
+
+/* Dark mode theming */
+:root[color-mode="dark"] {
+  --header-background: #202830;
+  --header-font-color: #ffffff;
+
+  --body-background: #343a44;
+  --body-font-color: #ced3d8;
+
+  --button-background: #62cb97;
+  --button-border-color: #4ec58a;
+
+  --link-color: #7ac29e;
+  --link-color-visited: #7ac29e;
+
+  --code-background: #2f353a;
+  --code-accent-color: #262b2f;
+  --code-accent-color-lite: #2b3035;
+
+  --accent-color: #2b3035;
+  --accent-color-lite: #2f353a;
+
+  --control-icons: #b2bac1;
+
+  --footer-background: #2f333e;
+  --footer-font-color: #cccccc;
+  --footer-link-color: #7ac29e;
+  --footer-link-color-visited: #7ac29e;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --header-background: #404070;
+    --header-font-color: #ffffff;
+
+    --body-background: #343a40;
+    --body-font-color: #ced3d8;
+
+    --button-background: #62cb97;
+    --button-border-color: #4ec58a;
+
+    --link-color: #7ac29e;
+    --link-color-visited: #7ac29e;
+
+    --code-background: #2f353a;
+    --code-accent-color: #262b2f;
+    --code-accent-color-lite: #2b3035;
+
+    --accent-color: #2b3035;
+    --accent-color-lite: #2f353a;
+
+    --control-icons: #b2bac1;
+
+    --footer-background: #2f333e;
+    --footer-font-color: #cccccc;
+    --footer-link-color: #7ac29e;
+    --footer-link-color-visited: #7ac29e;
+  }
+}
+
+.gdoc-brand__img {
+  width: 48px;
+  height: auto;
+  margin-top: -4px;
+  margin-bottom: -4px;
+}
+
+.gdoc-menu-header > span {
+  display: flex;
+  flex-direction: row-reverse;
+}
+
+span.gdoc-language {
+  margin-right: 20px;
+}

hugo/static/favicon/favicon.svg

@@ -0,0 +1,196 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:osb="http://www.openswatchbook.org/uri/2009/osb"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   sodipodi:docname="favicon.svg"
+   inkscape:version="1.0.2 (e86c870879, 2021-01-15)"
+   id="svg1340"
+   version="1.1"
+   viewBox="0 0 100 100"
+   height="100mm"
+   width="100mm"
+   inkscape:export-filename="/var/home/vitali/SVN/vitastor/docs/static/favicon/favicon-64x64.png"
+   inkscape:export-xdpi="16.26"
+   inkscape:export-ydpi="16.26">
+  <defs
+     id="defs1334">
+    <linearGradient
+       osb:paint="gradient"
+       id="linearGradient866">
+      <stop
+         id="stop862"
+         offset="0"
+         style="stop-color:#c0c0c0;stop-opacity:1" />
+      <stop
+         id="stop864"
+         offset="1"
+         style="stop-color:#000000;stop-opacity:0" />
+    </linearGradient>
+    <linearGradient
+       id="linearGradient846"
+       osb:paint="gradient">
+      <stop
+         style="stop-color:#ffd42a;stop-opacity:1"
+         offset="0"
+         id="stop842" />
+      <stop
+         style="stop-color:#ffa200;stop-opacity:1"
+         offset="1"
+         id="stop844" />
+    </linearGradient>
+    <radialGradient
+       r="50"
+       fy="159.11139"
+       fx="202.36813"
+       cy="159.11139"
+       cx="202.36813"
+       gradientTransform="matrix(1.2462942,-1.2279529,0.77712408,0.78873143,-190.96813,230.1331)"
+       gradientUnits="userSpaceOnUse"
+       id="radialGradient1530"
+       xlink:href="#linearGradient1352"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient1352">
+      <stop
+         style="stop-color:#00c9e6;stop-opacity:1"
+         offset="0"
+         id="stop1348" />
+      <stop
+         style="stop-color:#5240d3;stop-opacity:1"
+         offset="1"
+         id="stop1350" />
+    </linearGradient>
+    <linearGradient
+       y2="62.555599"
+       x2="51.484566"
+       y1="62.555599"
+       x1="38.105473"
+       gradientTransform="rotate(-16.930773,271.11609,-412.42594)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient1508"
+       xlink:href="#linearGradient1323"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient1323">
+      <stop
+         style="stop-color:#000000;stop-opacity:0.47178105"
+         offset="0"
+         id="stop1319" />
+      <stop
+         style="stop-color:#eeaaff;stop-opacity:0;"
+         offset="1"
+         id="stop1321" />
+    </linearGradient>
+    <filter
+       style="color-interpolation-filters:sRGB"
+       inkscape:label="Drop Shadow"
+       id="filter1497"
+       width="2"
+       height="2"
+       x="-0.5"
+       y="-0.5">
+      <feFlood
+         flood-opacity="0.498039"
+         flood-color="rgb(0,0,0)"
+         result="flood"
+         id="feFlood1487" />
+      <feComposite
+         in="flood"
+         in2="SourceGraphic"
+         operator="in"
+         result="composite1"
+         id="feComposite1489" />
+      <feGaussianBlur
+         in="composite1"
+         stdDeviation="6"
+         result="blur"
+         id="feGaussianBlur1491" />
+      <feOffset
+         dx="0"
+         dy="6"
+         result="offset"
+         id="feOffset1493" />
+      <feComposite
+         in="offset"
+         in2="offset"
+         operator="atop"
+         result="composite2"
+         id="feComposite1495" />
+    </filter>
+    <radialGradient
+       r="21.541935"
+       fy="24.614815"
+       fx="45.312912"
+       cy="24.614815"
+       cx="45.312912"
+       gradientTransform="matrix(1.6678615,0.20004527,-0.19318681,1.6106796,108.48083,22.966962)"
+       gradientUnits="userSpaceOnUse"
+       id="radialGradient1506"
+       xlink:href="#linearGradient846"
+       inkscape:collect="always" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:window-maximized="1"
+     inkscape:window-y="0"
+     inkscape:window-x="0"
+     inkscape:window-height="992"
+     inkscape:window-width="1920"
+     fit-margin-bottom="0"
+     fit-margin-right="0"
+     fit-margin-left="0"
+     fit-margin-top="0"
+     showgrid="false"
+     inkscape:document-rotation="0"
+     inkscape:current-layer="layer1"
+     inkscape:document-units="mm"
+     inkscape:cy="83.752268"
+     inkscape:cx="-103.69645"
+     inkscape:zoom="0.7"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0"
+     borderopacity="1.0"
+     bordercolor="#666666"
+     pagecolor="#000000"
+     id="base" />
+  <metadata
+     id="metadata1337">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(-133.26969,-35.630924)"
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Слой 1">
+    <path
+       style="fill:url(#radialGradient1530);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 133.26969,59.089473 50,75.000087 50,-75.000087 z"
+       id="path1528"
+       sodipodi:nodetypes="cccc" />
+    <path
+       d="m 194.29572,89.403603 -8.41706,2.562119 -2.50682,7.49308 7.17785,23.579008 9.60097,-14.40173 z"
+       style="fill:url(#linearGradient1508);fill-opacity:1;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.501961"
+       id="path1459" />
+    <path
+       sodipodi:nodetypes="cccccccc"
+       id="path1463"
+       d="m 157.01826,35.630924 h 53.91343 l -13.21471,31.09726 h 13.99432 l -33.7227,51.291496 4.61762,-37.02885 h -12.45344 z"
+       style="fill:url(#radialGradient1506);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
+  </g>
+</svg>

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/addon.cc

@@ -24,7 +24,7 @@ NAN_MODULE_INIT(InitAddon)
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorImage::Create);
     tpl->SetClassName(Nan::New("Image").ToLocalChecked());
-    tpl->InstanceTemplate()->SetInternalFieldCount(2);
+    tpl->InstanceTemplate()->SetInternalFieldCount(1);
 
     Nan::SetPrototypeMethod(tpl, "read", NodeVitastorImage::Read);
     Nan::SetPrototypeMethod(tpl, "write", NodeVitastorImage::Write);
@@ -67,7 +67,7 @@ NAN_MODULE_INIT(InitAddon)
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);
     tpl->SetClassName(Nan::New("KVListing").ToLocalChecked());
-    tpl->InstanceTemplate()->SetInternalFieldCount(2);
+    tpl->InstanceTemplate()->SetInternalFieldCount(1);
 
     Nan::SetPrototypeMethod(tpl, "next", NodeVitastorKVListing::Next);
     Nan::SetPrototypeMethod(tpl, "close", NodeVitastorKVListing::Close);

node-binding/addon.h

@@ -11,10 +11,10 @@
 
 #define ERRORF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
 
-#define TRACEF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
-#define TRACE(msg) fprintf(stderr, "%s\n", msg);
+//#define TRACEF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
+//#define TRACE(msg) fprintf(stderr, "%s\n", msg);
 
-//#define TRACEF(format, arg) ;
-//#define TRACE(msg) ;
+#define TRACEF(format, ...) ;
+#define TRACE(msg) ;
 
 #endif

node-binding/binding.gyp

@@ -13,7 +13,8 @@
         '<!(pkg-config --cflags vitastor)'
       ],
       'libraries': [
-        '<!(pkg-config --libs vitastor)'
+        '<!(pkg-config --libs vitastor)',
+        '-lvitastor_kv'
       ]
     }
   ]

node-binding/client.cc

@@ -18,12 +18,15 @@
 class NodeVitastorRequest: public Nan::AsyncResource
 {
 public:
-    NodeVitastorRequest(v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
+    NodeVitastorRequest(NodeVitastor *cli, v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
     {
+        this->cli = cli;
         callback.Reset(cb);
     }
 
     iovec iov;
+    std::vector<iovec> iov_list;
+    NodeVitastor *cli = NULL;
     NodeVitastorImage *img = NULL;
     int op = 0;
     uint64_t offset = 0, len = 0, version = 0;
@@ -31,6 +34,13 @@ public:
     Nan::Persistent<v8::Function> callback;
 };
 
+static uint64_t get_ui64(const v8::Local<v8::Value> & val)
+{
+    if (val->IsBigInt())
+        return val->ToBigInt(Nan::GetCurrentContext()).ToLocalChecked()->Uint64Value();
+    return Nan::To<int64_t>(val).FromJust();
+}
+
 //////////////////////////////////////////////////
 // NodeVitastor
 //////////////////////////////////////////////////
@@ -43,6 +53,7 @@ NodeVitastor::NodeVitastor(): Nan::ObjectWrap()
 
 NodeVitastor::~NodeVitastor()
 {
+    TRACE("NodeVitastor: destructor");
     uv_poll_stop(&poll_watcher);
     vitastor_c_destroy(c);
     c = NULL;
@@ -74,9 +85,10 @@ NAN_METHOD(NodeVitastor::Create)
     if (res >= 0)
     {
         cli->eventfd = res;
-        res = uv_poll_init_socket(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
+        res = uv_poll_init(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
         if (res >= 0)
             res = uv_poll_start(&cli->poll_watcher, UV_READABLE, on_io_readable);
+        on_io_readable(&cli->poll_watcher, 0, UV_READABLE);
     }
     if (res < 0)
     {
@@ -97,15 +109,14 @@ void NodeVitastor::on_io_readable(uv_poll_t* handle, int status, int revents)
     if (revents & UV_READABLE)
     {
         NodeVitastor* self = (NodeVitastor*)handle->data;
-        std::unique_lock<std::mutex> lock(self->mu);
         vitastor_c_uring_handle_events(self->c);
     }
 }
 
-static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[argpos+1]).FromJust();
+    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t len = get_ui64(info[argpos+1]);
     uint8_t *buf = (uint8_t*)malloc(len);
     if (!buf)
     {
@@ -113,7 +124,7 @@ static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::V
         return NULL;
     }
     v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(this, callback);
 
     req->offset = offset;
     req->len = len;
@@ -126,73 +137,101 @@ static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::V
 NAN_METHOD(NodeVitastor::Read)
 {
     TRACE("NodeVitastor::Read");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
 
-    auto req = getReadRequest(info, 2);
+    auto req = self->get_read_request(info, 2);
 
-    std::unique_lock<std::mutex> lock(self->mu);
+    self->Ref();
     vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
 }
 
-static NodeVitastorRequest* getWriteRequest(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    char *buf = node::Buffer::Data(info[argpos+1]);
-    uint64_t len = node::Buffer::Length(info[argpos+1]);
+    uint64_t offset = get_ui64(info[argpos+0]);
+    const auto & bufarg = info[argpos+1];
     uint64_t version = 0;
-
-    if (!info[argpos+2].IsEmpty() && info[argpos+2]->IsObject())
+    if (!info[argpos+2].IsEmpty() &&
+        !info[argpos+2]->IsFunction() &&
+        info[argpos+2]->IsObject())
     {
         auto key = Nan::New<v8::String>("version").ToLocalChecked();
         auto params = info[argpos+2].As<v8::Object>();
         auto versionObj = Nan::Get(params, key).ToLocalChecked();
         if (!versionObj.IsEmpty())
-            version = Nan::To<int64_t>(versionObj).FromJust();
+            version = get_ui64(versionObj);
         argpos++;
     }
 
     v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(this, callback);
 
     req->offset = offset;
-    req->len = len;
     req->version = version;
-    req->iov = { .iov_base = buf, .iov_len = req->len };
+
+    if (bufarg->IsArray())
+    {
+        auto buffers = bufarg.As<v8::Array>();
+        req->len = 0;
+        for (uint32_t i = 0; i < buffers->Length(); i++)
+        {
+            auto buffer_obj = Nan::Get(buffers, i).ToLocalChecked();
+            char *buf = node::Buffer::Data(buffer_obj);
+            uint64_t len = node::Buffer::Length(buffer_obj);
+            req->iov_list.push_back({ .iov_base = buf, .iov_len = len });
+            req->len += len;
+        }
+    }
+    else
+    {
+        char *buf = node::Buffer::Data(bufarg);
+        uint64_t len = node::Buffer::Length(bufarg);
+        req->iov = { .iov_base = buf, .iov_len = len };
+        req->len = len;
+    }
 
     return req;
 }
 
-// write(pool, inode, offset, buffer, { version }?, callback(err))
+// write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
 NAN_METHOD(NodeVitastor::Write)
 {
     TRACE("NodeVitastor::Write");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
 
-    auto req = getWriteRequest(info, 2);
+    auto req = self->get_write_request(info, 2);
 
-    std::unique_lock<std::mutex> lock(self->mu);
-    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version, &req->iov, 1, on_write_finish, req);
+    self->Ref();
+    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+        req->iov_list.size() ? req->iov_list.data() : &req->iov,
+        req->iov_list.size() ? req->iov_list.size() : 1,
+        on_write_finish, req);
 }
 
 // sync(callback(err))
 NAN_METHOD(NodeVitastor::Sync)
 {
     TRACE("NodeVitastor::Sync");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(self, callback);
 
-    std::unique_lock<std::mutex> lock(self->mu);
+    self->Ref();
     vitastor_c_sync(self->c, on_write_finish, req);
 }
 
@@ -200,17 +239,20 @@ NAN_METHOD(NodeVitastor::Sync)
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
     TRACE("NodeVitastor::ReadBitmap");
+    if (info.Length() < 6)
+        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
-    uint64_t offset = Nan::To<int64_t>(info[2]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[3]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
+    uint64_t offset = get_ui64(info[2]);
+    uint64_t len = get_ui64(info[3]);
     bool with_parents = Nan::To<bool>(info[4]).FromJust();
     v8::Local<v8::Function> callback = info[5].As<v8::Function>();
 
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(self, callback);
+    self->Ref();
     vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
 }
 
@@ -227,6 +269,7 @@ static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long
 
 void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
 {
+    TRACE("NodeVitastor::on_read_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
@@ -235,7 +278,7 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         free(req->iov.iov_base);
         nanCallback.Call(0, NULL, req);
     }
-    else if (retval < 0)
+    else if (retval < 0 || (uint64_t)retval != req->len)
     {
         free(req->iov.iov_base);
         on_error(req, nanCallback, retval);
@@ -248,20 +291,33 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         args[2] = v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), version);
         nanCallback.Call(3, args, req);
     }
+    req->cli->Unref();
     delete req;
 }
 
 void NodeVitastor::on_write_finish(void *opaque, long retval)
 {
+    TRACE("NodeVitastor::on_write_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
-    on_error(req, nanCallback, retval);
+    if (retval < 0 || (uint64_t)retval != req->len)
+    {
+        on_error(req, nanCallback, retval);
+    }
+    else
+    {
+        v8::Local<v8::Value> args[1];
+        args[0] = Nan::Null();
+        nanCallback.Call(1, args, req);
+    }
+    req->cli->Unref();
     delete req;
 }
 
 void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap)
 {
+    TRACE("NodeVitastor::on_read_bitmap_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
@@ -276,6 +332,7 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
         args[1] = Nan::NewBuffer((char*)bitmap, (retval+7)/8).ToLocalChecked();
         nanCallback.Call(2, args, req);
     }
+    req->cli->Unref();
     delete req;
 }
 
@@ -291,21 +348,23 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
 NAN_METHOD(NodeVitastorImage::Create)
 {
     TRACE("NodeVitastorImage::Create");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to Image(client, name)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     std::string name = std::string(*Nan::Utf8String(info[1].As<v8::String>()));
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
 
     NodeVitastorImage *img = new NodeVitastorImage();
+    img->Wrap(info.This());
+
     img->cli = cli;
     img->name = name;
 
     img->Ref();
     cli->Ref();
-    std::unique_lock<std::mutex> lock(cli->mu);
     vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
 
-    img->Wrap(info.This());
     info.GetReturnValue().Set(info.This());
 }
 
@@ -323,10 +382,12 @@ NodeVitastorImage::~NodeVitastorImage()
 NAN_METHOD(NodeVitastorImage::Read)
 {
     TRACE("NodeVitastorImage::Read");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    auto req = getReadRequest(info, 0);
+    auto req = img->cli->get_read_request(info, 0);
     req->img = img;
     req->op = NODE_VITASTOR_READ;
 
@@ -337,24 +398,29 @@ NAN_METHOD(NodeVitastorImage::Read)
 NAN_METHOD(NodeVitastorImage::Write)
 {
     TRACE("NodeVitastorImage::Write");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    auto req = getWriteRequest(info, 0);
+    auto req = img->cli->get_write_request(info, 0);
     req->img = img;
     req->op = NODE_VITASTOR_WRITE;
 
     img->exec_or_wait(req);
 }
 
+// sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
     TRACE("NodeVitastorImage::Sync");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(img->cli, callback);
     req->img = img;
     req->op = NODE_VITASTOR_SYNC;
 
@@ -365,15 +431,17 @@ NAN_METHOD(NodeVitastorImage::Sync)
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
     TRACE("NodeVitastorImage::ReadBitmap");
+    if (info.Length() < 4)
+        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    uint64_t offset = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t offset = get_ui64(info[0]);
+    uint64_t len = get_ui64(info[1]);
     bool with_parents = Nan::To<bool>(info[2]).FromJust();
     v8::Local<v8::Function> callback = info[3].As<v8::Function>();
 
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(img->cli, callback);
     req->img = img;
     req->op = NODE_VITASTOR_READ_BITMAP;
     req->offset = offset;
@@ -383,14 +451,17 @@ NAN_METHOD(NodeVitastorImage::ReadBitmap)
     img->exec_or_wait(req);
 }
 
+// get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
 NAN_METHOD(NodeVitastorImage::GetInfo)
 {
-    TRACE("NodeVitastorImage::Sync");
+    TRACE("NodeVitastorImage::GetInfo");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(img->cli, callback);
     req->img = img;
     req->op = NODE_VITASTOR_GET_INFO;
 
@@ -412,21 +483,26 @@ void NodeVitastorImage::exec_or_wait(NodeVitastorRequest *req)
 
 void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
 {
-    std::unique_lock<std::mutex> lock(cli->mu);
     if (req->op == NODE_VITASTOR_READ)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
         vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
     }
     else if (req->op == NODE_VITASTOR_WRITE)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        vitastor_c_write(cli->c, ino, req->offset, req->len, req->version, &req->iov, 1, NodeVitastor::on_write_finish, req);
+        cli->Ref();
+        vitastor_c_write(cli->c, ino, req->offset, req->len, req->version,
+            req->iov_list.size() ? req->iov_list.data() : &req->iov,
+            req->iov_list.size() ? req->iov_list.size() : 1,
+            NodeVitastor::on_write_finish, req);
     }
     else if (req->op == NODE_VITASTOR_SYNC)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
         uint32_t imm = vitastor_c_inode_get_immediate_commit(cli->c, ino);
+        cli->Ref();
         if (imm != IMMEDIATE_ALL)
         {
             vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
@@ -439,6 +515,7 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
     else if (req->op == NODE_VITASTOR_READ_BITMAP)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
         vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
     }
     else if (req->op == NODE_VITASTOR_GET_INFO)
@@ -508,10 +585,14 @@ void NodeVitastorImage::on_watch_start(void *opaque, long retval)
 // NodeVitastorKV
 //////////////////////////////////////////////////
 
+Nan::Persistent<v8::Function> NodeVitastorKV::listing_class;
+
 // constructor(node_vitastor)
 NAN_METHOD(NodeVitastorKV::Create)
 {
     TRACE("NodeVitastorKV::Create");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to new KV(client)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
@@ -519,29 +600,32 @@ NAN_METHOD(NodeVitastorKV::Create)
     NodeVitastorKV *kv = new NodeVitastorKV();
     kv->cli = cli;
     {
-        std::unique_lock<std::mutex> lock(cli->mu);
         kv->dbw = new vitastorkv_dbw_t((cluster_client_t*)vitastor_c_get_internal_client(cli->c));
     }
 
     kv->Wrap(info.This());
+    cli->Ref();
     info.GetReturnValue().Set(info.This());
 }
 
 NodeVitastorKV::~NodeVitastorKV()
 {
     delete dbw;
+    cli->Unref();
 }
 
-// open(inode_id, { ...config }, callback(err))
+// open(pool_id, inode_num, { ...config }, callback(err))
 NAN_METHOD(NodeVitastorKV::Open)
 {
     TRACE("NodeVitastorKV::Open");
+    if (info.Length() < 4)
+        Nan::ThrowError("Not enough arguments to open(pool_id, inode_num, { ...config }, callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
-    uint64_t inode_id = Nan::To<int64_t>(info[0]).FromJust();
+    uint64_t inode_id = INODE_WITH_POOL(get_ui64(info[0]), get_ui64(info[1]));
 
-    v8::Local<v8::Object> jsParams = info[1].As<v8::Object>();
+    v8::Local<v8::Object> jsParams = info[2].As<v8::Object>();
     v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
     std::map<std::string, std::string> cfg;
     for (uint32_t i = 0; i < keys->Length(); i++)
@@ -550,8 +634,8 @@ NAN_METHOD(NodeVitastorKV::Open)
         cfg[std::string(*Nan::Utf8String(key))] = std::string(*Nan::Utf8String(Nan::Get(jsParams, key).ToLocalChecked()));
     }
 
-    v8::Local<v8::Function> callback = info[2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    v8::Local<v8::Function> callback = info[3].As<v8::Function>();
+    auto req = new NodeVitastorRequest(kv->cli, callback);
 
     kv->Ref();
     kv->dbw->open(inode_id, cfg, [kv, req](int res)
@@ -570,11 +654,13 @@ NAN_METHOD(NodeVitastorKV::Open)
 NAN_METHOD(NodeVitastorKV::Close)
 {
     TRACE("NodeVitastorKV::Close");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to close(callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(kv->cli, callback);
 
     kv->Ref();
     kv->dbw->close([kv, req]()
@@ -591,6 +677,8 @@ NAN_METHOD(NodeVitastorKV::Close)
 NAN_METHOD(NodeVitastorKV::SetConfig)
 {
     TRACE("NodeVitastorKV::SetConfig");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to set_config({ ...config })");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -627,7 +715,7 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
     std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[1].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(kv->cli, callback);
 
     kv->Ref();
     kv->dbw->get(key, [kv, req](int res, const std::string & value)
@@ -647,6 +735,8 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
 NAN_METHOD(NodeVitastorKV::Get)
 {
     TRACE("NodeVitastorKV::Get");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to get(key, callback(err, value))");
     get_impl(info, false);
 }
 
@@ -654,6 +744,8 @@ NAN_METHOD(NodeVitastorKV::Get)
 NAN_METHOD(NodeVitastorKV::GetCached)
 {
     TRACE("NodeVitastorKV::GetCached");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to get_cached(key, callback(err, value))");
     get_impl(info, true);
 }
 
@@ -672,10 +764,12 @@ static std::function<bool(int, const std::string &)> make_cas_callback(NodeVitas
     };
 }
 
-// set(key, value, callback(err), cas_compare(old_value))
+// set(key, value, callback(err), cas_compare(old_value)?)
 NAN_METHOD(NodeVitastorKV::Set)
 {
     TRACE("NodeVitastorKV::Set");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to set(key, value, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -684,13 +778,13 @@ NAN_METHOD(NodeVitastorKV::Set)
     std::string value(*Nan::Utf8String(info[1].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[2].As<v8::Function>();
-    NodeVitastorRequest *req = new NodeVitastorRequest(callback), *cas_req = NULL;
+    NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
 
     std::function<bool(int, const std::string &)> cas_cb;
     if (info.Length() > 3 && info[3]->IsObject())
     {
         v8::Local<v8::Function> cas_callback = info[3].As<v8::Function>();
-        cas_req = new NodeVitastorRequest(cas_callback);
+        cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
         cas_cb = make_cas_callback(cas_req);
     }
 
@@ -709,10 +803,12 @@ NAN_METHOD(NodeVitastorKV::Set)
     }, cas_cb);
 }
 
-// del(key, callback(err), cas_compare(old_value))
+// del(key, callback(err), cas_compare(old_value)?)
 NAN_METHOD(NodeVitastorKV::Del)
 {
     TRACE("NodeVitastorKV::Del");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to del(key, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -720,13 +816,13 @@ NAN_METHOD(NodeVitastorKV::Del)
     std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[1].As<v8::Function>();
-    NodeVitastorRequest *req = new NodeVitastorRequest(callback), *cas_req = NULL;
+    NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
 
     std::function<bool(int, const std::string &)> cas_cb;
     if (info.Length() > 2 && info[2]->IsObject())
     {
         v8::Local<v8::Function> cas_callback = info[2].As<v8::Function>();
-        cas_req = new NodeVitastorRequest(cas_callback);
+        cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
         cas_cb = make_cas_callback(cas_req);
     }
 
@@ -762,6 +858,14 @@ NAN_METHOD(NodeVitastorKV::List)
     info.GetReturnValue().Set(Nan::NewInstance(cons, narg, args).ToLocalChecked());
 }
 
+/*NAN_METHOD(NodeVitastorKV::Destroy)
+{
+    TRACE("NodeVitastorKV::Destroy");
+    NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
+    if (!kv->dead)
+        kv->Unref();
+}*/
+
 //////////////////////////////////////////////////
 // NodeVitastorKVListing
 //////////////////////////////////////////////////
@@ -783,12 +887,10 @@ NAN_METHOD(NodeVitastorKVListing::Create)
 
     NodeVitastorKVListing *list = new NodeVitastorKVListing();
     list->kv = kv;
-    {
-        std::unique_lock<std::mutex> lock(kv->cli->mu);
-        list->handle = list->kv->dbw->list_start(start_key);
-    }
+    list->handle = list->kv->dbw->list_start(start_key);
 
     list->Wrap(info.This());
+    kv->Ref();
     info.GetReturnValue().Set(info.This());
 }
 
@@ -796,35 +898,54 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
 {
     if (handle)
     {
-        std::unique_lock<std::mutex> lock(kv->cli->mu);
         kv->dbw->list_close(handle);
         handle = NULL;
     }
+    if (iter)
+    {
+        delete iter;
+        iter = NULL;
+    }
+    kv->Unref();
 }
 
-// next(callback(err, value))
+// next(callback(err, value)?)
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
     TRACE("NodeVitastorKVListing::Next");
 
     NodeVitastorKVListing* list = Nan::ObjectWrap::Unwrap<NodeVitastorKVListing>(info.This());
 
-    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    if (info.Length() > 0)
+    {
+        v8::Local<v8::Function> callback = info[0].As<v8::Function>();
+        if (list->iter)
+        {
+            delete list->iter;
+        }
+        list->iter = new NodeVitastorRequest(list->kv->cli, callback);
+    }
     if (!list->handle)
     {
         // Already closed
-        Nan::Callback nanCallback(Nan::New(req->callback));
-        v8::Local<v8::Value> args[1];
-        args[0] = Nan::New<v8::Int32>(-EINVAL);
-        nanCallback.Call(1, args, req);
-        delete req;
+        if (list->iter)
+        {
+            auto req = list->iter;
+            list->iter = NULL;
+            Nan::Callback nanCallback(Nan::New(req->callback));
+            v8::Local<v8::Value> args[1];
+            args[0] = Nan::New<v8::Int32>(-EINVAL);
+            nanCallback.Call(1, args, req);
+            delete req;
+        }
         return;
     }
 
     list->kv->Ref();
-    list->kv->dbw->list_next(list->handle, [list, req](int res, const std::string & key, const std::string & value)
+    list->kv->dbw->list_next(list->handle, [list](int res, const std::string & key, const std::string & value)
     {
+        auto req = list->iter;
+        list->iter = NULL;
         Nan::HandleScope scope;
         Nan::Callback nanCallback(Nan::New(req->callback));
         v8::Local<v8::Value> args[3];
@@ -832,7 +953,10 @@ NAN_METHOD(NodeVitastorKVListing::Next)
         args[1] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(key).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         args[2] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         nanCallback.Call(3, args, req);
-        delete req;
+        if (list->iter)
+            delete req;
+        else
+            list->iter = req;
         list->kv->Unref();
     });
 }
@@ -846,8 +970,12 @@ NAN_METHOD(NodeVitastorKVListing::Close)
 
     if (list->handle)
     {
-        std::unique_lock<std::mutex> lock(list->kv->cli->mu);
         list->kv->dbw->list_close(list->handle);
         list->handle = NULL;
     }
+    if (list->iter)
+    {
+        delete list->iter;
+        list->iter = NULL;
+    }
 }

node-binding/client.h

@@ -4,8 +4,6 @@
 #ifndef NODE_VITASTOR_CLIENT_H
 #define NODE_VITASTOR_CLIENT_H
 
-#include <mutex>
-
 #include <nan.h>
 #include <vitastor_c.h>
 #include <vitastor_kv.h>
@@ -19,7 +17,7 @@ public:
     static NAN_METHOD(Create);
     // read(pool, inode, offset, len, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(pool, inode, offset, buffer, { version }?, callback(err))
+    // write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
     static NAN_METHOD(Write);
     // sync(callback(err))
     static NAN_METHOD(Sync);
@@ -34,8 +32,6 @@ private:
     vitastor_c *c = NULL;
     int eventfd = -1;
     uv_poll_t poll_watcher;
-    // FIXME: Is it really needed?
-    std::mutex mu;
 
     NodeVitastor();
 
@@ -44,6 +40,9 @@ private:
     static void on_write_finish(void *opaque, long retval);
     static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
 
+    NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+    NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+
     friend class NodeVitastorImage;
     friend class NodeVitastorKV;
     friend class NodeVitastorKVListing;
@@ -56,7 +55,7 @@ public:
     static NAN_METHOD(Create);
     // read(offset, len, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(offset, buffer, { version }?, callback(err))
+    // write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
     static NAN_METHOD(Write);
     // sync(callback(err))
     static NAN_METHOD(Sync);
@@ -74,8 +73,6 @@ private:
     std::vector<NodeVitastorRequest*> on_init;
     Nan::Persistent<v8::Object> cliObj;
 
-    NodeVitastorImage();
-
     static void on_watch_start(void *opaque, long retval);
     void exec_request(NodeVitastorRequest *req);
     void exec_or_wait(NodeVitastorRequest *req);
@@ -86,7 +83,7 @@ class NodeVitastorKV: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor)
     static NAN_METHOD(Create);
-    // open(inode_id, { ...config }, callback(err))
+    // open(pool_id, inode_num, { ...config }, callback(err))
     static NAN_METHOD(Open);
     // set_config({ ...config })
     static NAN_METHOD(SetConfig);
@@ -98,9 +95,9 @@ public:
     static NAN_METHOD(Get);
     // get_cached(key, callback(err, value))
     static NAN_METHOD(GetCached);
-    // set(key, value, callback(err), cas_compare(old_value))
+    // set(key, value, callback(err), cas_compare(old_value)?)
     static NAN_METHOD(Set);
-    // del(key, callback(err), cas_compare(old_value))
+    // del(key, callback(err), cas_compare(old_value)?)
     static NAN_METHOD(Del);
     // list(start_key?)
     static NAN_METHOD(List);
@@ -113,8 +110,6 @@ private:
     NodeVitastor *cli = NULL;
     vitastorkv_dbw_t *dbw = NULL;
 
-    NodeVitastorKV();
-
     static void get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info, bool allow_cache);
 
     friend class NodeVitastorKVListing;
@@ -125,7 +120,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor_kv, start_key?)
     static NAN_METHOD(Create);
-    // next(callback(err, value))
+    // next(callback(err, value)?)
     static NAN_METHOD(Next);
     // close()
     static NAN_METHOD(Close);
@@ -135,8 +130,7 @@ public:
 private:
     NodeVitastorKV *kv = NULL;
     void *handle = NULL;
-
-    NodeVitastorKVListing();
+    NodeVitastorRequest *iter = NULL;
 };
 
 #endif

node-binding/index.js

@@ -0,0 +1 @@
+module.exports = require('bindings')('addon.node');

opennebula/README.md

@@ -0,0 +1 @@
+See [../docs/installation/opennebula.en.md](../docs/installation/opennebula.en.md).

opennebula/install.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+reapply_patch() {
+	if ! patch -f --dry-run -F 0 -R $1 < $2 >/dev/null; then
+		already_applied=0
+		if ! patch --no-backup-if-mismatch -r - -F 0 -f $1 < $2; then
+			applied_ok=0
+			echo "ERROR: Failed to patch file $1, please apply the patch $2 manually"
+		fi
+	fi
+}
+echo "Reapplying Vitastor patches to OpenNebula's oned.conf, vmm_execrc and downloader.sh"
+already_applied=1
+applied_ok=1
+reapply_patch /var/lib/one/remotes/datastore/downloader.sh /var/lib/one/remotes/datastore/vitastor/downloader-vitastor.sh.diff
+reapply_patch /etc/one/oned.conf /var/lib/one/remotes/datastore/vitastor/oned.conf.diff
+reapply_patch /etc/one/vmm_exec/vmm_execrc /var/lib/one/remotes/datastore/vitastor/vmm_execrc.diff
+if [[ "$already_applied" = 1 ]]; then
+	echo "OK: Vitastor OpenNebula patches are already applied"
+elif [[ "$applied_ok" = 1 ]]; then
+	echo "OK: Vitastor OpenNebula patches successfully applied"
+fi
+if [[ -f /etc/apparmor.d/local/abstractions/libvirt-qemu ]]; then
+	if ! grep -q /etc/vitastor/vitastor.conf /etc/apparmor.d/local/abstractions/libvirt-qemu; then
+		echo '  "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu
+	fi
+fi

opennebula/remotes/datastore/vitastor/clone

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to copy a VM image (SRC) to the image repository as DST
+
+# -------- Set up the environment to source common tools & conf ------------
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get cp and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
+    /DS_DRIVER_ACTION_DATA/IMAGE/PATH \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+BASE_PATH="${XPATH_ELEMENTS[i++]}"
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+POOL_NAME="${XPATH_ELEMENTS[i++]}"
+IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
+SRC="${XPATH_ELEMENTS[i++]}"
+SIZE="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+if [ -n "$POOL_NAME" ]; then
+    CLI="$CLI --pool ${POOL_NAME}"
+fi
+
+SAFE_DIRS=""
+
+DST="${IMAGE_PREFIX}-${ID}"
+
+ssh_exec_and_log "$DST_HOST" "$CLI create --parent $SRC $DST" \
+    "Error during $CLI create --parent $SRC $DST in $DST_HOST"
+
+ssh_exec_and_log "$DST_HOST" "$CLI flatten $DST" \
+    "Error during $CLI create flatten $DST in $DST_HOST"
+
+echo "$DST raw"

opennebula/remotes/datastore/vitastor/cp

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to copy a local image SRC to the image repository as DST
+
+# -------- Set up the environment to source common tools & conf ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get cp and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+export DRV_ACTION
+
+UTILS_PATH="${DRIVER_PATH}/.."
+
+XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/STAGING_DIR \
+    /DS_DRIVER_ACTION_DATA/IMAGE/PATH \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+BASE_PATH="${XPATH_ELEMENTS[i++]}"
+RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}"
+SAFE_DIRS="${XPATH_ELEMENTS[i++]}"
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+POOL_NAME="${XPATH_ELEMENTS[i++]}"
+IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
+STAGING_DIR="${XPATH_ELEMENTS[i++]:-/var/tmp}"
+SRC="${XPATH_ELEMENTS[i++]}"
+SIZE="${XPATH_ELEMENTS[i++]}"
+MD5="${XPATH_ELEMENTS[i++]}"
+SHA1="${XPATH_ELEMENTS[i++]}"
+NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}"
+LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+QEMU_ARG=""
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+    QEMU_ARG=":config_path=${VITASTOR_CONF}"
+fi
+if [ -n "$POOL_NAME" ]; then
+    CLI="$CLI --pool ${POOL_NAME}"
+fi
+
+set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS"
+
+IMAGE_HASH=`generate_image_hash`
+TMP_DST="$STAGING_DIR/$IMAGE_HASH"
+
+DST="${IMAGE_PREFIX}-${ID}"
+
+DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
+
+COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
+
+case $SRC in
+http://*|https://*)
+    log "Downloading $SRC to the image repository"
+
+    DUMP="$COPY_COMMAND"
+    ;;
+
+*)
+    if [ `check_restricted $SRC` -eq 1 ]; then
+        log_error "Not allowed to copy images from $RESTRICTED_DIRS"
+        error_message "Not allowed to copy image file $SRC"
+        exit -1
+    fi
+
+    log "Copying local image $SRC to the image repository"
+
+    DUMP="$COPY_COMMAND"
+    ;;
+esac
+
+multiline_exec_and_log "set -e -o pipefail; $DUMP | $SSH $DST_HOST $DD of=$TMP_DST bs=1M" \
+    "Error copying $SRC to $DST_HOST:$TMP_DST"
+
+REGISTER_CMD=$(cat <<EOF
+    set -e -o pipefail
+
+    SIZE=\$($QEMU_IMG info --output json "$TMP_DST" | jq -r '.["virtual-size"]')
+    $CLI create -s \$SIZE "$DST"
+    $QEMU_IMG convert -O raw "$TMP_DST" "vitastor:image=$DST$QEMU_ARG"
+
+    # remove original
+    $RM -f $TMP_DST
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$REGISTER_CMD" "Error registering $DST in $DST_HOST"
+
+echo "$DST raw"

opennebula/remotes/datastore/vitastor/downloader-patched.sh

@@ -0,0 +1,555 @@
+#!/bin/bash
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+    VAR_LOCATION=/var/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+    VAR_LOCATION=$ONE_LOCATION/var
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+
+# Escape single quotes
+function esc_sq
+{
+    echo "$1" | sed -e "s/'/'\\\''/g"
+}
+
+# Execute a command (first parameter) and use the first kb of stdout
+# to determine the file type
+function get_type
+{
+    if [ "$NO_DECOMPRESS" = "yes" ]; then
+        echo "application/octet-stream"
+    else
+        command=$1
+
+        ( eval "$command" | head -n 1024 | file -b --mime-type - ) 2>/dev/null
+    fi
+}
+
+# Gets the command needed to decompress an stream.
+function get_decompressor
+{
+    type=$1
+
+    case "$type" in
+    "application/x-gzip"|"application/gzip")
+        echo "gunzip -c -"
+        ;;
+    "application/x-bzip2")
+        echo "bunzip2 -qc -"
+        ;;
+    "application/x-xz")
+        echo "unxz -c -"
+        ;;
+    *)
+        echo "cat"
+        ;;
+    esac
+}
+
+# Function called to decompress a stream. The first parameter is the command
+# used to decompress the stream. Second parameter is the output file or
+# - for stdout.
+function decompress
+{
+    command="$1"
+    to="$2"
+
+    if [ "$to" = "-" ]; then
+        $command
+    else
+        $command > "$to"
+    fi
+}
+
+# Function called to hash a stream. First parameter is the algorithm name.
+function hasher
+{
+    if [ -n "$1" ]; then
+        openssl dgst -$1 | awk '{print $NF}' > $HASH_FILE
+    else
+        # Needs something consuming stdin or the pipe will break
+        cat >/dev/null
+    fi
+}
+
+# Unarchives a tar or a zip a file to a directory with the same name.
+function unarchive
+{
+    TO="$1"
+
+    file_type=$(get_type "cat $TO")
+
+    tmp="$TO"
+
+    # Add full path if it is relative
+    if [ ${tmp:0:1} != "/" ]; then
+        tmp="$PWD/$tmp"
+    fi
+
+    IN="$tmp.tmp"
+    OUT="$tmp"
+
+    case "$file_type" in
+    "application/x-tar")
+        command="tar -xf $IN -C $OUT"
+        ;;
+    "application/zip")
+        command="unzip -d $OUT $IN"
+        ;;
+    *)
+        command=""
+        ;;
+    esac
+
+    if [ -n "$command" ]; then
+        mv "$OUT" "$IN"
+        mkdir "$OUT"
+
+        $command
+
+        if [ "$?" != "0" ]; then
+            echo "Error uncompressing archive" >&2
+            exit -1
+        fi
+
+        rm "$IN"
+    fi
+}
+
+function s3_env
+{
+    XPATH="$DRIVER_PATH/xpath.rb -b $DRV_ACTION"
+
+    unset i j XPATH_ELEMENTS
+
+    while IFS= read -r -d '' element; do
+        XPATH_ELEMENTS[i++]="$element"
+    done < <($XPATH     /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ACCESS_KEY_ID \
+                        /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY \
+                        /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/REGION \
+                        /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/AWS \
+                        /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ENDPOINT)
+
+    S3_ACCESS_KEY_ID="${XPATH_ELEMENTS[j++]}"
+    S3_SECRET_ACCESS_KEY="${XPATH_ELEMENTS[j++]}"
+    S3_REGION="${XPATH_ELEMENTS[j++]}"
+    S3_AWS="${XPATH_ELEMENTS[j++]}"
+    S3_ENDPOINT="${XPATH_ELEMENTS[j++]}"
+
+    CURRENT_DATE_DAY="$(date -u '+%Y%m%d')"
+    CURRENT_DATE_ISO8601="${CURRENT_DATE_DAY}T$(date -u '+%H%M%S')Z"
+}
+
+# Create an SHA-256 hash in hexadecimal.
+# Usage:
+#   hash_sha256 <string>
+function hash_sha256 {
+  printf "${1}" | openssl dgst -sha256 | sed 's/^.* //'
+}
+
+# Create an SHA-256 hmac in hexadecimal.
+# Usage:
+#   hmac_sha256 <key> <data>
+function hmac_sha256 {
+  printf "${2}" | openssl dgst -sha256 -mac HMAC -macopt "${1}" | sed 's/^.* //'
+}
+
+# Create the signature.
+# Usage:
+#   create_signature
+function create_signature {
+    stringToSign="AWS4-HMAC-SHA256\n${CURRENT_DATE_ISO8601}\n${CURRENT_DATE_DAY}/${S3_REGION}/s3/aws4_request\n$(hash_sha256 "${HTTP_CANONICAL_REQUEST}")"
+    dateKey=$(hmac_sha256 key:"AWS4${S3_SECRET_ACCESS_KEY}" "${CURRENT_DATE_DAY}")
+    regionKey=$(hmac_sha256 hexkey:"${dateKey}" "${S3_REGION}")
+    serviceKey=$(hmac_sha256 hexkey:"${regionKey}" "s3")
+    signingKey=$(hmac_sha256 hexkey:"${serviceKey}" "aws4_request")
+
+    printf "${stringToSign}" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"${signingKey}" | sed 's/.*(stdin)= //'
+}
+
+function s3_curl_args
+{
+    FROM="$1"
+
+    ENDPOINT="$S3_ENDPOINT"
+    OBJECT=$(basename "$FROM")
+    BUCKET=$(basename $(dirname "$FROM"))
+
+    DATE="`date -u +'%a, %d %b %Y %H:%M:%S GMT'`"
+    AUTH_STRING="GET\n\n\n${DATE}\n/${BUCKET}/${OBJECT}"
+
+    SIGNED_AUTH_STRING=`echo -en "$AUTH_STRING" | \
+                        openssl sha1 -hmac ${S3_SECRET_ACCESS_KEY} -binary | \
+                        base64`
+
+    echo " -H \"Date: ${DATE}\"" \
+         " -H \"Authorization: AWS ${S3_ACCESS_KEY_ID}:${SIGNED_AUTH_STRING}\"" \
+         " '$(esc_sq "${ENDPOINT}/${BUCKET}/${OBJECT}")'"
+}
+
+function s3_curl_args_aws
+{
+    FROM="$1"
+
+    OBJECT=$(basename "$FROM")
+    BUCKET=$(basename "$(dirname "$FROM")")
+
+    ENDPOINT="$BUCKET.s3.amazonaws.com"
+
+    AWS_S3_PATH="$(echo $OBJECT | sed 's;^\([^/]\);/\1;')"
+
+    HTTP_REQUEST_PAYLOAD_HASH="$(echo "" | openssl dgst -sha256 | sed 's/^.* //')"
+    HTTP_CANONICAL_REQUEST_URI="${AWS_S3_PATH}"
+    HTTP_REQUEST_CONTENT_TYPE='application/octet-stream'
+
+    HTTP_CANONICAL_REQUEST_HEADERS="content-type:${HTTP_REQUEST_CONTENT_TYPE}
+host:${ENDPOINT}
+x-amz-content-sha256:${HTTP_REQUEST_PAYLOAD_HASH}
+x-amz-date:${CURRENT_DATE_ISO8601}"
+
+    HTTP_REQUEST_SIGNED_HEADERS="content-type;host;x-amz-content-sha256;x-amz-date"
+HTTP_CANONICAL_REQUEST="GET
+${HTTP_CANONICAL_REQUEST_URI}\n
+${HTTP_CANONICAL_REQUEST_HEADERS}\n
+${HTTP_REQUEST_SIGNED_HEADERS}
+${HTTP_REQUEST_PAYLOAD_HASH}"
+
+    SIGNATURE="$(create_signature)"
+    HTTP_REQUEST_AUTHORIZATION_HEADER="AWS4-HMAC-SHA256 Credential=${S3_ACCESS_KEY_ID}/${CURRENT_DATE_DAY}/${S3_REGION}/s3/aws4_request, SignedHeaders=${HTTP_REQUEST_SIGNED_HEADERS}, Signature=${SIGNATURE}"
+
+    echo " -H \"Authorization: ${HTTP_REQUEST_AUTHORIZATION_HEADER}\"" \
+         " -H \"content-type: ${HTTP_REQUEST_CONTENT_TYPE}\"" \
+         " -H \"x-amz-content-sha256: ${HTTP_REQUEST_PAYLOAD_HASH}\"" \
+         " -H \"x-amz-date: ${CURRENT_DATE_ISO8601}\"" \
+         " \"https://${ENDPOINT}${HTTP_CANONICAL_REQUEST_URI}\""
+}
+
+function get_rbd_cmd
+{
+    local i j URL_ELEMENTS
+
+    FROM="$1"
+
+    URL_RB="$DRIVER_PATH/url.rb"
+
+    while IFS= read -r -d '' element; do
+        URL_ELEMENTS[i++]="$element"
+    done < <($URL_RB    "$FROM" \
+                        USER \
+                        HOST \
+                        SOURCE \
+                        PARAM_DS \
+                        PARAM_CEPH_USER \
+                        PARAM_CEPH_KEY \
+                        PARAM_CEPH_CONF)
+
+    USER="${URL_ELEMENTS[j++]}"
+    DST_HOST="${URL_ELEMENTS[j++]}"
+    SOURCE="${URL_ELEMENTS[j++]}"
+    DS="${URL_ELEMENTS[j++]}"
+    CEPH_USER="${URL_ELEMENTS[j++]}"
+    CEPH_KEY="${URL_ELEMENTS[j++]}"
+    CEPH_CONF="${URL_ELEMENTS[j++]}"
+
+    # Remove leading '/'
+    SOURCE="${SOURCE#/}"
+
+    if [ -n "$USER" ]; then
+        DST_HOST="$USER@$DST_HOST"
+    fi
+
+    if [ -n "$CEPH_USER" ]; then
+        RBD="$RBD --id '$(esc_sq "${CEPH_USER}")'"
+    fi
+
+    if [ -n "$CEPH_KEY" ]; then
+        RBD="$RBD --keyfile '$(esc_sq "${CEPH_KEY}")'"
+    fi
+
+    if [ -n "$CEPH_CONF" ]; then
+        RBD="$RBD --conf '$(esc_sq "${CEPH_CONF}")'"
+    fi
+
+    echo "ssh '$(esc_sq "$DST_HOST")' \"$RBD export '$(esc_sq "$SOURCE")' -\""
+}
+
+function get_vitastor_cmd
+{
+    local i j URL_ELEMENTS
+
+    FROM="$1"
+
+    URL_RB="$DRIVER_PATH/url.rb"
+
+    while IFS= read -r -d '' element; do
+        URL_ELEMENTS[i++]="$element"
+    done < <($URL_RB    "$FROM" \
+                        USER \
+                        HOST \
+                        SOURCE \
+                        PARAM_DS \
+                        PARAM_VITASTOR_CONF)
+
+    USER="${URL_ELEMENTS[j++]}"
+    DST_HOST="${URL_ELEMENTS[j++]}"
+    SOURCE="${URL_ELEMENTS[j++]}"
+    DS="${URL_ELEMENTS[j++]}"
+    VITASTOR_CONF="${URL_ELEMENTS[j++]}"
+
+    # Remove leading '/'
+    SOURCE="${SOURCE#/}"
+
+    if [ -n "$USER" ]; then
+        DST_HOST="$USER@$DST_HOST"
+    fi
+
+    local CLI
+    CLI="vitastor-cli"
+    if [ -n "$VITASTOR_CONF" ]; then
+        CLI="$CLI --config_path '$(esc_sq "${VITASTOR_CONF}")'"
+    fi
+
+    echo "ssh '$(esc_sq "$DST_HOST")' \"$CLI dd iimg='$(esc_sq "$SOURCE")'\""
+}
+
+# Compare 2 version strings using sort -V
+# Usage:
+#   verlte "3.2.9" "3.4.0"
+function verlte() {
+    [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+}
+
+# Returns curl retry options based on its version
+function curl_retry_args {
+    [ "$NO_RETRY" = "yes" ] && return
+
+    RETRY_ARGS="--retry 3 --retry-delay 3"
+
+    CURL_VER=`curl --version | grep -o 'curl [0-9\.]*' | awk '{print $2}'`
+
+    # To retry also on conn-reset-by-peer fresh curl is needed
+    if verlte "7.71.0" "$CURL_VER" && [ -z ${MAX_SIZE} ] ; then
+        RETRY_ARGS+=" --retry-all-errors"
+    fi
+
+    echo $RETRY_ARGS
+}
+
+TEMP=`getopt -o m:s:l:c:no -l md5:,sha1:,limit:,max-size:,nodecomp,noretry -- "$@"`
+
+if [ $? != 0 ] ; then
+    echo "Arguments error" >&2
+    exit -1
+fi
+
+eval set -- "$TEMP"
+
+while true; do
+    case "$1" in
+        -m|--md5)
+            HASH_TYPE=md5
+            HASH=$2
+            shift 2
+            ;;
+        -s|--sha1)
+            HASH_TYPE=sha1
+            HASH=$2
+            shift 2
+            ;;
+        -n|--nodecomp)
+            export NO_DECOMPRESS="yes"
+            shift
+            ;;
+        -l|--limit)
+            export LIMIT_RATE="$2"
+            shift 2
+            ;;
+        -c|--max-size)
+            export MAX_SIZE="$2"
+            shift 2
+            ;;
+        -o|--noretry)
+            export NO_RETRY="yes"
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+FROM="$1"
+TO="$2"
+
+if [ -n "${HASH_TYPE}" -a -n "${MAX_SIZE}" ]; then
+    echo "Hash check not supported for partial downloads" >&2
+    exit -1
+else
+    # File used by the hasher function to store the resulting hash
+    export HASH_FILE="/tmp/downloader.hash.$$"
+fi
+
+GLOBAL_CURL_ARGS="--fail -sS -k -L $(curl_retry_args)"
+
+case "$FROM" in
+http://*|https://*)
+    # -k  so it does not check the certificate
+    # -L  to follow redirects
+    # -sS to hide output except on failure
+    # --limit_rate to limit the bw
+    curl_args="$GLOBAL_CURL_ARGS '$(esc_sq "${FROM}")'"
+
+    if [ -n "$LIMIT_RATE" ]; then
+        curl_args="--limit-rate $LIMIT_RATE $curl_args"
+    fi
+
+    command="curl $curl_args"
+    ;;
+ssh://*)
+    # pseudo-url for ssh transfers ssh://user@host:path
+    # -l to limit the bw
+    ssh_src=${FROM#ssh://}
+    ssh_arg=(${ssh_src/:/ })
+
+    rmt_cmd="\"cat '$(esc_sq "${ssh_arg[1]}")'\""
+
+    command="ssh ${ssh_arg[0]} $rmt_cmd"
+    ;;
+s3://*)
+    # Read s3 environment
+    s3_env
+
+    if [ -z "$S3_ACCESS_KEY_ID" -o -z "$S3_SECRET_ACCESS_KEY" ]; then
+        echo "S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are required" >&2
+        exit -1
+    fi
+
+    curl_args=""
+
+    if [[ "$S3_AWS" =~ (no|NO) ]]; then
+        curl_args="$(s3_curl_args "$FROM")"
+    else
+        curl_args="$(s3_curl_args_aws "$FROM")"
+    fi
+
+    command="curl $GLOBAL_CURL_ARGS $curl_args"
+    ;;
+rbd://*)
+    command="$(get_rbd_cmd "$FROM")"
+    ;;
+vitastor://*)
+    command="$(get_vitastor_cmd "$FROM")"
+    ;;
+vcenter://*)
+    command="$VAR_LOCATION/remotes/datastore/vcenter_downloader.rb '$(esc_sq "$FROM")'"
+    ;;
+lxd://*)
+    file_type="application/octet-stream"
+    command="$VAR_LOCATION/remotes/datastore/lxd_downloader.sh \"$FROM\""
+    ;;
+restic://*)
+    eval `$VAR_LOCATION/remotes/datastore/restic_downloader.rb "$FROM" | grep -e '^command=' -e '^clean_command='`
+    ;;
+rsync://*)
+    eval `$VAR_LOCATION/remotes/datastore/rsync_downloader.rb "$FROM" | grep -e '^command=' -e '^clean_command='`
+    ;;
+*)
+    if [ ! -r $FROM ]; then
+        echo "Cannot read from $FROM" >&2
+        exit -1
+    fi
+    command="cat '$(esc_sq "$FROM")'"
+    ;;
+esac
+
+[ -z "$file_type" ] && file_type=$(get_type "$command")
+decompressor=$(get_decompressor "$file_type")
+
+if [ -z "${MAX_SIZE}" ]; then
+    eval "$command" | \
+        tee >( hasher $HASH_TYPE) | \
+        decompress "$decompressor" "$TO"
+
+    if [ "$?" != "0" -o "$PIPESTATUS" != "0" ]; then
+        echo "Error copying" >&2
+        exit -1
+    fi
+else
+    # Order of the 'head' command is here on purpose:
+    # 1. We want to download more bytes than needed to get a requested
+    #    number of bytes on the output. Decompressor may need more
+    #    data to decompress the stream.
+    # 2. Decompressor command is also misused to detect SIGPIPE error.
+    eval "$command" | \
+        decompress "$decompressor" "$TO" 2>/dev/null | \
+        head -c "${MAX_SIZE}"
+
+    # Following table shows exit codes of each command
+    # in the pipe for various scenarios:
+    #
+    # ----------------------------------------------------
+    # | $COMMAND | TYPE          | PIPESTATUS | BEHAVIOUR
+    # ----------------------------------------------------
+    # | cat      | partial       | 141 141  0 | OK
+    # | cat      | full          |   0   0  0 | OK
+    # | cat      | error         |   1   0  0 | fail
+    # | curl     | partial       |  23 141  0 | OK
+    # | curl     | full          |   0   0  0 | OK
+    # | curl     | error         |  22   0  0 | fail
+    # | ssh      | partial       | 255 141  0 | OK
+    # | ssh      | full          |   0   0  0 | OK
+    # | ssh      | error ssh     | 255   0  0 | fail
+    # | ssh      | error ssh cat |   1   0  0 | fail
+    if [ \( "${PIPESTATUS[0]}" != '0' -a "${PIPESTATUS[1]}" = '0' \) \
+         -o \( "${PIPESTATUS[1]}" != '0' -a "${PIPESTATUS[1]}" != '141' \) \
+         -o \( "${PIPESTATUS[2]}" != "0" \) ];
+    then
+        echo "Error copying" >&2
+        exit -1
+    fi
+fi
+
+if [ -n "$HASH_TYPE" ]; then
+    HASH_RESULT=$( cat $HASH_FILE)
+    rm $HASH_FILE
+    if [ "$HASH_RESULT" != "$HASH" ]; then
+        echo "Hash does not match" >&2
+        exit -1
+    fi
+fi
+
+# Unarchive only if the destination is filesystem
+if [ "$TO" != "-" ]; then
+    unarchive "$TO"
+fi
+
+# Perform any clean operation
+if [ -n "${clean_command}" ]; then
+    eval "$clean_command"
+fi

opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff

@@ -0,0 +1,60 @@
+diff --git /var/lib/one/remotes/datastore/downloader.sh /var/lib/one/remotes/datastore/downloader.sh
+index 9b75d8ee4b..09d2a5d41d 100755
+--- /var/lib/one/remotes/datastore/downloader.sh
++++ /var/lib/one/remotes/datastore/downloader.sh
+@@ -295,6 +295,45 @@ function get_rbd_cmd
+     echo "ssh '$(esc_sq "$DST_HOST")' \"$RBD export '$(esc_sq "$SOURCE")' -\""
+ }
+ 
++function get_vitastor_cmd
++{
++    local i j URL_ELEMENTS
++
++    FROM="$1"
++
++    URL_RB="$DRIVER_PATH/url.rb"
++
++    while IFS= read -r -d '' element; do
++        URL_ELEMENTS[i++]="$element"
++    done < <($URL_RB    "$FROM" \
++                        USER \
++                        HOST \
++                        SOURCE \
++                        PARAM_DS \
++                        PARAM_VITASTOR_CONF)
++
++    USER="${URL_ELEMENTS[j++]}"
++    DST_HOST="${URL_ELEMENTS[j++]}"
++    SOURCE="${URL_ELEMENTS[j++]}"
++    DS="${URL_ELEMENTS[j++]}"
++    VITASTOR_CONF="${URL_ELEMENTS[j++]}"
++
++    # Remove leading '/'
++    SOURCE="${SOURCE#/}"
++
++    if [ -n "$USER" ]; then
++        DST_HOST="$USER@$DST_HOST"
++    fi
++
++    local CLI
++    CLI="vitastor-cli"
++    if [ -n "$VITASTOR_CONF" ]; then
++        CLI="$CLI --config_path '$(esc_sq "${VITASTOR_CONF}")'"
++    fi
++
++    echo "ssh '$(esc_sq "$DST_HOST")' \"$CLI dd iimg='$(esc_sq "$SOURCE")'\""
++}
++
+ # Compare 2 version strings using sort -V
+ # Usage:
+ #   verlte "3.2.9" "3.4.0"
+@@ -424,6 +463,9 @@ s3://*)
+ rbd://*)
+     command="$(get_rbd_cmd "$FROM")"
+     ;;
++vitastor://*)
++    command="$(get_vitastor_cmd "$FROM")"
++    ;;
+ vcenter://*)
+     command="$VAR_LOCATION/remotes/datastore/vcenter_downloader.rb '$(esc_sq "$FROM")'"
+     ;;

opennebula/remotes/datastore/vitastor/export

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to export an image to qcow2 file
+
+# ------------ Set up the environment to source common tools ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get rm and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/FORMAT \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+SRC="${XPATH_ELEMENTS[i++]}"
+SIZE="${XPATH_ELEMENTS[i++]}"
+MD5="${XPATH_ELEMENTS[i++]}"
+SHA1="${XPATH_ELEMENTS[i++]}"
+FORMAT="${XPATH_ELEMENTS[i++]:-raw}"
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+IMPORT_SOURCE="vitastor://$DST_HOST/$SRC"
+IS_JOIN="?"
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path $VITASTOR_CONF"
+    IMPORT_SOURCE="${IMPORT_SOURCE}${IS_JOIN}VITASTOR_CONF=${VITASTOR_CONF}"
+fi
+
+# FIXME: this is inefficient - it pipes the image twice...
+
+INFO_SCRIPT=$(cat <<EOF
+if [ -z "$MD5" ]; then
+    CHECKSUM=\$(
+        $CLI dd iimg=${SRC} | ${MD5SUM} | cut -f1 -d' '
+        ps=\$PIPESTATUS
+
+        if [ "\$ps" != "0" ]; then
+            exit \$ps
+        fi
+    )
+
+    status=\$?
+
+    [ "\$status" != "0" ] && exit \$status
+else
+    CHECKSUM="$MD5"
+fi
+
+if [ -z "\$CHECKSUM" ]; then
+    exit 1
+fi
+
+cat <<EOT
+<MD5><![CDATA[\$CHECKSUM]]></MD5>
+<SIZE><![CDATA[$SIZE]]></SIZE>
+<FORMAT><![CDATA[${FORMAT}]]></FORMAT>
+EOT
+EOF
+)
+
+INFO=$(ssh_monitor_and_log "$DST_HOST" "$INFO_SCRIPT" "Image info script" 2>&1)
+INFO_STATUS=$?
+
+if [ "$INFO_STATUS" != "0" ]; then
+    echo "$INFO"
+    exit $INFO_STATUS
+fi
+
+cat <<EOF
+<IMPORT_INFO>
+    <IMPORT_SOURCE><![CDATA[$IMPORT_SOURCE]]></IMPORT_SOURCE>
+    $INFO
+    <DISPOSE>NO</DISPOSE>
+</IMPORT_INFO>"
+EOF

opennebula/remotes/datastore/vitastor/mkfs

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to create a VM image (SRC) of size (SIZE) and formatted as (FS)
+
+# -------- Set up the environment to source common tools & conf ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+source ${DRIVER_PATH}/../libfs.sh
+source ${DRIVER_PATH}/../../etc/datastore/datastore.conf
+
+# -------- Get mkfs and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
+    /DS_DRIVER_ACTION_DATA/IMAGE/FORMAT \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/FS \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+BASE_PATH="${XPATH_ELEMENTS[i++]}"
+RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}"
+SAFE_DIRS="${XPATH_ELEMENTS[i++]}"
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+POOL_NAME="${XPATH_ELEMENTS[i++]}"
+IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
+FORMAT="${XPATH_ELEMENTS[i++]}"
+SIZE="${XPATH_ELEMENTS[i++]}"
+FS="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+if [ -n "$POOL_NAME" ]; then
+    CLI="$CLI --pool ${POOL_NAME}"
+fi
+
+set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS"
+
+IMAGE_NAME="${IMAGE_PREFIX}-${ID}"
+
+# ------------ Image to save_as disk, no need to create a new image ------------
+
+if [ "$FORMAT" = "save_as" ]; then
+    echo "$IMAGE_NAME"
+    exit 0
+fi
+
+# ------------ Create the image in the repository ------------
+
+# FIXME: Duplicate code with tm/vitastor/mkimage
+
+MKIMAGE_CMD=$(cat <<EOF
+    set -e -o pipefail
+    export PATH=/usr/sbin:/sbin:\$PATH
+    vitastor-cli $CLI create --pool "${POOL_NAME}" "$IMAGE_NAME" --size "${SIZE}M"
+EOF
+)
+
+if [ -n "$FS" -o "$FORMAT" = "swap" ]; then
+    MKFS_CMD=`mkfs_command '$NBD' raw "$SIZE" "$SUPPORTED_FS" "$FS" "$FS_OPTS" | grep -v $QEMU_IMG`
+fi
+
+MKIMAGE_CMD=$(cat <<EOF
+    set -e -o pipefail
+    export PATH=/usr/sbin:/sbin:\$PATH
+    vitastor-cli $CLI create --pool "${POOL_NAME}" "$IMAGE_NAME" --size "${SIZE}M"
+EOF
+)
+if [ ! -z $FS ]; then
+    set -e -o pipefail
+
+    IMAGE_HASH=`generate_image_hash`
+    FS_OPTS=$(eval $(echo "echo \$FS_OPTS_$FS"))
+
+    MKFS_CMD=`mkfs_command '$NBD' raw "$SIZE" "$SUPPORTED_FS" "$FS" "$FS_OPTS" | grep -v $QEMU_IMG`
+    MKIMAGE_CMD=$(cat <<EOF
+    $MKIMAGE_CMD
+    NBD=\$(sudo vitastor-nbd $CLI map --image "$IMAGE_NAME")
+    trap "sudo vitastor-nbd $CLI unmap \$NBD" EXIT
+    $MKFS_CMD
+EOF
+)
+fi
+
+ssh_exec_and_log "$DST_HOST" "$MKIMAGE_CMD" "Error registering $IMAGE_NAME in $DST_HOST"
+
+echo "$IMAGE_NAME"

opennebula/remotes/datastore/vitastor/monitor

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to monitor the free and used space of a datastore
+
+# -------- Set up the environment to source common tools & conf ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+source ${DRIVER_PATH}/../../datastore/libfs.sh
+
+# -------- Get datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb -b $DRV_ACTION"
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+BRIDGE_LIST="${XPATH_ELEMENTS[j++]}"
+POOL_NAME="${XPATH_ELEMENTS[j++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+
+HOST=`get_destination_host`
+
+if [ -z "$HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+
+# ------------ Compute datastore usage -------------
+
+MONITOR_SCRIPT=$(cat <<EOF
+vitastor-cli df --json | jq -r '.[] | select(.name == "${POOL_NAME}") |
+    "TOTAL_MB="+(.total_raw/.raw_to_usable/1024/1024 | tostring)+
+    "\nUSED_MB="+(.used_raw/.raw_to_usable/1024/1024 | tostring)+
+    "\nFREE_MB="+(.max_available/1024/1024 | tostring)'
+EOF
+)
+
+ssh_monitor_and_log $HOST "$MONITOR_SCRIPT 2>&1" "Error monitoring ${POOL_NAME} in $HOST"

opennebula/remotes/datastore/vitastor/oned.conf.diff

@@ -0,0 +1,73 @@
+diff --git /etc/one/oned.conf /etc/one/oned.conf
+index be02d646a8..27f876ec36 100644
+--- /etc/one/oned.conf
++++ /etc/one/oned.conf
+@@ -481,7 +481,7 @@ VM_MAD = [
+     NAME           = "kvm",
+     SUNSTONE_NAME  = "KVM",
+     EXECUTABLE     = "one_vmm_exec",
+-    ARGUMENTS      = "-t 15 -r 0 kvm -p",
++    ARGUMENTS      = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
+     DEFAULT        = "vmm_exec/vmm_exec_kvm.conf",
+     TYPE           = "kvm",
+     KEEP_SNAPSHOTS = "yes",
+@@ -592,7 +592,7 @@ VM_MAD = [
+ 
+ TM_MAD = [
+     EXECUTABLE = "one_tm",
+-    ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
++    ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
+ ]
+ 
+ #*******************************************************************************
+@@ -612,7 +612,7 @@ TM_MAD = [
+ 
+ DATASTORE_MAD = [
+     EXECUTABLE = "one_datastore",
+-    ARGUMENTS  = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
++    ARGUMENTS  = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
+ ]
+ 
+ #*******************************************************************************
+@@ -1050,6 +1050,9 @@ INHERIT_DATASTORE_ATTR  = "VCENTER_DS_IMAGE_DIR"
+ INHERIT_DATASTORE_ATTR  = "VCENTER_DS_VOLATILE_DIR"
+ INHERIT_DATASTORE_ATTR  = "VCENTER_INSTANCE_ID"
+ 
++INHERIT_DATASTORE_ATTR  = "VITASTOR_CONF"
++INHERIT_DATASTORE_ATTR  = "IMAGE_PREFIX"
++
+ INHERIT_IMAGE_ATTR      = "DISK_TYPE"
+ INHERIT_IMAGE_ATTR      = "VCENTER_ADAPTER_TYPE"
+ INHERIT_IMAGE_ATTR      = "VCENTER_DISK_TYPE"
+@@ -1180,6 +1183,14 @@ TM_MAD_CONF = [
+     CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "RBD"
+ ]
+ 
++TM_MAD_CONF = [
++    NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
++    DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
++    TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
++    DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
++    CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
++]
++
+ TM_MAD_CONF = [
+     NAME = "iscsi_libvirt", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
+     DS_MIGRATE = "NO", DRIVER = "raw"
+@@ -1219,9 +1230,16 @@ DS_MAD_CONF = [
+     NAME = "ceph",
+     REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
+     PERSISTENT_ONLY = "NO",
+     MARKETPLACE_ACTIONS = "export"
+ ]
+ 
++DS_MAD_CONF = [
++    NAME = "vitastor",
++    REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
++    PERSISTENT_ONLY = "NO",
++    MARKETPLACE_ACTIONS = "export"
++]
++
+ DS_MAD_CONF = [
+     NAME = "dev", REQUIRED_ATTRS = "DISK_TYPE", PERSISTENT_ONLY = "YES"
+ ]

opennebula/remotes/datastore/vitastor/rm

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to remove a VM image from the image repository
+
+# ------------ Set up the environment to source common tools ------------
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get rm and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+IMAGE_NAME="${XPATH_ELEMENTS[j++]}"
+BRIDGE_LIST="${XPATH_ELEMENTS[j++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+
+# -------- Remove Image from Datastore ------------
+
+log "Removing $IMAGE_NAME from the image repository in $DST_HOST"
+
+DELETE_CMD=$(cat <<EOF
+    $CLI rm $IMAGE_NAME
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$DELETE_CMD" "Error deleting $IMAGE_NAME in $DST_HOST"

opennebula/remotes/datastore/vitastor/snap_delete

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to delete a snapshot of an image
+
+# -------- Set up the environment to source common tools & conf ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get image and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TARGET_SNAPSHOT \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+IMAGE_NAME="${XPATH_ELEMENTS[i++]}"
+SNAP_ID="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+
+SNAP_DELETE_CMD=$(cat <<EOF
+    $CLI rm ${IMAGE_NAME}@${SNAP_ID}
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$SNAP_DELETE_CMD" "Error deleting snapshot $IMAGE_NAME-$SNAP_ID@$SNAP_ID"

opennebula/remotes/datastore/vitastor/snap_flatten

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to flatten a snapshot of a persistent image
+
+# -------- Set up the environment to source common tools & conf ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get image and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TARGET_SNAPSHOT \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+POOL_NAME="${XPATH_ELEMENTS[i++]}"
+IMAGE_NAME="${XPATH_ELEMENTS[i++]}"
+SNAP_ID="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+
+SNAP_FLATTEN_CMD=$(cat <<EOF
+    set -e
+    $CLI flatten "$IMAGE_NAME@$SNAP_ID"
+    $CLI modify "$IMAGE_NAME@$SNAP_ID" --rename "$IMAGE_NAME"
+    $CLI rm --matching "$IMAGE_NAME@*"
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$SNAP_FLATTEN_CMD" "Error flattening snapshot $SNAP_ID for $IMAGE_NAME"

opennebula/remotes/datastore/vitastor/snap_revert

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# This script is used to revert a snapshot of an image
+
+# -------- Set up the environment to source common tools & conf ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+
+source ${DRIVER_PATH}/../libfs.sh
+
+# -------- Get image and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=`cat -`
+ID=$1
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+    /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
+    /DS_DRIVER_ACTION_DATA/IMAGE/TARGET_SNAPSHOT \
+    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
+
+unset i
+
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+POOL_NAME="${XPATH_ELEMENTS[i++]}"
+IMAGE_NAME="${XPATH_ELEMENTS[i++]}"
+SNAP_ID="${XPATH_ELEMENTS[i++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+if [ -n "$POOL_NAME" ]; then
+    CLI="$CLI --pool ${POOL_NAME}"
+fi
+
+SNAP_REVERT_CMD=$(cat <<EOF
+    $CLI rm ${IMAGE_NAME}.flatten || true
+    $CLI create --pool "${POOL_NAME}" --parent ${IMAGE_NAME}@${SNAP_ID} ${IMAGE_NAME}.flatten
+    $CLI rm ${IMAGE_NAME} || true
+    $CLI modify ${IMAGE_NAME}.flatten --rename ${IMAGE_NAME}
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$SNAP_REVERT_CMD" "Error reverting snapshot $SNAP_ID for $IMAGE_NAME"

opennebula/remotes/datastore/vitastor/stat

@@ -0,0 +1 @@
+../ceph/stat

opennebula/remotes/datastore/vitastor/vmm_execrc.diff

@@ -0,0 +1,12 @@
+diff --git /etc/one/vmm_exec/vmm_execrc /etc/one/vmm_exec/vmm_execrc
+index e210526e63..cb51d3b5e8 100644
+--- /etc/one/vmm_exec/vmm_execrc
++++ /etc/one/vmm_exec/vmm_execrc
+@@ -1,6 +1,6 @@
+ # Space separated list of VMM-TM pairs that support live disk snapshots. VMM
+ # and TM must be separated by '-'
+-LIVE_DISK_SNAPSHOTS="kvm-qcow2 kvm-shared kvm-ceph kvm-ssh qemu-qcow2 qemu-shared qemu-ceph qemu-ssh"
++LIVE_DISK_SNAPSHOTS="kvm-qcow2 kvm-shared kvm-ceph kvm-vitastor kvm-ssh qemu-qcow2 qemu-shared qemu-ceph qemu-ssh"
+ 
+ # Space separated list VNM_MAD-ACTION pairs that run locally
+ VNMAD_LOCAL_ACTIONS="elastic-post elastic-clean"

opennebula/remotes/tm/vitastor/clone

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# clone fe:SOURCE host:remote_system_ds/disk.i size
+#   - fe is the front-end hostname
+#   - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk
+#   - host is the target host to deploy the VM
+#   - remote_system_ds is the path for the system datastore in the host
+
+SRC=$1
+DST=$2
+VM_ID=$3
+DS_ID=$4
+
+#--------------------------------------------------------------------------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
+    LIB_LOCATION=/usr/lib/one
+else
+    TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+DRIVER_PATH=$(dirname $0)
+
+source $TMCOMMON
+
+#-------------------------------------------------------------------------------
+# Compute the destination image name
+#-------------------------------------------------------------------------------
+
+DST_HOST=`arg_host $DST`
+
+SRC_PATH=`arg_path $SRC`
+DST_PATH=`arg_path $DST`
+
+DST_DIR=`dirname $DST_PATH`
+
+DISK_ID=$(echo $DST|awk -F. '{print $NF}')
+VM_DST="${SRC_PATH}-${VM_ID}-${DISK_ID}"
+DST_DS_ID=`echo $DST | sed s#//*#/#g | awk -F/ '{print $(NF-2)}'`
+
+#-------------------------------------------------------------------------------
+# Get Image information
+#-------------------------------------------------------------------------------
+
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onevm show -x $VM_ID | $XPATH  \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/VITASTOR_CONF \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE)
+
+VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+SIZE="${XPATH_ELEMENTS[j++]}"
+
+#-------------------------------------------------------------------------------
+# Get Datastore information
+#-------------------------------------------------------------------------------
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onedatastore show -x $DST_DS_ID | $XPATH \
+    /DATASTORE/TEMPLATE/POOL_NAME)
+
+POOL_NAME="${XPATH_ELEMENTS[j++]}"
+
+disable_local_monitoring $DST_HOST $DST_DIR
+
+#-------------------------------------------------------------------------------
+# Clone the image
+#-------------------------------------------------------------------------------
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+if [ -n "$POOL_NAME" ]; then
+    CLI="$CLI --pool ${POOL_NAME}"
+fi
+
+CLONE_CMD=$(cat <<EOF
+    $CLI create --parent $SRC_PATH --size ${SIZE}M $VM_DST
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$CLONE_CMD" "Error cloning $SRC_PATH to $VM_DST in $DST_HOST"
+exit 0

opennebula/remotes/tm/vitastor/context

@@ -0,0 +1 @@
+../ceph/context

opennebula/remotes/tm/vitastor/cpds

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# cpds host:remote_system_ds/disk.i fe:SOURCE snapid vmid dsid
+#   - fe is the front-end hostname
+#   - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk
+#   - host is the target host to deploy the VM
+#   - remote_system_ds is the path for the system datastore in the host
+#   - snapid is the snapshot id. "-1" for none
+
+SRC=$1
+DST=$2
+SNAP_ID=$3
+VM_ID=$4
+DS_ID=$5
+
+#--------------------------------------------------------------------------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
+    LIB_LOCATION=/usr/lib/one
+else
+    TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+DRIVER_PATH=$(dirname $0)
+
+source $TMCOMMON
+source ${DRIVER_PATH}/../../datastore/libfs.sh
+source ${DRIVER_PATH}/../../etc/vmm/kvm/kvmrc
+
+#-------------------------------------------------------------------------------
+# Set dst path and dir
+#-------------------------------------------------------------------------------
+
+SRC_HOST=`arg_host $SRC`
+SRC_PATH=`arg_path $SRC`
+
+#-------------------------------------------------------------------------------
+# Get Image information
+#-------------------------------------------------------------------------------
+
+DISK_ID=$(echo "$SRC_PATH" | $AWK -F. '{print $NF}')
+
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onevm show -x $VM_ID | $XPATH \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/VITASTOR_CONF \
+    /VM/LCM_STATE)
+
+SRC_IMAGE="${XPATH_ELEMENTS[j++]}"
+CLONE="${XPATH_ELEMENTS[j++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+LCM_STATE="${XPATH_ELEMENTS[j++]}"
+
+#-------------------------------------------------------------------------------
+# Get Datastore information
+#-------------------------------------------------------------------------------
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onedatastore show -x $DS_ID | $XPATH \
+    /DATASTORE/TEMPLATE/POOL_NAME \
+    /DATASTORE/TEMPLATE/BRIDGE_LIST)
+
+POOL_NAME="${XPATH_ELEMENTS[j++]}"
+BRIDGE_LIST="${XPATH_ELEMENTS[j++]}"
+
+#-------------------------------------------------------------------------------
+# Copy Image back to the datastore
+#-------------------------------------------------------------------------------
+
+if [ "$CLONE" = "YES" ]; then
+    SRC_IMAGE="${SRC_IMAGE}-${VM_ID}-${DISK_ID}"
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+if [ -n "$POOL_NAME" ]; then
+    CLI="$CLI --pool ${POOL_NAME}"
+fi
+
+# Undeployed VM state, do not use front-end, choose host from bridge_list
+if [ "$LCM_STATE" = '67' ] || [ "$LCM_STATE" = '68' ]; then
+    if [ -n "$BRIDGE_LIST" ]; then
+        SRC_HOST=`get_destination_host`
+    fi
+fi
+
+if [ "$SNAP_ID" != "-1" ]; then
+    SRC_IMAGE=$SRC_IMAGE@$SNAP_ID
+fi
+
+COPY_CMD=$(cat <<EOF
+    $CLI dd iimg=$SRC_IMAGE oimg=$DST
+EOF
+)
+
+ssh_exec_and_log "$SRC_HOST" "$COPY_CMD" "Error cloning $SRC_IMAGE to $DST in $SRC_HOST"

opennebula/remotes/tm/vitastor/delete

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# DELETE <host:remote_system_ds/disk.i|host:remote_system_ds/>
+#   - host is the target host to deploy the VM
+#   - remote_system_ds is the path for the system datastore in the host
+
+DST=$1
+VM_ID=$2
+DS_ID=$3
+
+#--------------------------------------------------------------------------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
+    LIB_LOCATION=/usr/lib/one
+else
+    TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+DRIVER_PATH=$(dirname $0)
+
+source $TMCOMMON
+source ${DRIVER_PATH}/../../datastore/libfs.sh
+
+#-------------------------------------------------------------------------------
+# Process destination
+#-------------------------------------------------------------------------------
+
+DST_PATH=`arg_path $DST`
+DST_HOST=`arg_host $DST`
+
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
+
+#-------------------------------------------------------------------------------
+# Delete and exit if directory
+#-------------------------------------------------------------------------------
+
+if [ `is_disk $DST_PATH` -eq 0 ]; then
+    # Directory: delete checkpoint and directory
+
+    unset i j XPATH_ELEMENTS
+
+    while IFS= read -r -d '' element; do
+        XPATH_ELEMENTS[i++]="$element"
+    done < <(onedatastore show -x $DS_ID | $XPATH \
+        /DATASTORE/TEMPLATE/SOURCE \
+        /DATASTORE/TEMPLATE/CLONE \
+        /DATASTORE/TEMPLATE/VITASTOR_CONF \
+        /DATASTORE/TEMPLATE/IMAGE_PREFIX \
+        /DATASTORE/TEMPLATE/POOL_NAME)
+
+    SRC="${XPATH_ELEMENTS[j++]}"
+    CLONE="${XPATH_ELEMENTS[j++]}"
+    VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+    IMAGE_PREFIX="${XPATH_ELEMENTS[j++]:-one}"
+    POOL_NAME="${XPATH_ELEMENTS[j++]}"
+
+    CLI=vitastor-cli
+    if [ -n "$VITASTOR_CONF" ]; then
+        CLI="$CLI --config_path ${VITASTOR_CONF}"
+    fi
+
+    SRC_CHECKPOINT="${IMAGE_PREFIX}-sys-${VM_ID}-checkpoint"
+
+    ssh_exec_and_log "$DST_HOST" "$CLI rm $SRC_CHECKPOINT 2>/dev/null || exit 0" \
+        "Error deleting $SRC_CHECKPOINT in $DST_HOST"
+
+    log "Deleting $DST_PATH"
+    ssh_exec_and_log "$DST_HOST" "rm -rf $DST_PATH" "Error deleting $DST_PATH"
+    exit 0
+fi
+
+#-------------------------------------------------------------------------------
+# Get Image information
+#-------------------------------------------------------------------------------
+
+DISK_ID=$(echo "$DST_PATH" | $AWK -F. '{print $NF}')
+
+# Reads the disk parameters -- taken from image datastore
+unset i j XPATH_ELEMENTS
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onevm show -x $VM_ID | $XPATH \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/VITASTOR_CONF \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/IMAGE_PREFIX \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/POOL_NAME)
+
+SRC="${XPATH_ELEMENTS[j++]}"
+CLONE="${XPATH_ELEMENTS[j++]}"
+VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+IMAGE_PREFIX="${XPATH_ELEMENTS[j++]:-one}"
+POOL_NAME="${XPATH_ELEMENTS[j++]}"
+
+if is_undeployed "$VM_ID" "$DST_HOST"; then
+    # get BRIDGE_LIST from datastore
+    XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
+    IFS= read -r -d '' BRIDGE_LIST < <(onedatastore show -x "$DS_ID" \
+        | $XPATH /DATASTORE/TEMPLATE/BRIDGE_LIST )
+
+    if [ -n "$BRIDGE_LIST" ]; then
+        DST_HOST=$(get_destination_host)
+    fi
+fi
+
+# No need to delete not cloned images
+if [ "$CLONE" = "NO" ]; then
+    exit 0
+fi
+
+CLI=vitastor-cli
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+fi
+
+if [ -n "$SRC" ]; then
+    # cloned, so the name will be "one-<imageid>-<vmid>-<diskid>"
+    SRC_IMAGE="${SRC}-${VM_ID}-${DISK_ID}"
+else
+    # volatile
+    SRC_IMAGE="${IMAGE_PREFIX}-sys-${VM_ID}-${DISK_ID}"
+fi
+
+# Delete the image
+
+log "Deleting $DST_PATH"
+
+DELETE_CMD=$(cat <<EOF
+    $CLI rm $SRC_IMAGE
+EOF
+)
+
+ssh_exec_and_log "$DST_HOST" "$DELETE_CMD" "Error deleting $SRC_IMAGE in $DST_HOST"

opennebula/remotes/tm/vitastor/failmigrate

@@ -0,0 +1 @@
+../ceph/failmigrate

opennebula/remotes/tm/vitastor/ln

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# <CLONE|LN>(.tm_mad_system) tm_mad fe:SOURCE host:remote_system_ds/disk.i vmid dsid
+
+# LN = Attach disk to a VM (Vitastor doesn't need to do anything in this case)
+
+SRC=$1
+DST=$2
+VM_ID=$3
+DS_ID=$4
+
+exit 0

opennebula/remotes/tm/vitastor/mkimage

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# mkimage size format host:remote_system_ds/disk.i vmid dsid
+#   - size in MB of the image
+#   - format for the image
+#   - host is the target host to deploy the VM
+#   - remote_system_ds is the path for the system datastore in the host
+#   - vmid is the id of the VM
+#   - dsid is the target datastore (0 is the system datastore)
+
+SIZE=$1
+FORMAT=$2
+DST=$3
+
+VMID=$4
+DSID=$5
+
+#-------------------------------------------------------------------------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
+    LIB_LOCATION=/usr/lib/one
+else
+    TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+DRIVER_PATH=$(dirname $0)
+
+source $TMCOMMON
+
+source ${DRIVER_PATH}/../../etc/datastore/datastore.conf
+source ${DRIVER_PATH}/../../datastore/libfs.sh
+
+#-------------------------------------------------------------------------------
+# Set dst path and dir
+#-------------------------------------------------------------------------------
+
+DST_PATH=`arg_path $DST`
+DST_HOST=`arg_host $DST`
+DST_DIR=`dirname $DST_PATH`
+
+DISK_ID=$(echo $DST|awk -F. '{print $NF}')
+
+#-------------------------------------------------------------------------------
+# Get Image information
+#-------------------------------------------------------------------------------
+
+XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
+
+unset i j XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <(onevm show -x $VMID | $XPATH \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/VITASTOR_CONF \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/POOL_NAME \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/IMAGE_PREFIX \
+    /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/FS)
+
+VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
+POOL_NAME="${XPATH_ELEMENTS[j++]}"
+IMAGE_PREFIX="${XPATH_ELEMENTS[j++]:-one}"
+FS="${XPATH_ELEMENTS[j++]}"
+
+CLI=
+QEMU_ARG=""
+if [ -n "$VITASTOR_CONF" ]; then
+    CLI="$CLI --config_path ${VITASTOR_CONF}"
+    QEMU_ARG=":config_path=${VITASTOR_CONF}"
+fi
+
+IMAGE_NAME="${IMAGE_PREFIX}-sys-${VMID}-${DISK_ID}"
+
+ssh_make_path $DST_HOST $DST_DIR
+
+set -e -o pipefail
+
+# if user requested a swap or specifies a FS, we need to create a local
+# formatted image and upload into existing Vitastor image
+FS_OPTS=$(eval $(echo "echo \$FS_OPTS_$FS"))
+
+MKIMAGE_CMD=$(cat <<EOF
+    set -e -o pipefail
+    export PATH=/usr/sbin:/sbin:\$PATH
+    vitastor-cli $CLI create --pool "${POOL_NAME}" "$IMAGE_NAME" --size "${SIZE}M"
+EOF
+)
+
+if [ -n "$FS" -o "$FORMAT" = "swap" ]; then
+    MKFS_CMD=`mkfs_command '$NBD' raw "$SIZE" "$SUPPORTED_FS" "$FS" "$FS_OPTS" | grep -v $QEMU_IMG`
+    MKIMAGE_CMD=$(cat <<EOF
+    $MKIMAGE_CMD
+    NBD=\$(sudo vitastor-nbd $CLI map --image "$IMAGE_NAME")
+    trap "sudo vitastor-nbd $CLI unmap \$NBD" EXIT
+    $MKFS_CMD
+EOF
+)
+fi
+
+DELIMAGE_CMD=$(cat <<EOF
+    vitastor-cli $CLI rm "$IMAGE_NAME"
+EOF
+)
+
+log "Making volatile disk of ${SIZE}M at $DST"
+
+ssh_exec_and_log_no_error "$DST_HOST" "$MKIMAGE_CMD" "Error creating volatile disk.$DISK_ID ($IMAGE_NAME) in $DST_HOST in pool $POOL_NAME."
+
+rc=$?
+
+if [ $rc != 0 ]; then
+    ssh_exec_and_log_no_error "$DST_HOST" "$DELIMAGE_CMD" "Error removing image"
+fi
+
+exit $rc

opennebula/remotes/tm/vitastor/mkswap

@@ -0,0 +1 @@
+../ceph/mkswap

opennebula/remotes/tm/vitastor/monitor

@@ -0,0 +1 @@
+../../datastore/vitastor/monitor

opennebula/remotes/tm/vitastor/mv

@@ -0,0 +1 @@
+../ceph/mv

opennebula/remotes/tm/vitastor/mvds

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Vitastor OpenNebula driver
+# Copyright (c) Vitaliy Filippov, 2024+
+# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+# mvds host:remote_system_ds/disk.i fe:SOURCE vmid dsid
+#   - fe is the front-end hostname
+#   - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk
+#   - host is the target host to deploy the VM
+#   - remote_system_ds is the path for the system datastore in the host
+#   - vmid is the id of the VM
+#   - dsid is the target datastore (0 is the system datastore)
+
+exit 0

opennebula/remotes/tm/vitastor/postbackup

@@ -0,0 +1 @@
+postbackup_live

opennebula/remotes/tm/vitastor/postbackup_live

@@ -0,0 +1 @@
+../ceph/postbackup_live

opennebula/remotes/tm/vitastor/postmigrate

@@ -0,0 +1 @@
+../ceph/postmigrate