Compare commits

...

22 Commits

Author SHA1 Message Date
Vitaliy Filippov fe8e611e23 Release 1.9.0
Test / test_dd (push) Successful in 9s Details
Test / test_rebalance_verify_ec (push) Successful in 1m33s Details
Test / test_root_node (push) Successful in 9s Details
Test / test_switch_primary (push) Successful in 32s Details
Test / test_write (push) Successful in 31s Details
Test / test_write_no_same (push) Successful in 8s Details
Test / test_write_xor (push) Successful in 33s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 1m33s Details
Test / test_heal_pg_size_2 (push) Successful in 2m16s Details
Test / test_heal_ec (push) Successful in 2m16s Details
Test / test_heal_antietcd (push) Successful in 2m17s Details
Test / test_heal_csum_32k_dmj (push) Successful in 2m17s Details
Test / test_heal_csum_32k_dj (push) Successful in 2m18s Details
Test / test_heal_csum_32k (push) Successful in 2m17s Details
Test / test_osd_tags (push) Successful in 7s Details
Test / test_enospc (push) Successful in 11s Details
Test / test_enospc_xor (push) Successful in 14s Details
Test / test_enospc_imm (push) Successful in 10s Details
Test / test_enospc_imm_xor (push) Successful in 12s Details
Test / test_scrub (push) Successful in 13s Details
Test / test_scrub_zero_osd_2 (push) Successful in 12s Details
Test / test_scrub_xor (push) Successful in 13s Details
Test / test_heal_csum_4k_dmj (push) Successful in 2m17s Details
Test / test_scrub_pg_size_3 (push) Successful in 16s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 17s Details
Test / test_heal_csum_4k_dj (push) Successful in 2m18s Details
Test / test_nfs (push) Successful in 11s Details
Test / test_scrub_ec (push) Successful in 15s Details
Test / test_heal_csum_4k (push) Successful in 2m19s Details
Test / test_etcd_fail (push) Successful in 42s Details
- OpenNebula support! [Installation instructions](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/installation/opennebula.en.md)
- Added [vitastor-cli rm --exact|--matching](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/usage/cli.en.md#rm) command
- Added [vitastor-cli dd](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/usage/cli.en.md#dd) command - copy files between Vitastor images, files and pipes
- Add a startup timeout to vitastor-cli to not wait for etcd infinitely
- Fix non-working OSD_OP_READ_CHAIN_BITMAP O_o
- Autodetect block_size/bitmap_granularity/immediate_commit when creating pools
- Do not allow to create multiple pools with the same name from vitastor-cli
- Fix skip_cache_check option not applied due to type issue (see github issue #70)
2024-09-06 01:46:16 +03:00
Vitaliy Filippov 7636f9c726 Turn off brp-python-bytecompile in RPM specs 2024-09-06 01:44:44 +03:00
Vitaliy Filippov d5f7005ddd Add dd and rm --exact|--matching documentation 2024-09-05 02:22:05 +03:00
Vitaliy Filippov 70d6fcd32a Add OpenNebula to README 2024-09-05 02:00:14 +03:00
Vitaliy Filippov 17caaa59af vitastor-opennebula is probably more correct than opennebula-vitastor
Test / test_rebalance_verify_imm (push) Successful in 1m29s Details
Test / test_root_node (push) Successful in 8s Details
Test / test_rebalance_verify_ec (push) Successful in 1m35s Details
Test / test_dd (push) Successful in 12s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 1m37s Details
Test / test_write_no_same (push) Successful in 9s Details
Test / test_write (push) Successful in 31s Details
Test / test_switch_primary (push) Successful in 34s Details
Test / test_write_xor (push) Successful in 35s Details
Test / test_heal_pg_size_2 (push) Successful in 2m17s Details
Test / test_heal_ec (push) Successful in 2m17s Details
Test / test_heal_antietcd (push) Successful in 2m16s Details
Test / test_heal_csum_32k_dmj (push) Successful in 2m21s Details
Test / test_heal_csum_32k_dj (push) Successful in 2m25s Details
Test / test_heal_csum_32k (push) Successful in 2m17s Details
Test / test_heal_csum_4k_dmj (push) Successful in 2m18s Details
Test / test_osd_tags (push) Successful in 9s Details
Test / test_heal_csum_4k_dj (push) Successful in 2m19s Details
Test / test_enospc (push) Successful in 11s Details
Test / test_enospc_xor (push) Successful in 12s Details
Test / test_enospc_imm (push) Successful in 11s Details
Test / test_enospc_imm_xor (push) Successful in 13s Details
Test / test_scrub (push) Successful in 13s Details
Test / test_scrub_zero_osd_2 (push) Successful in 12s Details
Test / test_scrub_xor (push) Successful in 14s Details
Test / test_scrub_pg_size_3 (push) Successful in 14s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 14s Details
Test / test_scrub_ec (push) Successful in 15s Details
Test / test_nfs (push) Successful in 11s Details
Test / test_heal_csum_4k (push) Successful in 2m19s Details
2024-09-05 01:44:16 +03:00
Vitaliy Filippov 2dac6ee38b Fix OpenNebula reinstall
Test / test_rebalance_verify_imm (push) Successful in 1m27s Details
Test / test_dd (push) Successful in 11s Details
Test / test_root_node (push) Successful in 7s Details
Test / test_rebalance_verify_ec (push) Successful in 1m35s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 1m36s Details
Test / test_write_no_same (push) Successful in 9s Details
Test / test_write (push) Successful in 30s Details
Test / test_switch_primary (push) Successful in 32s Details
Test / test_write_xor (push) Successful in 33s Details
Test / test_heal_pg_size_2 (push) Successful in 2m16s Details
Test / test_heal_antietcd (push) Successful in 2m17s Details
Test / test_heal_csum_32k_dmj (push) Successful in 2m19s Details
Test / test_heal_csum_32k_dj (push) Successful in 2m25s Details
Test / test_heal_csum_32k (push) Successful in 2m17s Details
Test / test_heal_csum_4k_dmj (push) Successful in 2m19s Details
Test / test_osd_tags (push) Successful in 7s Details
Test / test_heal_csum_4k_dj (push) Successful in 2m19s Details
Test / test_enospc (push) Successful in 10s Details
Test / test_enospc_xor (push) Successful in 12s Details
Test / test_enospc_imm (push) Successful in 10s Details
Test / test_enospc_imm_xor (push) Successful in 13s Details
Test / test_scrub (push) Successful in 13s Details
Test / test_scrub_zero_osd_2 (push) Successful in 12s Details
Test / test_scrub_xor (push) Successful in 14s Details
Test / test_scrub_pg_size_3 (push) Successful in 15s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 16s Details
Test / test_scrub_ec (push) Successful in 16s Details
Test / test_nfs (push) Successful in 11s Details
Test / test_heal_csum_4k (push) Successful in 2m17s Details
Test / test_heal_ec (push) Successful in 2m21s Details
2024-09-04 11:05:56 +03:00
Vitaliy Filippov 8be67a2d5b Fix OpenNebula save/restore 2024-09-04 11:05:56 +03:00
Vitaliy Filippov 9c2132882c Fix unaligned last block read/write in cli_dd 2024-09-04 11:05:56 +03:00
Vitaliy Filippov 9f25bb059b Use just IMAGE_PREFIX, not IMAGE_PREFIX+"one" 2024-09-04 01:23:00 +03:00
Vitaliy Filippov ee3094c5e5 Add OpenNebula plugin docs 2024-09-04 01:22:39 +03:00
Vitaliy Filippov ba9f263b75 Add wildcard removal command
Test / test_rebalance_verify_imm (push) Successful in 1m27s Details
Test / test_root_node (push) Successful in 8s Details
Test / test_dd (push) Successful in 13s Details
Test / test_rebalance_verify_ec (push) Successful in 1m35s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 1m36s Details
Test / test_write_no_same (push) Successful in 8s Details
Test / test_write (push) Successful in 31s Details
Test / test_switch_primary (push) Successful in 33s Details
Test / test_write_xor (push) Successful in 34s Details
Test / test_heal_pg_size_2 (push) Successful in 2m15s Details
Test / test_heal_ec (push) Successful in 2m18s Details
Test / test_heal_antietcd (push) Successful in 2m17s Details
Test / test_heal_csum_32k_dmj (push) Successful in 2m18s Details
Test / test_heal_csum_32k_dj (push) Successful in 2m21s Details
Test / test_heal_csum_32k (push) Successful in 2m19s Details
Test / test_heal_csum_4k_dmj (push) Successful in 2m20s Details
Test / test_heal_csum_4k_dj (push) Successful in 2m20s Details
Test / test_osd_tags (push) Successful in 9s Details
Test / test_enospc (push) Successful in 12s Details
Test / test_enospc_xor (push) Successful in 13s Details
Test / test_enospc_imm (push) Successful in 10s Details
Test / test_enospc_imm_xor (push) Successful in 14s Details
Test / test_scrub (push) Successful in 14s Details
Test / test_scrub_zero_osd_2 (push) Successful in 13s Details
Test / test_scrub_xor (push) Successful in 13s Details
Test / test_scrub_pg_size_3 (push) Successful in 16s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 16s Details
Test / test_scrub_ec (push) Successful in 15s Details
Test / test_nfs (push) Successful in 11s Details
Test / test_heal_csum_4k (push) Successful in 2m9s Details
2024-08-31 14:13:09 +03:00
Vitaliy Filippov 30eaa1a8e6 Add vitastor-cli ls --exact 2024-08-31 02:36:25 +03:00
Vitaliy Filippov 6a8daedbe2 rm --wildcard 2024-08-31 02:36:25 +03:00
Vitaliy Filippov 2b96ac0b44 Implement OpenNebula driver 2024-08-30 23:46:37 +03:00
Vitaliy Filippov 986cd11705 Implement CLI "dd" command - copy data between Vitastor images, files and pipes 2024-08-30 02:31:06 +03:00
Vitaliy Filippov b804051eaf Remove debug print in nbd-proxy 2024-08-30 02:31:06 +03:00
Vitaliy Filippov 3cc326500e Fix non-working OSD_OP_READ_CHAIN_BITMAP O_o 2024-08-30 01:25:05 +03:00
Vitaliy Filippov f848c450a4 Clients should not wait infinitely for etcd to start if it's unavailable 2024-08-28 02:03:35 +03:00
Vitaliy Filippov 4121c66281 Autodetect block_size/bitmap_granularity/immediate_commit when creating pools 2024-08-28 02:03:35 +03:00
Vitaliy Filippov b3716fbe23 Validate pool name when creating a pool 2024-08-28 02:03:35 +03:00
Vitaliy Filippov 97f49d7d94 Fix #70 from github - skip_cache_check type issue
Test / test_rebalance_verify (push) Successful in 1m21s Details
Test / test_rebalance_verify_imm (push) Successful in 1m22s Details
Test / test_root_node (push) Successful in 9s Details
Test / test_rebalance_verify_ec (push) Successful in 1m30s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 1m30s Details
Test / test_write_no_same (push) Successful in 9s Details
Test / test_switch_primary (push) Successful in 31s Details
Test / test_write (push) Successful in 30s Details
Test / test_write_xor (push) Successful in 37s Details
Test / test_heal_pg_size_2 (push) Successful in 2m18s Details
Test / test_heal_ec (push) Successful in 2m19s Details
Test / test_heal_antietcd (push) Successful in 2m18s Details
Test / test_heal_csum_32k_dmj (push) Successful in 2m20s Details
Test / test_heal_csum_4k_dmj (push) Successful in 2m19s Details
Test / test_heal_csum_32k_dj (push) Successful in 2m27s Details
Test / test_heal_csum_32k (push) Successful in 2m26s Details
Test / test_osd_tags (push) Successful in 9s Details
Test / test_heal_csum_4k_dj (push) Successful in 2m16s Details
Test / test_enospc (push) Successful in 9s Details
Test / test_enospc_xor (push) Successful in 13s Details
Test / test_enospc_imm (push) Successful in 12s Details
Test / test_enospc_imm_xor (push) Successful in 12s Details
Test / test_scrub (push) Successful in 15s Details
Test / test_scrub_zero_osd_2 (push) Successful in 14s Details
Test / test_scrub_xor (push) Successful in 14s Details
Test / test_scrub_pg_size_3 (push) Successful in 14s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 14s Details
Test / test_scrub_ec (push) Successful in 14s Details
Test / test_nfs (push) Successful in 12s Details
Test / test_heal_csum_4k (push) Successful in 2m17s Details
2024-08-14 01:35:43 +03:00
Vitaliy Filippov 131de4b790 Disable trace in header 2024-08-13 11:21:35 +03:00
96 changed files with 4913 additions and 90 deletions

View File

@ -558,6 +558,24 @@ jobs:
echo ""
done
test_dd:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_dd.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_root_node:
runs-on: ubuntu-latest
needs: build

View File

@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
project(vitastor)
set(VITASTOR_VERSION "1.8.0")
set(VITASTOR_VERSION "1.9.0")
add_subdirectory(src)

View File

@ -19,7 +19,7 @@ Vitastor нацелен в первую очередь на SSD и SSD+HDD кл
TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, Proxmox, Kubernetes.
Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, OpenNebula, Proxmox, Kubernetes.
Другие драйверы могут также быть легко реализованы.
Подробности смотрите в документации по ссылкам ниже.
@ -42,6 +42,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
- Установка
- [Пакеты](docs/installation/packages.ru.md)
- [Proxmox](docs/installation/proxmox.ru.md)
- [OpenNebula](docs/installation/opennebula.ru.md)
- [OpenStack](docs/installation/openstack.ru.md)
- [Kubernetes CSI](docs/installation/kubernetes.ru.md)
- [Сборка из исходных кодов](docs/installation/source.ru.md)

View File

@ -19,7 +19,7 @@ supports TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1
with proper hardware which is ~10 times faster than other popular SDS's like Ceph
or internal systems of public clouds.
Vitastor supports QEMU, NBD, NFS protocols, OpenStack, Proxmox, Kubernetes drivers.
Vitastor supports QEMU, NBD, NFS protocols, OpenStack, OpenNebula, Proxmox, Kubernetes drivers.
More drivers may be created easily.
Read more details below in the documentation.
@ -42,6 +42,7 @@ Read more details below in the documentation.
- Installation
- [Packages](docs/installation/packages.en.md)
- [Proxmox](docs/installation/proxmox.en.md)
- [OpenNebula](docs/installation/opennebula.en.md)
- [OpenStack](docs/installation/openstack.en.md)
- [Kubernetes CSI](docs/installation/kubernetes.en.md)
- [Building from Source](docs/installation/source.en.md)

View File

@ -1,4 +1,4 @@
VITASTOR_VERSION ?= v1.8.0
VITASTOR_VERSION ?= v1.9.0
all: build push

View File

@ -49,7 +49,7 @@ spec:
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: vitalif/vitastor-csi:v1.8.0
image: vitalif/vitastor-csi:v1.9.0
args:
- "--node=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"

View File

@ -121,7 +121,7 @@ spec:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: vitalif/vitastor-csi:v1.8.0
image: vitalif/vitastor-csi:v1.9.0
args:
- "--node=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"

View File

@ -5,7 +5,7 @@ package vitastor
const (
vitastorCSIDriverName = "csi.vitastor.io"
vitastorCSIDriverVersion = "1.8.0"
vitastorCSIDriverVersion = "1.9.0"
)
// Config struct fills the parameters of request or user input

2
debian/changelog vendored
View File

@ -1,4 +1,4 @@
vitastor (1.8.0-1) unstable; urgency=medium
vitastor (1.9.0-1) unstable; urgency=medium
* Bugfixes

6
debian/control vendored
View File

@ -53,3 +53,9 @@ Architecture: amd64
Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version})
Description: Vitastor Proxmox Virtual Environment storage plugin
Vitastor storage plugin for Proxmox Virtual Environment.
Package: vitastor-opennebula
Architecture: amd64
Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client, patch, python3, jq
Description: Vitastor OpenNebula storage plugin
Vitastor storage plugin for OpenNebula.

3
debian/vitastor-opennebula.install vendored Normal file
View File

@ -0,0 +1,3 @@
opennebula/remotes var/lib/one/
opennebula/sudoers.d etc/
opennebula/install.sh var/lib/one/remotes/datastore/vitastor/

7
debian/vitastor-opennebula.postinst vendored Normal file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -e
if [ "$1" = "configure" ]; then
/var/lib/one/remotes/datastore/vitastor/install.sh
fi

4
debian/vitastor-opennebula.triggers vendored Normal file
View File

@ -0,0 +1,4 @@
interest /var/lib/one/remotes/datastore/downloader.sh
interest /etc/one/oned.conf
interest /etc/one/vmm_exec/vmm_execrc
interest /etc/apparmor.d/local/abstractions/libvirt-qemu

View File

@ -0,0 +1,184 @@
[Documentation](../../README.md#documentation) → Installation → OpenNebula
-----
[Читать на русском](opennebula.ru.md)
## Automatic Installation
OpenNebula plugin is packaged as `vitastor-opennebula` Debian and RPM package since Vitastor 1.9.0. So:
- Run `apt-get install vitastor-opennebula` or `yum install vitastor-opennebula` after installing OpenNebula on all nodes
- Check that it prints "OK, Vitastor OpenNebula patches successfully applied" or "OK, Vitastor OpenNebula patches are already applied"
- If it does not, refer to [Manual Installation](#manual-installation) and apply configuration file changes manually
- Make sure that Vitastor patched versions of QEMU and libvirt are installed
(`dpkg -l qemu-system-x86`, `dpkg -l | grep libvirt`, `rpm -qa | grep qemu`, `rpm -qa | grep qemu`, `rpm -qa | grep libvirt-libs` should show "vitastor" in version names)
- [Block VM access to Vitastor cluster](#block-vm-access-to-vitastor-cluster)
## Manual Installation
Install OpenNebula. Then, on each node:
- Copy [opennebula/remotes](../../opennebula/remotes) into `/var/lib/one` recursively: `cp -r opennebula/remotes /var/lib/one/`
- Copy [opennebula/sudoers.d](../../opennebula/sudoers.d) to `/etc`: `cp -r opennebula/sudoers.d /etc/`
- Apply [downloader-vitastor.sh.diff](../../opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff) to `/var/lib/one/remotes/datastore/downloader.sh`:
`patch /var/lib/one/remotes/datastore/downloader.sh < opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff` - or read the patch and apply the same change manually
- Add `kvm-vitastor` to `LIVE_DISK_SNAPSHOTS` in `/etc/one/vmm_exec/vmm_execrc`
- If on Debian or Ubuntu (and AppArmor is used), add Vitastor config file path(s) to `/etc/apparmor.d/local/abstractions/libvirt-qemu`: for example,
`echo ' "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu`
- Apply changes to `/etc/one/oned.conf`
### oned.conf changes
1. Add deploy script override in kvm VM_MAD: add `-l deploy.vitastor` to ARGUMENTS.
```diff
VM_MAD = [
NAME = "kvm",
SUNSTONE_NAME = "KVM",
EXECUTABLE = "one_vmm_exec",
- ARGUMENTS = "-t 15 -r 0 kvm -p",
+ ARGUMENTS = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
DEFAULT = "vmm_exec/vmm_exec_kvm.conf",
TYPE = "kvm",
KEEP_SNAPSHOTS = "yes",
LIVE_RESIZE = "yes",
SUPPORT_SHAREABLE = "yes",
IMPORTED_VMS_ACTIONS = "terminate, terminate-hard, hold, release, suspend,
resume, delete, reboot, reboot-hard, resched, unresched, disk-attach,
disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete,
resize, updateconf, update"
]
```
Optional: if you also want to save VM RAM checkpoints to Vitastor, use
`-l deploy=deploy.vitastor,save=save.vitastor,restore=restore.vitastor`
instead of just `-l deploy=deploy.vitastor`.
2. Add `vitastor` to TM_MAD.ARGUMENTS and DATASTORE_MAD.ARGUMENTS:
```diff
TM_MAD = [
EXECUTABLE = "one_tm",
- ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
+ ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
]
DATASTORE_MAD = [
EXECUTABLE = "one_datastore",
- ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
+ ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
]
```
3. Add INHERIT_DATASTORE_ATTR for two Vitastor attributes:
```
INHERIT_DATASTORE_ATTR = "VITASTOR_CONF"
INHERIT_DATASTORE_ATTR = "IMAGE_PREFIX"
```
4. Add TM_MAD_CONF and DS_MAD_CONF for Vitastor:
```
TM_MAD_CONF = [
NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
]
DS_MAD_CONF = [
NAME = "vitastor",
REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
PERSISTENT_ONLY = "NO",
MARKETPLACE_ACTIONS = "export"
]
```
## Create Datastores
Example Image and System Datastore definitions:
[opennebula/vitastor-imageds.conf](../../opennebula/vitastor-imageds.conf) and
[opennebula/vitastor-systemds.conf](../../opennebula/vitastor-systemds.conf).
Change parameters to your will:
- POOL_NAME is Vitastor pool name to store images.
- IMAGE_PREFIX is a string prepended to all Vitastor image names.
- BRIDGE_LIST is a list of hosts with access to Vitastor cluster, mostly used for image (not system) datastore operations.
- VITASTOR_CONF is the path to cluster configuration. Note that it should be also added to `/etc/apparmor.d/local/abstractions/libvirt-qemu` if you use AppArmor.
- STAGING_DIR is a temporary directory used when importing external images. Should have free space sufficient for downloading external images.
Then create datastores using `onedatastore create vitastor-imageds.conf` and `onedatastore create vitastor-systemds.conf` (or use UI).
## Block VM access to Vitastor cluster
Vitastor doesn't support any authentication yet, so you MUST block VM guest access to the Vitastor cluster at the network level.
If you use VLAN networking for VMs - make sure you use different VLANs for VMs and hypervisor/storage network and
block access between them using your firewall/switch configuration.
If you use something more stupid like bridged networking, you probably have to use manual firewall/iptables setup
to only allow access to Vitastor from hypervisor IPs.
Also you need to switch network to "Bridged & Security Groups" and enable IP spoofing filters in OpenNebula.
Problem is that OpenNebula's IP spoofing filter doesn't affect local interfaces of the hypervisor i.e. when
it's enabled a VM can't talk to other VMs or to the outer world using a spoofed IP, but it CAN talk to the
hypervisor if it takes an IP from its subnet. To fix that you also need some more iptables.
So the complete "stupid" bridged network filter setup could look like the following
(here `10.0.3.0/24` is the VM subnet and `10.0.2.0/24` is the hypervisor subnet):
```
# Allow incoming traffic from physical device
iptables -A INPUT -m physdev --physdev-in eth0 -j ACCEPT
# Do not allow incoming traffic from VMs, but not from VM subnet
iptables -A INPUT ! -s 10.0.3.0/24 -i onebr0 -j DROP
# Drop traffic from VMs to hypervisor/storage subnet
iptables -I FORWARD 1 -s 10.0.3.0/24 -d 10.0.2.0/24 -j DROP
```
## Testing
The OpenNebula plugin includes quite a bit of bash scripts, so here's their description to get an idea about what they actually do.
| Script | Action | How to Test |
| ----------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------ |
| vmm/kvm/deploy.vitastor | Start a VM | Create and start a VM with Vitastor disk(s): persistent / non-persistent / volatile. |
| vmm/kvm/save.vitastor | Save VM memory checkpoint | Stop a VM using "Stop" command. |
| vmm/kvm/restore.vitastor| Restore VM memory checkpoint | Start a VM back after stopping it. |
| datastore/clone | Copy an image as persistent | Create a VM template and instantiate it as persistent. |
| datastore/cp | Import an external image | Import a VM template with images from Marketplace. |
| datastore/export | Export an image as URL | Probably: export a VM template with images to Marketplace. |
| datastore/mkfs | Create an image with FS | Storage → Images → Create → Type: Datablock, Location: Empty disk image, Filesystem: Not empty. |
| datastore/monitor | Monitor used space in image datastore | Check reported used/free space in image datastore list. |
| datastore/rm | Remove a persistent image | Storage → Images → Select an image → Delete. |
| datastore/snap_delete | Delete a snapshot of a persistent image | Storage → Images → Select an image → Select a snapshot → Delete; <br> To create an image with snapshot: attach a persistent image to a VM; create a snapshot; detach the image. |
| datastore/snap_flatten | Revert an image to snapshot and delete other snapshots | Storage → Images → Select an image → Select a snapshot → Flatten. |
| datastore/snap_revert | Revert an image to snapshot | Storage → Images → Select an image → Select a snapshot → Revert. |
| datastore/stat | Get virtual size of an image in MB | No idea. Seems to be unused both in Vitastor and Ceph datastores. |
| tm/clone | Clone a non-persistent image to a VM disk | Attach a non-persistent image to a VM. |
| tm/context | Generate a contextualisation VM disk | Create a VM with enabled contextualisation (default). Common host FS-based version is used in Vitastor and Ceph datastores. |
| tm/cpds | Copy a VM disk / its snapshot to an image | Select a VM → Select a disk → Optionally select a snapshot → Save as. |
| tm/delete | Delete a cloned or volatile VM disk | Detach a volatile disk or a non-persistent image from a VM. |
| tm/failmigrate | Handle live migration failure | No action. Script is empty in Vitastor and Ceph. In other datastores, should roll back actions done by tm/premigrate. |
| tm/ln | Attach a persistent image to a VM | No action. Script is empty in Vitastor and Ceph. |
| tm/mkimage | Create a volatile disk, maybe with FS | Attach a volatile disk to a VM, with or without file system. |
| tm/mkswap | Create a volatile swap disk | Attach a volatile disk to a VM, formatted as swap. |
| tm/monitor | Monitor used space in system datastore | Check reported used/free space in system datastore list. |
| tm/mv | Move a migrated VM disk between hosts | Migrate a VM between hosts. In Vitastor and Ceph datastores, doesn't do any storage action. |
| tm/mvds | Detach a persistent image from a VM | No action. The opposite of tm/ln. Script is empty in Vitastor and Ceph. In other datastores, script may copy the image from VM host back to the datastore. |
| tm/postbackup | Executed after backup | Seems that the script just removes temporary files after backup. Perform a VM backup and check that temporary files are cleaned up. |
| tm/postbackup_live | Executed after backup of a running VM | Same as tm/postbackup, but for a running VM. |
| tm/postmigrate | Executed after VM live migration | No action. Only executed for system datastore, so the script tries to call other TMs for other disks. Except that, the script does nothing in Vitastor and Ceph datastores. |
| tm/prebackup | Actual backup script: backup VM disks | Set up "rsync" backup datastore → Backup a VM to it. |
| tm/prebackup_live | Backup VM disks of a running VM | Same as tm/prebackup, but also does fsfreeze/thaw. So perform a live backup, restore it and check that disks are consistent. |
| tm/premigrate | Executed before live migration | No action. Only executed for system datastore, so the script tries to call other TMs for other disks. Except that, the script does nothing in Vitastor and Ceph datastores. |
| tm/resize | Resize a VM disk | Select a VM → Select a non-persistent disk → Resize. |
| tm/restore | Restore VM disks from backup | Set up "rsync" backup datastore → Backup a VM to it → Restore it back. |
| tm/snap_create | Create a VM disk snapshot | Select a VM → Select a disk → Create snapshot. |
| tm/snap_create_live | Create a VM disk snapshot for a live VM | Select a running VM → Select a disk → Create snapshot. |
| tm/snap_delete | Delete a VM disk snapshot | Select a VM → Select a disk → Select a snapshot → Delete. |
| tm/snap_revert | Revert a VM disk to a snapshot | Select a VM → Select a disk → Select a snapshot → Revert. |

View File

@ -0,0 +1,187 @@
[Документация](../../README-ru.md#документация) → Установка → OpenNebula
-----
[Read in English](opennebula.en.md)
## Автоматическая установка
Плагин OpenNebula Vitastor распространяется как Debian и RPM пакет `vitastor-opennebula`, начиная с версии Vitastor 1.9.0. Так что:
- Запустите `apt-get install vitastor-opennebula` или `yum install vitastor-opennebula` после установки OpenNebula на всех серверах
- Проверьте, что он выводит "OK, Vitastor OpenNebula patches successfully applied" или "OK, Vitastor OpenNebula patches are already applied" в процессе установки
- Если сообщение не выведено, пройдите по шагам инструкцию [Ручная установка](#ручная-установка) и примените правки файлов конфигурации вручную
- Удостоверьтесь, что установлены версии QEMU и libvirt с изменениями Vitastor
(`dpkg -l qemu-system-x86`, `dpkg -l | grep libvirt`, `rpm -qa | grep qemu`, `rpm -qa | grep qemu`, `rpm -qa | grep libvirt-libs` должны показывать "vitastor" в номере версии)
- [Заблокируйте доступ виртуальных машин в Vitastor](#блокировка-доступа-вм-в-vitastor)
## Ручная установка
Сначала установите саму OpenNebula. После этого, на каждом сервере:
- Скопируйте директорию [opennebula/remotes](../../opennebula/remotes) в `/var/lib/one`: `cp -r opennebula/remotes /var/lib/one/`
- Скопируйте директорию [opennebula/sudoers.d](../../opennebula/sudoers.d) в `/etc`: `cp -r opennebula/sudoers.d /etc/`
- Примените патч [downloader-vitastor.sh.diff](../../opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff) к `/var/lib/one/remotes/datastore/downloader.sh`:
`patch /var/lib/one/remotes/datastore/downloader.sh < opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff` - либо прочитайте патч и примените изменение вручную
- Добавьте `kvm-vitastor` в список `LIVE_DISK_SNAPSHOTS` в файле `/etc/one/vmm_exec/vmm_execrc`
- Если вы используете Debian или Ubuntu (и AppArmor), добавьте пути к файлу(ам) конфигурации Vitastor в файл `/etc/apparmor.d/local/abstractions/libvirt-qemu`: например,
`echo ' "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu`
- Примените изменения `/etc/one/oned.conf`
### Изменения oned.conf
1. Добавьте переопределение скрипта deploy в VM_MAD kvm, добавив `-l deploy.vitastor` в `ARGUMENTS`:
```diff
VM_MAD = [
NAME = "kvm",
SUNSTONE_NAME = "KVM",
EXECUTABLE = "one_vmm_exec",
- ARGUMENTS = "-t 15 -r 0 kvm -p",
+ ARGUMENTS = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
DEFAULT = "vmm_exec/vmm_exec_kvm.conf",
TYPE = "kvm",
KEEP_SNAPSHOTS = "yes",
LIVE_RESIZE = "yes",
SUPPORT_SHAREABLE = "yes",
IMPORTED_VMS_ACTIONS = "terminate, terminate-hard, hold, release, suspend,
resume, delete, reboot, reboot-hard, resched, unresched, disk-attach,
disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete,
resize, updateconf, update"
]
```
Опционально: если вы хотите также сохранять снимки памяти ВМ в Vitastor, добавьте
`-l deploy=deploy.vitastor,save=save.vitastor,restore=restore.vitastor`
вместо просто `-l deploy=deploy.vitastor`.
2. Добавьте `vitastor` в значения TM_MAD.ARGUMENTS и DATASTORE_MAD.ARGUMENTS:
```diff
TM_MAD = [
EXECUTABLE = "one_tm",
- ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
+ ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
]
DATASTORE_MAD = [
EXECUTABLE = "one_datastore",
- ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
+ ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
]
```
3. Добавьте строчки с INHERIT_DATASTORE_ATTR для двух атрибутов Vitastor-хранилищ:
```
INHERIT_DATASTORE_ATTR = "VITASTOR_CONF"
INHERIT_DATASTORE_ATTR = "IMAGE_PREFIX"
```
4. Добавьте TM_MAD_CONF и DS_MAD_CONF для Vitastor:
```
TM_MAD_CONF = [
NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
]
DS_MAD_CONF = [
NAME = "vitastor",
REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
PERSISTENT_ONLY = "NO",
MARKETPLACE_ACTIONS = "export"
]
```
## Создайте хранилища
Примеры настроек хранилищ образов (image) и дисков ВМ (system):
[opennebula/vitastor-imageds.conf](../../opennebula/vitastor-imageds.conf) и
[opennebula/vitastor-systemds.conf](../../opennebula/vitastor-systemds.conf).
Скопируйте настройки и поменяйте следующие параметры так, как вам необходимо:
- POOL_NAME - имя пула Vitastor для сохранения образов дисков.
- IMAGE_PREFIX - строка, добавляемая в начало имён образов дисков.
- BRIDGE_LIST - список серверов с доступом к кластеру Vitastor, используемых для операций с хранилищем образов (image, не system).
- VITASTOR_CONF - путь к конфигурации Vitastor. Имейте в виду, что этот путь также надо добавить в `/etc/apparmor.d/local/abstractions/libvirt-qemu`, если вы используете AppArmor.
- STAGING_DIR - путь к временному каталогу, используемому при импорте внешних образов. Должен иметь достаточно свободного места, чтобы вмещать скачанные образы.
После этого создайте хранилища с помощью команд `onedatastore create vitastor-imageds.conf` и `onedatastore create vitastor-systemds.conf` (либо через UI).
## Блокировка доступа ВМ в Vitastor
Vitastor пока не поддерживает никакую аутентификацию, так что вы ДОЛЖНЫ заблокировать доступ гостевых ВМ
в кластер Vitastor на сетевом уровне.
Если вы используете VLAN-сети для ВМ - удостоверьтесь, что ВМ и гипервизор/сеть хранения помещены в разные
изолированные друг от друга VLAN-ы.
Если вы используете что-то более примитивное, например, мосты (bridge), вам, скорее всего, придётся вручную
настроить iptables / межсетевой экран, чтобы разрешить доступ к Vitastor только с IP гипервизоров.
Также в этом случае нужно будет переключить обычные мосты на "Bridged & Security Groups" и включить фильтр
спуфинга IP в OpenNebula. Правда, реализация этого фильтра пока не полная, и она не блокирует доступ к
локальным интерфейсам гипервизора. То есть, включённый фильтр спуфинга IP запрещает ВМ отправлять трафик
с чужими IP к другим ВМ или во внешний мир, но не запрещает отправлять его напрямую гипервизору. Чтобы
исправить это, тоже нужны дополнительные правила iptables.
Таким образом, более-менее полная блокировка при использовании простой сети на сетевых мостах может
выглядеть так (здесь `10.0.3.0/24` - подсеть ВМ, `10.0.2.0/24` - подсеть гипервизора):
```
# Разрешаем входящий трафик с физического устройства
iptables -A INPUT -m physdev --physdev-in eth0 -j ACCEPT
# Запрещаем трафик со всех ВМ, но с IP не из подсети ВМ
iptables -A INPUT ! -s 10.0.3.0/24 -i onebr0 -j DROP
# Запрещаем трафик от ВМ к сети гипервизора
iptables -I FORWARD 1 -s 10.0.3.0/24 -d 10.0.2.0/24 -j DROP
```
## Тестирование
Плагин OpenNebula по большей части состоит из bash-скриптов, и чтобы было понятнее, что они
вообще делают - ниже приведены описания процедур, которыми можно протестировать каждый из них.
| Скрипт | Описание | Как протестировать |
| ----------------------- | --------------------------------------------- | ------------------------------------------------------------------------------------ |
| vmm/kvm/deploy.vitastor | Запустить виртуальную машину | Создайте и запустите виртуальную машину с дисками Vitastor: постоянным / непостоянным / волатильным (временным). |
| vmm/kvm/save.vitastor | Сохранить снимок памяти ВМ | Остановите виртуальную машину командой "Остановить". |
| vmm/kvm/restore.vitastor| Восстановить снимок памяти ВМ | Запустите ВМ после остановки обратно. |
| datastore/clone | Скопировать образ как "постоянный" | Создайте шаблон ВМ и создайте из него постоянную ВМ. |
| datastore/cp | Импортировать внешний образ | Импортируйте шаблон ВМ с образами дисков из Магазина OpenNebula. |
| datastore/export | Экспортировать образ как URL | Вероятно: экспортируйте шаблон ВМ с образами в Магазин. |
| datastore/mkfs | Создать образ с файловой системой | Хранилище → Образы → Создать → Тип: базовый блок данных, Расположение: пустой образ диска, Файловая система: любая непустая. |
| datastore/monitor | Вывод статистики места в хранилище образов | Проверьте статистику свободного/занятого места в списке хранилищ образов. |
| datastore/rm | Удалить "постоянный" образ | Хранилище → Образы → Выберите образ → Удалить. |
| datastore/snap_delete | Удалить снимок "постоянного" образа | Хранилище → Образы → Выберите образ → Выберите снимок → Удалить; <br> Чтобы создать образ со снимком: подключите постоянный образ к ВМ, создайте снимок, отключите образ. |
| datastore/snap_flatten | Откатить образ к снимку, удалив другие снимки | Хранилище → Образы → Выберите образ → Выберите снимок → "Выровнять" (flatten). |
| datastore/snap_revert | Откатить образ к снимку | Хранилище → Образы → Выберите образ → Выберите снимок → Откатить. |
| datastore/stat | Показать виртуальный размер образа в МБ | Неизвестно. По-видимому, в плагинах Vitastor и Ceph не используется. |
| tm/clone | Клонировать "непостоянный" образ в диск ВМ | Подключите "непостоянный" образ к ВМ. |
| tm/context | Создать диск контекстуализации ВМ | Создайте ВМ с контекстуализацией, как обычно. Но тестировать особенно нечего: в плагинах Vitastor и Ceph образ контекста хранится в локальной ФС гипервизора. |
| tm/cpds | Копировать диск ВМ/его снимок в новый образ | Выберите ВМ → Выберите диск → Опционально выберите снимок → "Сохранить как". |
| tm/delete | Удалить диск-клон или волатильный диск ВМ | Отключите волатильный или не-постоянный диск от ВМ. |
| tm/failmigrate | Обработать неудачную миграцию | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. В других плагинах скрипт должен откатывать действия tm/premigrate. |
| tm/ln | Подключить "постоянный" образ к ВМ | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. |
| tm/mkimage | Создать волатильный диск, без или с ФС | Подключите волатильный диск к ВМ, с или без файловой системы. |
| tm/mkswap | Создать волатильный диск подкачки | Подключите волатильный диск к ВМ, форматированный как диск подкачки (swap). |
| tm/monitor | Вывод статистики места в хранилище дисков ВМ | Проверьте статистику свободного/занятого места в списке хранилищ дисков ВМ. |
| tm/mv | Мигрировать диск ВМ между хостами | Мигрируйте ВМ между серверами. Правда, с точки зрения хранилища в плагинах Vitastor и Ceph этот скрипт ничего не делает. |
| tm/mvds | Отключить "постоянный" образ от ВМ | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. В целом же скрипт обратный к tm/ln и в других хранилищах он может, например, копировать образ ВМ с диска гипервизора обратно в хранилище. |
| tm/postbackup | Выполняется после бэкапа | По-видимому, скрипт просто удаляет временные файлы после резервного копирования. Так что можно провести его и проверить, что на серверах не осталось временных файлов. |
| tm/postbackup_live | Выполняется после бэкапа запущенной ВМ | То же, что tm/postbackup, но для запущенной ВМ. |
| tm/postmigrate | Выполняется после миграции ВМ | Тестировать нечего. Однако, OpenNebula запускает скрипт только для системного хранилища, поэтому он вызывает аналогичные скрипты для хранилищ других дисков той же ВМ. Помимо этого в плагинах Vitastor и Ceph скрипт ничего не делает. |
| tm/prebackup | Выполнить резервное копирование дисков ВМ | Создайте хранилище резервных копий типа "rsync" → Забэкапьте в него ВМ. |
| tm/prebackup_live | То же самое для запущенной ВМ | То же, что tm/prebackup, но запускает fsfreeze/thaw (остановку доступа к дискам). Так что смысл теста - проведите резервное копирование и проверьте, что данные скопировались консистентно. |
| tm/premigrate | Выполняется перед миграцией ВМ | Тестировать нечего. Аналогично tm/postmigrate запускается только для системного хранилища. |
| tm/resize | Изменить размер диска ВМ | Выберите ВМ → Выберите непостоянный диск → Измените его размер. |
| tm/restore | Восстановить диски ВМ из бэкапа | Создайте хранилище резервных копий → Забэкапьте в него ВМ → Восстановите её обратно. |
| tm/snap_create | Создать снимок диска ВМ | Выберите ВМ → Выберите диск → Создайте снимок. |
| tm/snap_create_live | Создать снимок диска запущенной ВМ | Выберите запущенную ВМ → Выберите диск → Создайте снимок. |
| tm/snap_delete | Удалить снимок диска ВМ | Выберите ВМ → Выберите диск → Выберите снимок → Удалить. |
| tm/snap_revert | Откатить диск ВМ к снимку | Выберите ВМ → Выберите диск → Выберите снимок → Откатить. |

View File

@ -39,6 +39,10 @@
## Plugins and tools
- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
- [OpenNebula storage plugin](../installation/opennebula.en.md)
- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
- [Debian and CentOS packages](../installation/packages.en.md)
- [Image management CLI (vitastor-cli)](../usage/cli.en.md)
- [Disk management CLI (vitastor-disk)](../usage/disk.en.md)
@ -46,9 +50,6 @@
- [Native QEMU driver](../usage/qemu.en.md)
- [Loadable fio engine for benchmarks](../usage/fio.en.md)
- [NBD proxy for kernel mounts](../usage/nbd.en.md)
- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
- [Simplified NFS proxy for file-based image access emulation (suitable for VMWare)](../usage/nfs.en.md#pseudo-fs)
## Roadmap
@ -58,7 +59,6 @@ The following features are planned for the future:
- Control plane optimisation
- Other administrative tools
- Web GUI
- OpenNebula plugin
- iSCSI and NVMeoF gateways
- Multi-threaded client
- Faster failover

View File

@ -41,6 +41,10 @@
## Драйверы и инструменты
- [Плагин для Proxmox](../installation/proxmox.ru.md)
- [Плагин для OpenNebula](../installation/opennebula.ru.md)
- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
- [Пакеты для Debian и CentOS](../installation/packages.ru.md)
- [Консольный интерфейс управления образами (vitastor-cli)](../usage/cli.ru.md)
- [Инструмент управления дисками (vitastor-disk)](../usage/disk.ru.md)
@ -48,9 +52,6 @@
- [Драйвер диска для QEMU](../usage/qemu.ru.md)
- [Драйвер диска для утилиты тестирования производительности fio](../usage/fio.ru.md)
- [NBD-прокси для монтирования образов ядром](../usage/nbd.ru.md) ("блочное устройство в режиме пользователя")
- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
- [Плагин для Proxmox](../installation/proxmox.ru.md)
- [Упрощённая NFS-прокси для эмуляции файлового доступа к образам (подходит для VMWare)](../usage/nfs.ru.md#псевдо-фс)
## Планы развития
@ -58,7 +59,6 @@
- Оптимизация слоя управления
- Другие инструменты администрирования
- Web-интерфейс
- Плагин для OpenNebula
- iSCSI и NVMeoF прокси
- Многопоточный клиент
- Более быстрое переключение при отказах

View File

@ -16,6 +16,7 @@ It supports the following commands:
- [create](#create)
- [snap-create](#create)
- [modify](#modify)
- [dd](#dd)
- [rm](#rm)
- [flatten](#flatten)
- [rm-data](#rm-data)
@ -148,19 +149,60 @@ You should resize file system in the image, if present, before shrinking it.
* `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
* `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
## dd
```
vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
[count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress] \
[conv=nocreat,noerror,nofsync,trunc,nosparse] [iflag=direct] [oflag=direct,append]
```
Copy data between Vitastor images, files and pipes.
Options can be specified in classic dd style (`key=value`) or like usual (`--key value`).
| <!-- --> | <!-- --> |
|-----------------|-------------------------------------------------------------------------|
| `iimg=<image>` | Copy from Vitastor image `<image>` |
| `if=<file>` | Copy from file `<file>` |
| `oimg=<image>` | Copy to Vitastor image `<image>` |
| `of=<file>` | Copy to file `<file>` |
| `bs=1M` | Set copy block size |
| `count=N` | Copy only N input blocks. If N ends in B it counts bytes, not blocks |
| `seek/oseek=N` | Skip N output blocks. If N ends in B it counts bytes, not blocks |
| `skip/iseek=N` | Skip N input blocks. If N ends in B it counts bytes, not blocks |
| `iodepth=N` | Send N reads or writes in parallel (default 4) |
| `status=LEVEL` | The LEVEL of information to print to stderr: none/noxfer/progress |
| `size=N` | Specify size for the created output file/image (defaults to input size) |
| `iflag=direct` | For input files only: use direct I/O |
| `oflag=direct` | For output files only: use direct I/O |
| `oflag=append` | For files only: append to output file |
| `conv=nocreat` | Do not create output file/image |
| `conv=trunc` | Truncate output file/image |
| `conv=noerror` | Continue copying after errors |
| `conv=nofsync` | Do not call fsync before finishing (default behaviour is fsync) |
| `conv=nosparse` | Write all output blocks including all-zero blocks |
## rm
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
Remove `<from>` or all layers between `<from>` and `<to>` (`<to>` must be a child of `<from>`),
rebasing all their children accordingly. --writers-stopped allows merging to be a bit
more effective in case of a single 'slim' read-write child and 'fat' removed parent:
the child is merged into parent and parent is renamed to child in that case.
In other cases parent layers are always merged into children.
`vitastor-cli rm (--exact|--matching) <glob> ...`
Other options:
Remove layer(s) and rebase all their children accordingly.
* `--down-ok` - Continue deletion/merging even if some data will be left on unavailable OSDs.
In the first form, remove `<from>` or layers between `<from>` and its child `<to>`.
In the second form, remove all images with exact or pattern-matched names.
Options:
* `--writers-stopped` allows optimised removal in case of a single 'slim' read-write
child and 'fat' removed parent: the child is merged into parent and parent is renamed
to child in that case. In other cases parent layers are always merged into children.
* `--exact` - remove multiple images with names matching given glob patterns.
* `--matching` - remove multiple images with given names
* `--down-ok` - continue deletion/merging even if some data will be left on unavailable OSDs.
## flatten

View File

@ -17,6 +17,7 @@ vitastor-cli - интерфейс командной строки для адм
- [create](#create)
- [snap-create](#create)
- [modify](#modify)
- [dd](#dd)
- [rm](#rm)
- [flatten](#flatten)
- [rm-data](#rm-data)
@ -151,23 +152,61 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
* `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
* `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
## dd
```
vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
[count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress] \
[conv=nocreat,noerror,nofsync,trunc,nosparse] [iflag=direct] [oflag=direct,append]
```
Копировать данные между образами Vitastor, файлами и каналами.
Опции можно передавать в классическом стиле dd (`key=value`) или как обычно (`--key value`).
| <!-- --> | <!-- --> |
|-----------------|-------------------------------------------------------------------------|
| `iimg=<image>` | Копировать из образа Vitastor `<image>` |
| `if=<file>` | Копировать из файла `<file>` |
| `oimg=<image>` | Копировать в образ Vitastor `<image>` |
| `of=<file>` | Копировать в файл `<file>` |
| `bs=1M` | Задать размер блока копирования |
| `count=N` | Копировать не более N блоков. Если N заканчивается на B - то N байт. |
| `seek/oseek=N` | Пропустить N выходных блоков. Если N заканчивается на B - то N байт. |
| `skip/iseek=N` | Пропустить N входных блоков. Если N заканчивается на B - то N байт. |
| `iodepth=N` | Отправлять N чтений/записей параллельно (по умолчанию 4). |
| `status=LEVEL` | Уровень вывода в консоль: none/noxfer/progress |
| `size=N` | Задать размер выходного файла/образа (по умолчанию равен размеру входа).|
| `iflag=direct` | Только для входного файла: использовать прямой ввод-вывод |
| `oflag=direct` | Только для выходного файла: использовать прямой ввод-вывод |
| `oflag=append` | Только для файлов: дописывать в конец выходного файла |
| `conv=nocreat` | Не создавать выходной файл/образ |
| `conv=trunc` | Обрезать выходной файл/образ до размера входа |
| `conv=noerror` | Продолжать копирование после ошибок |
| `conv=nofsync` | Не вызывать fsync перед завершением |
| `conv=nosparse` | Записывать все выходные блоки, включая пустые |
## rm
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
Удалить образ `<from>` или все слои от `<from>` до `<to>` (`<to>` должен быть дочерним
образом `<from>`), одновременно меняя родительские образы их клонов (если таковые есть).
`vitastor-cli rm (--exact|--matching) <glob> ...`
`--writers-stopped` позволяет чуть более эффективно удалять образы в частом случае, когда
у удаляемой цепочки есть только один дочерний образ, содержащий небольшой объём данных.
В этом случае дочерний образ вливается в родительский и удаляется, а родительский
переименовывается в дочерний.
Удалить образ(ы), корректно перебазируя их дочерние образы.
В других случаях родительские слои вливаются в дочерние.
В первой форме удаляет один образ `<from>` или все слои между `<from>` и его дочерним `<to>`.
Другие опции:
Во второй форме, удаляет все образы с точными именами или именами, подходящими под шаблон(ы).
* `--down-ok` - Продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
Опции:
* `--writers-stopped` позволяет чуть более эффективно удалять образы в частом случае, когда
у удаляемой цепочки есть только один дочерний образ, содержащий небольшой объём данных.
В этом случае дочерний образ вливается в родительский и удаляется, а родительский
переименовывается в дочерний.
* `--exact` - удалить все образы с именами, подходящими под переданные glob-шаблоны.
* `--matching` - удалить все образы с точно заданными именами.
* `--down-ok` - продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
## flatten

View File

@ -1,6 +1,6 @@
{
"name": "vitastor-mon",
"version": "1.8.0",
"version": "1.9.0",
"description": "Vitastor SDS monitor service",
"main": "mon-main.js",
"scripts": {

View File

@ -11,10 +11,10 @@
#define ERRORF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
#define TRACEF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
#define TRACE(msg) fprintf(stderr, "%s\n", msg);
//#define TRACEF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
//#define TRACE(msg) fprintf(stderr, "%s\n", msg);
//#define TRACEF(format, arg) ;
//#define TRACE(msg) ;
#define TRACEF(format, ...) ;
#define TRACE(msg) ;
#endif

1
opennebula/README.md Normal file
View File

@ -0,0 +1 @@
See [../docs/installation/opennebula.en.md](../docs/installation/opennebula.en.md).

29
opennebula/install.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
set -e
reapply_patch() {
if ! patch -f --dry-run -F 0 -R $1 < $2 >/dev/null; then
already_applied=0
if ! patch --no-backup-if-mismatch -r - -F 0 -f $1 < $2; then
applied_ok=0
echo "ERROR: Failed to patch file $1, please apply the patch $2 manually"
fi
fi
}
echo "Reapplying Vitastor patches to OpenNebula's oned.conf, vmm_execrc and downloader.sh"
already_applied=1
applied_ok=1
reapply_patch /var/lib/one/remotes/datastore/downloader.sh /var/lib/one/remotes/datastore/vitastor/downloader-vitastor.sh.diff
reapply_patch /etc/one/oned.conf /var/lib/one/remotes/datastore/vitastor/oned.conf.diff
reapply_patch /etc/one/vmm_exec/vmm_execrc /var/lib/one/remotes/datastore/vitastor/vmm_execrc.diff
if [[ "$already_applied" = 1 ]]; then
echo "OK: Vitastor OpenNebula patches are already applied"
elif [[ "$applied_ok" = 1 ]]; then
echo "OK: Vitastor OpenNebula patches successfully applied"
fi
if [[ -f /etc/apparmor.d/local/abstractions/libvirt-qemu ]]; then
if ! grep -q /etc/vitastor/vitastor.conf /etc/apparmor.d/local/abstractions/libvirt-qemu; then
echo ' "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu
fi
fi

View File

@ -0,0 +1,76 @@
#!/bin/bash
# Vitastor OpenNebula driver
# Copyright (c) Vitaliy Filippov, 2024+
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
# This script is used to copy a VM image (SRC) to the image repository as DST
# -------- Set up the environment to source common tools & conf ------------
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
fi
. $LIB_LOCATION/sh/scripts_common.sh
DRIVER_PATH=$(dirname $0)
source ${DRIVER_PATH}/../libfs.sh
# -------- Get cp and datastore arguments from OpenNebula core ------------
DRV_ACTION=`cat -`
ID=$1
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
unset i XPATH_ELEMENTS
while IFS= read -r -d '' element; do
XPATH_ELEMENTS[i++]="$element"
done < <($XPATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
unset i
BASE_PATH="${XPATH_ELEMENTS[i++]}"
BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
POOL_NAME="${XPATH_ELEMENTS[i++]}"
IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
SRC="${XPATH_ELEMENTS[i++]}"
SIZE="${XPATH_ELEMENTS[i++]}"
VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
DST_HOST=`get_destination_host $ID`
if [ -z "$DST_HOST" ]; then
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
exit -1
fi
CLI=vitastor-cli
if [ -n "$VITASTOR_CONF" ]; then
CLI="$CLI --config_path ${VITASTOR_CONF}"
fi
if [ -n "$POOL_NAME" ]; then
CLI="$CLI --pool ${POOL_NAME}"
fi
SAFE_DIRS=""
DST="${IMAGE_PREFIX}-${ID}"
ssh_exec_and_log "$DST_HOST" "$CLI create --parent $SRC $DST" \
"Error during $CLI create --parent $SRC $DST in $DST_HOST"
ssh_exec_and_log "$DST_HOST" "$CLI flatten $DST" \
"Error during $CLI create flatten $DST in $DST_HOST"
echo "$DST raw"

View File

@ -0,0 +1,135 @@
#!/bin/bash
# Vitastor OpenNebula driver
# Copyright (c) Vitaliy Filippov, 2024+
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
# This script is used to copy a local image SRC to the image repository as DST
# -------- Set up the environment to source common tools & conf ------------
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
fi
. $LIB_LOCATION/sh/scripts_common.sh
DRIVER_PATH=$(dirname $0)
source ${DRIVER_PATH}/../libfs.sh
# -------- Get cp and datastore arguments from OpenNebula core ------------
DRV_ACTION=`cat -`
ID=$1
export DRV_ACTION
UTILS_PATH="${DRIVER_PATH}/.."
XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION"
unset i XPATH_ELEMENTS
while IFS= read -r -d '' element; do
XPATH_ELEMENTS[i++]="$element"
done < <($XPATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/STAGING_DIR \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
unset i
BASE_PATH="${XPATH_ELEMENTS[i++]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}"
SAFE_DIRS="${XPATH_ELEMENTS[i++]}"
BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
POOL_NAME="${XPATH_ELEMENTS[i++]}"
IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
STAGING_DIR="${XPATH_ELEMENTS[i++]:-/var/tmp}"
SRC="${XPATH_ELEMENTS[i++]}"
SIZE="${XPATH_ELEMENTS[i++]}"
MD5="${XPATH_ELEMENTS[i++]}"
SHA1="${XPATH_ELEMENTS[i++]}"
NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}"
LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}"
VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
DST_HOST=`get_destination_host $ID`
if [ -z "$DST_HOST" ]; then
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
exit -1
fi
CLI=vitastor-cli
QEMU_ARG=""
if [ -n "$VITASTOR_CONF" ]; then
CLI="$CLI --config_path ${VITASTOR_CONF}"
QEMU_ARG=":config_path=${VITASTOR_CONF}"
fi
if [ -n "$POOL_NAME" ]; then
CLI="$CLI --pool ${POOL_NAME}"
fi
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS"
IMAGE_HASH=`generate_image_hash`
TMP_DST="$STAGING_DIR/$IMAGE_HASH"
DST="${IMAGE_PREFIX}-${ID}"
DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
case $SRC in
http://*|https://*)
log "Downloading $SRC to the image repository"
DUMP="$COPY_COMMAND"
;;
*)
if [ `check_restricted $SRC` -eq 1 ]; then
log_error "Not allowed to copy images from $RESTRICTED_DIRS"
error_message "Not allowed to copy image file $SRC"
exit -1
fi
log "Copying local image $SRC to the image repository"
DUMP="$COPY_COMMAND"
;;
esac
multiline_exec_and_log "set -e -o pipefail; $DUMP | $SSH $DST_HOST $DD of=$TMP_DST bs=1M" \
"Error copying $SRC to $DST_HOST:$TMP_DST"
REGISTER_CMD=$(cat <<EOF
set -e -o pipefail
SIZE=\$($QEMU_IMG info --output json "$TMP_DST" | jq -r '.["virtual-size"]')
$CLI create -s \$SIZE "$DST"
$QEMU_IMG convert -O raw "$TMP_DST" "vitastor:image=$DST$QEMU_ARG"
# remove original
$RM -f $TMP_DST
EOF
)
ssh_exec_and_log "$DST_HOST" "$REGISTER_CMD" "Error registering $DST in $DST_HOST"
echo "$DST raw"

View File

@ -0,0 +1,555 @@
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
VAR_LOCATION=/var/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
VAR_LOCATION=$ONE_LOCATION/var
fi
. $LIB_LOCATION/sh/scripts_common.sh
DRIVER_PATH=$(dirname $0)
# Escape single quotes
function esc_sq
{
echo "$1" | sed -e "s/'/'\\\''/g"
}
# Execute a command (first parameter) and use the first kb of stdout
# to determine the file type
function get_type
{
if [ "$NO_DECOMPRESS" = "yes" ]; then
echo "application/octet-stream"
else
command=$1
( eval "$command" | head -n 1024 | file -b --mime-type - ) 2>/dev/null
fi
}
# Gets the command needed to decompress an stream.
function get_decompressor
{
type=$1
case "$type" in
"application/x-gzip"|"application/gzip")
echo "gunzip -c -"
;;
"application/x-bzip2")
echo "bunzip2 -qc -"
;;
"application/x-xz")
echo "unxz -c -"
;;
*)
echo "cat"
;;
esac
}
# Function called to decompress a stream. The first parameter is the command
# used to decompress the stream. Second parameter is the output file or
# - for stdout.
function decompress
{
command="$1"
to="$2"
if [ "$to" = "-" ]; then
$command
else
$command > "$to"
fi
}
# Function called to hash a stream. First parameter is the algorithm name.
function hasher
{
if [ -n "$1" ]; then
openssl dgst -$1 | awk '{print $NF}' > $HASH_FILE
else
# Needs something consuming stdin or the pipe will break
cat >/dev/null
fi
}
# Unarchives a tar or a zip a file to a directory with the same name.
function unarchive
{
TO="$1"
file_type=$(get_type "cat $TO")
tmp="$TO"
# Add full path if it is relative
if [ ${tmp:0:1} != "/" ]; then
tmp="$PWD/$tmp"
fi
IN="$tmp.tmp"
OUT="$tmp"
case "$file_type" in
"application/x-tar")
command="tar -xf $IN -C $OUT"
;;
"application/zip")
command="unzip -d $OUT $IN"
;;
*)
command=""
;;
esac
if [ -n "$command" ]; then
mv "$OUT" "$IN"
mkdir "$OUT"
$command
if [ "$?" != "0" ]; then
echo "Error uncompressing archive" >&2
exit -1
fi
rm "$IN"
fi
}
function s3_env
{
XPATH="$DRIVER_PATH/xpath.rb -b $DRV_ACTION"
unset i j XPATH_ELEMENTS
while IFS= read -r -d '' element; do
XPATH_ELEMENTS[i++]="$element"
done < <($XPATH /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ACCESS_KEY_ID \
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY \
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/REGION \
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/AWS \
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ENDPOINT)
S3_ACCESS_KEY_ID="${XPATH_ELEMENTS[j++]}"
S3_SECRET_ACCESS_KEY="${XPATH_ELEMENTS[j++]}"
S3_REGION="${XPATH_ELEMENTS[j++]}"
S3_AWS="${XPATH_ELEMENTS[j++]}"
S3_ENDPOINT="${XPATH_ELEMENTS[j++]}"
CURRENT_DATE_DAY="$(date -u '+%Y%m%d')"
CURRENT_DATE_ISO8601="${CURRENT_DATE_DAY}T$(date -u '+%H%M%S')Z"
}
# Create an SHA-256 hash in hexadecimal.
# Usage:
# hash_sha256 <string>
function hash_sha256 {
printf "${1}" | openssl dgst -sha256 | sed 's/^.* //'
}
# Create an SHA-256 hmac in hexadecimal.